From 278e1bc110c736d9087bb6f816e1537fce35bafa Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 4 Jan 2019 12:21:56 -0500
Subject: [PATCH 001/555] reads number of elements

---
 test/CMakeLists.txt          |   1 +
 test/matchedNodeElmReader.cc | 248 +++++++++++++++++++++++++++++++++++
 2 files changed, 249 insertions(+)
 create mode 100644 test/matchedNodeElmReader.cc

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 4f1fff06c..46a359723 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -47,6 +47,7 @@ util_exe_func(from_neper neper.cc)
 util_exe_func(from_ugrid ugrid.cc)
 util_exe_func(nektar_align nektar_align.cc)
 util_exe_func(icesheet icesheet.cc)
+util_exe_func(matchedNodeElmReader matchedNodeElmReader.cc)
 if(ENABLE_OMEGA_H)
   util_exe_func(smb2osh smb2osh.cc)
   util_exe_func(osh2smb osh2smb.cc)
diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
new file mode 100644
index 000000000..95b06a55f
--- /dev/null
+++ b/test/matchedNodeElmReader.cc
@@ -0,0 +1,248 @@
+#include <gmi_mesh.h>
+#include <gmi_null.h>
+#include <apfMDS.h>
+#include <apfMesh2.h>
+#include <apfConvert.h>
+#include <apf.h>
+#include <PCU.h>
+#include <lionPrint.h>
+#include <pcu_util.h>
+#include <cstdlib>
+#include <string.h>
+
+unsigned getElmType(int numVtxPerElm) {
+  if (numVtxPerElm == 4) {
+    return apf::Mesh::TET;
+  } else if (numVtxPerElm == 6) {
+    return apf::Mesh::PRISM;
+  } else {
+    fprintf(stderr, "unknown element type in %s\n", __func__);
+    exit(EXIT_FAILURE);
+  }
+}
+
+bool skipLine(char* line) {
+  // lines that start with either a '#' or a single white space
+  // are skipped
+  return (line[0] == '#' || line[0] == ' ' );
+} 
+
+void getNumElms(FILE* f, unsigned& elms) {
+  size_t linelimit = 1024;
+  char* line = new char[linelimit];
+  while( gmi_getline(&line,&linelimit,f) != -1 ) {
+    if( ! skipLine(line) )
+      elms++;
+  }
+}
+
+void readCoords(FILE* f, unsigned numvtx, double* coordinates) {
+  const double huge = 1024*1024;
+  double min[3] = {huge,huge,huge};
+  double max[3] = {-huge,-huge,-huge};
+  for(unsigned i=0; i<numvtx; i++) {
+    for(unsigned j=0; j<3; j++) {
+      double pos = 0;
+      gmi_fscanf(f, 1, "%lf", &pos);
+      coordinates[i*3+j] = pos;
+      if( pos < min[j] )
+        min[j] = pos;
+      if( pos > max[j] )
+        max[j] = pos;
+    }
+  }
+  char d[3] = {'x','y','z'};
+  fprintf(stderr, "mesh bounding box:\n");
+  for(unsigned i=0; i<3; i++)
+    fprintf(stderr, "%c %lf %lf \n", d[i], min[i], max[i]);
+}
+
+void readElements(FILE* f, unsigned numelms, int numVtxPerElm,
+    unsigned numVerts, int* elements) {
+  unsigned i;
+  std::map<int, int> count;
+  for (i = 0; i < numelms*numVtxPerElm; i++) {
+    int vtxid;
+    gmi_fscanf(f, 1, "%u", &vtxid);
+    elements[i] = --vtxid; //export from matlab using 1-based indices
+    count[elements[i]]++;
+  }
+  PCU_ALWAYS_ASSERT(count.size() == numVerts);
+}
+
+struct MeshInfo {
+  double* coords;
+  int* elements;
+  unsigned elementType;
+  unsigned numVerts;
+  unsigned numElms;
+  unsigned numVtxPerElm;
+};
+
+void readMesh(const char* meshfilename, const char* coordfilename, MeshInfo& mesh) {
+  FILE* f = fopen(meshfilename, "r");
+  PCU_ALWAYS_ASSERT(f);
+  getNumElms(f,mesh.numElms);
+  fprintf(stderr, "numElms %u\n",
+      mesh.numElms);
+  (void)coordfilename;
+  /*
+  mesh.coords = new double[mesh.numVerts*3];
+  FILE* fc = fopen(coordfilename, "r");
+  readCoords(fc, mesh.numVerts, mesh.coords);
+  fclose(fc);
+  mesh.elements = new int [mesh.numElms*mesh.numVtxPerElm];
+  readElements(f, mesh.numElms, mesh.numVtxPerElm, mesh.numVerts, mesh.elements);
+  mesh.elementType = getElmType(mesh.numVtxPerElm);
+  */
+  fclose(f);
+}
+
+int* readIntArray(const char* fname, unsigned len) {
+  FILE* f = fopen(fname, "r");
+  PCU_ALWAYS_ASSERT(f);
+  unsigned n;
+  gmi_fscanf(f, 1, "%u", &n);
+  PCU_ALWAYS_ASSERT( n == len );
+  int* data = new int[len];
+  for(unsigned i = 0; i< len; i++)
+    gmi_fscanf(f, 1, "%d", &data[i]);
+  fclose(f);
+  return data;
+}
+
+double* readScalarArray(const char* fname, unsigned len) {
+  FILE* f = fopen(fname, "r");
+  PCU_ALWAYS_ASSERT(f);
+  unsigned n;
+  gmi_fscanf(f, 1, "%u", &n);
+  PCU_ALWAYS_ASSERT( n == len );
+  double* data = new double[len];
+  for(unsigned i = 0; i< len; i++)
+    gmi_fscanf(f, 1, "%lf", &data[i]);
+  fclose(f);
+  return data;
+}
+
+std::string getName(const char* fname) {
+  using std::string;
+  std::string in(fname);
+  if( in.find("vertex_type") != string::npos ) {
+    return string("vertex_type");
+  } else if( in.find("basal_friction") != string::npos ) {
+    return string("basal_friction");
+  } else if( in.find("temperature") != string::npos ) {
+    return string("temperature");
+  } else if( in.find("surface_elevation") != string::npos ) {
+    return string("surface_height");
+  } else if( in.find("solution_x") != string::npos ) {
+    return string("solution_x");
+  } else if( in.find("solution_y") != string::npos ) {
+    return string("solution_y");
+  } else {
+    fprintf(stderr, "unknown field name in %s\n", __func__);
+    exit(EXIT_FAILURE);
+  }
+}
+
+apf::MeshTag* attachVtxTag(apf::Mesh2* mesh, const char* fname,
+    apf::GlobalToVert& vtxMap) {
+  unsigned numverts = mesh->count(0);
+  std::string fldname = getName(fname);
+  fprintf(stderr, "attaching %s\n", fldname.c_str());
+  int* arr = readIntArray(fname, numverts);
+  apf::MeshTag* tag = mesh->createIntTag(fldname.c_str(), 1);
+  for(unsigned i=0; i<numverts; i++)
+    mesh->setIntTag(vtxMap[i], tag, &(arr[i]));
+  delete [] arr;
+  return tag;
+}
+
+/** \brief attach a field to the vertices
+ *  \detail all the fields being attached need to exist with the mesh so the
+ *  field pointer is not returned
+ */
+void attachVtxField(apf::Mesh2* mesh, const char* fname,
+    apf::GlobalToVert& vtxMap) {
+  std::string fldname = getName(fname);
+  unsigned numverts = mesh->count(0);
+  fprintf(stderr, "attaching %s\n", fldname.c_str());
+  double* arr = readScalarArray(fname, numverts);
+  apf::Field* fld = apf::createFieldOn(mesh,fldname.c_str(),apf::SCALAR);
+  for(unsigned i=0; i<numverts; i++)
+    apf::setScalar(fld,vtxMap[i],0,arr[i]);
+  delete [] arr;
+}
+
+void mergeSolutionFields(apf::Mesh2* mesh) {
+  apf::Field* x = mesh->findField("solution_x");
+  apf::Field* y = mesh->findField("solution_y");
+  apf::Field* xy = apf::createFieldOn(mesh,"Solution",apf::VECTOR);
+  apf::MeshIterator* it = mesh->begin(0);
+  apf::MeshEntity* vtx;
+  while( (vtx = mesh->iterate(it)) ) {
+    apf::Vector3 v;
+    v[0] = apf::getScalar(x,vtx,0);
+    v[1] = apf::getScalar(y,vtx,0);
+    v[2] = 0.0;
+    apf::setVector(xy, vtx, 0, v);
+  }
+  mesh->end(it);
+  apf::destroyField(x);
+  apf::destroyField(y);
+}
+
+void detachVtxTag(apf::Mesh2* mesh, apf::MeshTag* t) {
+  apf::MeshIterator* it = mesh->begin(0);
+  apf::MeshEntity* vtx;
+  while( (vtx = mesh->iterate(it)) )
+    mesh->removeTag(vtx,t);
+  mesh->end(it);
+  mesh->destroyTag(t);
+}
+
+int main(int argc, char** argv)
+{
+  if( argc != 6 ) {
+    printf("Usage: %s <ascii mesh connectivity .cnn> "
+        "<ascii vertex coordinates .crd> "
+        "<ascii vertex matching flag .match> "
+        "<output model .dmg> <output mesh .smb>\n",
+        argv[0]);
+    return 0;
+  }
+
+  MPI_Init(&argc,&argv);
+  PCU_Comm_Init();
+  lion_set_verbosity(1);
+  gmi_register_mesh();
+  gmi_register_null();
+
+  //gmi_model* model = gmi_load(".null");
+
+  MeshInfo m;
+  readMesh(argv[1],argv[2],m);
+
+  /*
+  const int dim = 3;
+  apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, dim, false);
+  apf::GlobalToVert outMap;
+  apf::construct(mesh, m.elements, m.numElms, m.elementType, outMap);
+  delete [] m.elements;
+  apf::alignMdsRemotes(mesh);
+  apf::deriveMdsModel(mesh);
+  apf::setCoords(mesh, m.coords, m.numVerts, outMap);
+  delete [] m.coords;
+  outMap.clear();
+  mesh->verify();
+
+  gmi_write_dmg(model, argv[4]);
+  mesh->writeNative(argv[5]);
+  apf::writeVtkFiles("rendered",mesh);
+
+  mesh->destroyNative();
+  apf::destroyMesh(mesh);
+  */
+  PCU_Comm_Free();
+  MPI_Finalize();
+}

From d68c1eb88cae2fb66906a4b35eaea434f4587271 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 4 Jan 2019 12:48:56 -0500
Subject: [PATCH 002/555] reads coords and connectivity

hardcoded for hexahedron
---
 test/matchedNodeElmReader.cc | 50 ++++++++++++++++++++++++++----------
 1 file changed, 37 insertions(+), 13 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 95b06a55f..eab41b820 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -15,6 +15,8 @@ unsigned getElmType(int numVtxPerElm) {
     return apf::Mesh::TET;
   } else if (numVtxPerElm == 6) {
     return apf::Mesh::PRISM;
+  } else if (numVtxPerElm == 8) {
+    return apf::Mesh::HEX;
   } else {
     fprintf(stderr, "unknown element type in %s\n", __func__);
     exit(EXIT_FAILURE);
@@ -28,6 +30,8 @@ bool skipLine(char* line) {
 } 
 
 void getNumElms(FILE* f, unsigned& elms) {
+  rewind(f);
+  elms = 0;
   size_t linelimit = 1024;
   char* line = new char[linelimit];
   while( gmi_getline(&line,&linelimit,f) != -1 ) {
@@ -36,11 +40,25 @@ void getNumElms(FILE* f, unsigned& elms) {
   }
 }
 
+void getNumVerts(FILE* f, unsigned& verts) {
+  rewind(f);
+  verts = 0;
+  size_t linelimit = 1024;
+  char* line = new char[linelimit];
+  while( gmi_getline(&line,&linelimit,f) != -1 ) {
+    if( ! skipLine(line) )
+      verts++;
+  }
+}
+
 void readCoords(FILE* f, unsigned numvtx, double* coordinates) {
+  rewind(f);
   const double huge = 1024*1024;
   double min[3] = {huge,huge,huge};
   double max[3] = {-huge,-huge,-huge};
   for(unsigned i=0; i<numvtx; i++) {
+    int id;
+    gmi_fscanf(f, 1, "%d", &id);
     for(unsigned j=0; j<3; j++) {
       double pos = 0;
       gmi_fscanf(f, 1, "%lf", &pos);
@@ -57,15 +75,21 @@ void readCoords(FILE* f, unsigned numvtx, double* coordinates) {
     fprintf(stderr, "%c %lf %lf \n", d[i], min[i], max[i]);
 }
 
-void readElements(FILE* f, unsigned numelms, int numVtxPerElm,
+void readElements(FILE* f, unsigned numelms, unsigned numVtxPerElm,
     unsigned numVerts, int* elements) {
-  unsigned i;
+  rewind(f);
+  unsigned i, j;
   std::map<int, int> count;
-  for (i = 0; i < numelms*numVtxPerElm; i++) {
-    int vtxid;
-    gmi_fscanf(f, 1, "%u", &vtxid);
-    elements[i] = --vtxid; //export from matlab using 1-based indices
-    count[elements[i]]++;
+  for (i = 0; i < numelms; i++) {
+    int elmid;
+    gmi_fscanf(f, 1, "%u", &elmid);
+    for (j = 0; j < numVtxPerElm; j++) {
+      int elmVtxIdx = i*numVtxPerElm+j;
+      int vtxid;
+      gmi_fscanf(f, 1, "%u", &vtxid);
+      elements[elmVtxIdx] = --vtxid; //export from matlab using 1-based indices
+      count[elements[elmVtxIdx]]++;
+    }
   }
   PCU_ALWAYS_ASSERT(count.size() == numVerts);
 }
@@ -82,19 +106,19 @@ struct MeshInfo {
 void readMesh(const char* meshfilename, const char* coordfilename, MeshInfo& mesh) {
   FILE* f = fopen(meshfilename, "r");
   PCU_ALWAYS_ASSERT(f);
+  FILE* fc = fopen(coordfilename, "r");
+  PCU_ALWAYS_ASSERT(fc);
   getNumElms(f,mesh.numElms);
-  fprintf(stderr, "numElms %u\n",
-      mesh.numElms);
-  (void)coordfilename;
-  /*
+  getNumVerts(fc,mesh.numVerts);
+  fprintf(stderr, "numElms %u numVerts %u\n",
+      mesh.numElms, mesh.numVerts);
   mesh.coords = new double[mesh.numVerts*3];
-  FILE* fc = fopen(coordfilename, "r");
   readCoords(fc, mesh.numVerts, mesh.coords);
   fclose(fc);
+  mesh.numVtxPerElm = 8; //hack!
   mesh.elements = new int [mesh.numElms*mesh.numVtxPerElm];
   readElements(f, mesh.numElms, mesh.numVtxPerElm, mesh.numVerts, mesh.elements);
   mesh.elementType = getElmType(mesh.numVtxPerElm);
-  */
   fclose(f);
 }
 

From 24e2366e00d0809898161dcad2c57dd6b3275bc9 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 4 Jan 2019 12:51:07 -0500
Subject: [PATCH 003/555] creates a mesh

---
 test/matchedNodeElmReader.cc | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index eab41b820..5a9f06957 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -242,12 +242,11 @@ int main(int argc, char** argv)
   gmi_register_mesh();
   gmi_register_null();
 
-  //gmi_model* model = gmi_load(".null");
+  gmi_model* model = gmi_load(".null");
 
   MeshInfo m;
   readMesh(argv[1],argv[2],m);
 
-  /*
   const int dim = 3;
   apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, dim, false);
   apf::GlobalToVert outMap;
@@ -266,7 +265,6 @@ int main(int argc, char** argv)
 
   mesh->destroyNative();
   apf::destroyMesh(mesh);
-  */
   PCU_Comm_Free();
   MPI_Finalize();
 }

From 8119236ed2ea486330dd984cddd6df43cbc316cf Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 4 Jan 2019 14:50:30 -0500
Subject: [PATCH 004/555] remove unused fns

---
 test/matchedNodeElmReader.cc | 102 -----------------------------------
 1 file changed, 102 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 5a9f06957..069bc2e32 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -122,108 +122,6 @@ void readMesh(const char* meshfilename, const char* coordfilename, MeshInfo& mes
   fclose(f);
 }
 
-int* readIntArray(const char* fname, unsigned len) {
-  FILE* f = fopen(fname, "r");
-  PCU_ALWAYS_ASSERT(f);
-  unsigned n;
-  gmi_fscanf(f, 1, "%u", &n);
-  PCU_ALWAYS_ASSERT( n == len );
-  int* data = new int[len];
-  for(unsigned i = 0; i< len; i++)
-    gmi_fscanf(f, 1, "%d", &data[i]);
-  fclose(f);
-  return data;
-}
-
-double* readScalarArray(const char* fname, unsigned len) {
-  FILE* f = fopen(fname, "r");
-  PCU_ALWAYS_ASSERT(f);
-  unsigned n;
-  gmi_fscanf(f, 1, "%u", &n);
-  PCU_ALWAYS_ASSERT( n == len );
-  double* data = new double[len];
-  for(unsigned i = 0; i< len; i++)
-    gmi_fscanf(f, 1, "%lf", &data[i]);
-  fclose(f);
-  return data;
-}
-
-std::string getName(const char* fname) {
-  using std::string;
-  std::string in(fname);
-  if( in.find("vertex_type") != string::npos ) {
-    return string("vertex_type");
-  } else if( in.find("basal_friction") != string::npos ) {
-    return string("basal_friction");
-  } else if( in.find("temperature") != string::npos ) {
-    return string("temperature");
-  } else if( in.find("surface_elevation") != string::npos ) {
-    return string("surface_height");
-  } else if( in.find("solution_x") != string::npos ) {
-    return string("solution_x");
-  } else if( in.find("solution_y") != string::npos ) {
-    return string("solution_y");
-  } else {
-    fprintf(stderr, "unknown field name in %s\n", __func__);
-    exit(EXIT_FAILURE);
-  }
-}
-
-apf::MeshTag* attachVtxTag(apf::Mesh2* mesh, const char* fname,
-    apf::GlobalToVert& vtxMap) {
-  unsigned numverts = mesh->count(0);
-  std::string fldname = getName(fname);
-  fprintf(stderr, "attaching %s\n", fldname.c_str());
-  int* arr = readIntArray(fname, numverts);
-  apf::MeshTag* tag = mesh->createIntTag(fldname.c_str(), 1);
-  for(unsigned i=0; i<numverts; i++)
-    mesh->setIntTag(vtxMap[i], tag, &(arr[i]));
-  delete [] arr;
-  return tag;
-}
-
-/** \brief attach a field to the vertices
- *  \detail all the fields being attached need to exist with the mesh so the
- *  field pointer is not returned
- */
-void attachVtxField(apf::Mesh2* mesh, const char* fname,
-    apf::GlobalToVert& vtxMap) {
-  std::string fldname = getName(fname);
-  unsigned numverts = mesh->count(0);
-  fprintf(stderr, "attaching %s\n", fldname.c_str());
-  double* arr = readScalarArray(fname, numverts);
-  apf::Field* fld = apf::createFieldOn(mesh,fldname.c_str(),apf::SCALAR);
-  for(unsigned i=0; i<numverts; i++)
-    apf::setScalar(fld,vtxMap[i],0,arr[i]);
-  delete [] arr;
-}
-
-void mergeSolutionFields(apf::Mesh2* mesh) {
-  apf::Field* x = mesh->findField("solution_x");
-  apf::Field* y = mesh->findField("solution_y");
-  apf::Field* xy = apf::createFieldOn(mesh,"Solution",apf::VECTOR);
-  apf::MeshIterator* it = mesh->begin(0);
-  apf::MeshEntity* vtx;
-  while( (vtx = mesh->iterate(it)) ) {
-    apf::Vector3 v;
-    v[0] = apf::getScalar(x,vtx,0);
-    v[1] = apf::getScalar(y,vtx,0);
-    v[2] = 0.0;
-    apf::setVector(xy, vtx, 0, v);
-  }
-  mesh->end(it);
-  apf::destroyField(x);
-  apf::destroyField(y);
-}
-
-void detachVtxTag(apf::Mesh2* mesh, apf::MeshTag* t) {
-  apf::MeshIterator* it = mesh->begin(0);
-  apf::MeshEntity* vtx;
-  while( (vtx = mesh->iterate(it)) )
-    mesh->removeTag(vtx,t);
-  mesh->end(it);
-  mesh->destroyTag(t);
-}
 
 int main(int argc, char** argv)
 {

From 1c1745a8ae935825b2b0aba3597bcd3782ee3887 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 4 Jan 2019 15:13:38 -0500
Subject: [PATCH 005/555] attempt to support serial matching

fails in verify
---
 test/matchedNodeElmReader.cc | 38 +++++++++++++++++++++++++++++++++---
 1 file changed, 35 insertions(+), 3 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 069bc2e32..6c9320bf5 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -75,6 +75,17 @@ void readCoords(FILE* f, unsigned numvtx, double* coordinates) {
     fprintf(stderr, "%c %lf %lf \n", d[i], min[i], max[i]);
 }
 
+void readMatches(FILE* f, unsigned numvtx, int* matches) {
+  rewind(f);
+  for(unsigned i=0; i<numvtx; i++) {
+    int ignored, matchedVtx;
+    gmi_fscanf(f, 2, "%d %d", &ignored, &matchedVtx); //export from matlab using 1-based indices
+    PCU_ALWAYS_ASSERT( matchedVtx == -1 ||
+        ( matchedVtx > 1 && matchedVtx <= static_cast<int>(numvtx) ));
+    matches[i] = matchedVtx--;
+  }
+}
+
 void readElements(FILE* f, unsigned numelms, unsigned numVtxPerElm,
     unsigned numVerts, int* elements) {
   rewind(f);
@@ -97,13 +108,17 @@ void readElements(FILE* f, unsigned numelms, unsigned numVtxPerElm,
 struct MeshInfo {
   double* coords;
   int* elements;
+  int* matches;
   unsigned elementType;
   unsigned numVerts;
   unsigned numElms;
   unsigned numVtxPerElm;
 };
 
-void readMesh(const char* meshfilename, const char* coordfilename, MeshInfo& mesh) {
+void readMesh(const char* meshfilename,
+    const char* coordfilename,
+    const char* matchfilename,
+    MeshInfo& mesh) {
   FILE* f = fopen(meshfilename, "r");
   PCU_ALWAYS_ASSERT(f);
   FILE* fc = fopen(coordfilename, "r");
@@ -115,6 +130,11 @@ void readMesh(const char* meshfilename, const char* coordfilename, MeshInfo& mes
   mesh.coords = new double[mesh.numVerts*3];
   readCoords(fc, mesh.numVerts, mesh.coords);
   fclose(fc);
+  FILE* fm = fopen(matchfilename, "r");
+  PCU_ALWAYS_ASSERT(fm);
+  mesh.matches = new int[mesh.numVerts];
+  readMatches(fm, mesh.numVerts, mesh.matches);
+  fclose(fm);
   mesh.numVtxPerElm = 8; //hack!
   mesh.elements = new int [mesh.numElms*mesh.numVtxPerElm];
   readElements(f, mesh.numElms, mesh.numVtxPerElm, mesh.numVerts, mesh.elements);
@@ -122,6 +142,15 @@ void readMesh(const char* meshfilename, const char* coordfilename, MeshInfo& mes
   fclose(f);
 }
 
+void setMatches(apf::Mesh2* m, unsigned numVerts, int* matches,
+    apf::GlobalToVert& globToVtx) {
+  int self = PCU_Comm_Self();
+  for(unsigned i=0; i<numVerts; i++) {
+    if( matches[i] != -1 ) {
+      m->addMatch(globToVtx[i], self, globToVtx[matches[i]]);
+    }
+  }
+}
 
 int main(int argc, char** argv)
 {
@@ -143,10 +172,10 @@ int main(int argc, char** argv)
   gmi_model* model = gmi_load(".null");
 
   MeshInfo m;
-  readMesh(argv[1],argv[2],m);
+  readMesh(argv[1],argv[2],argv[3],m);
 
   const int dim = 3;
-  apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, dim, false);
+  apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, dim, true);
   apf::GlobalToVert outMap;
   apf::construct(mesh, m.elements, m.numElms, m.elementType, outMap);
   delete [] m.elements;
@@ -154,7 +183,10 @@ int main(int argc, char** argv)
   apf::deriveMdsModel(mesh);
   apf::setCoords(mesh, m.coords, m.numVerts, outMap);
   delete [] m.coords;
+  setMatches(mesh,m.numVerts,m.matches,outMap);
+  delete [] m.matches;
   outMap.clear();
+  fprintf(stderr, "verifying mesh...\n");
   mesh->verify();
 
   gmi_write_dmg(model, argv[4]);

From e47d3ced06fd6d8ff120df80d8c59ec477e9443d Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 4 Jan 2019 16:02:55 -0500
Subject: [PATCH 006/555] optional matching

---
 test/matchedNodeElmReader.cc | 29 ++++++++++++++++++++---------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 6c9320bf5..5d1fb48e6 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -27,7 +27,7 @@ bool skipLine(char* line) {
   // lines that start with either a '#' or a single white space
   // are skipped
   return (line[0] == '#' || line[0] == ' ' );
-} 
+}
 
 void getNumElms(FILE* f, unsigned& elms) {
   rewind(f);
@@ -130,11 +130,13 @@ void readMesh(const char* meshfilename,
   mesh.coords = new double[mesh.numVerts*3];
   readCoords(fc, mesh.numVerts, mesh.coords);
   fclose(fc);
-  FILE* fm = fopen(matchfilename, "r");
-  PCU_ALWAYS_ASSERT(fm);
-  mesh.matches = new int[mesh.numVerts];
-  readMatches(fm, mesh.numVerts, mesh.matches);
-  fclose(fm);
+  if( strcmp(matchfilename, "NULL") ) {
+    FILE* fm = fopen(matchfilename, "r");
+    PCU_ALWAYS_ASSERT(fm);
+    mesh.matches = new int[mesh.numVerts];
+    readMatches(fm, mesh.numVerts, mesh.matches);
+    fclose(fm);
+  }
   mesh.numVtxPerElm = 8; //hack!
   mesh.elements = new int [mesh.numElms*mesh.numVtxPerElm];
   readElements(f, mesh.numElms, mesh.numVtxPerElm, mesh.numVerts, mesh.elements);
@@ -174,8 +176,15 @@ int main(int argc, char** argv)
   MeshInfo m;
   readMesh(argv[1],argv[2],argv[3],m);
 
+  bool isMatched = true;
+  if( !strcmp(argv[3], "NULL") )
+    isMatched = false;
+
+  if(!PCU_Comm_Self())
+    fprintf(stderr, "isMatched %d\n", isMatched);
+
   const int dim = 3;
-  apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, dim, true);
+  apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, dim, isMatched);
   apf::GlobalToVert outMap;
   apf::construct(mesh, m.elements, m.numElms, m.elementType, outMap);
   delete [] m.elements;
@@ -183,8 +192,10 @@ int main(int argc, char** argv)
   apf::deriveMdsModel(mesh);
   apf::setCoords(mesh, m.coords, m.numVerts, outMap);
   delete [] m.coords;
-  setMatches(mesh,m.numVerts,m.matches,outMap);
-  delete [] m.matches;
+  if( isMatched ) {
+    setMatches(mesh,m.numVerts,m.matches,outMap);
+    delete [] m.matches;
+  }
   outMap.clear();
   fprintf(stderr, "verifying mesh...\n");
   mesh->verify();

From 9a04db5e1d44af6b3598c5a107ac63d44f1ee7b1 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 4 Jan 2019 16:03:21 -0500
Subject: [PATCH 007/555] fix memory leak

---
 test/matchedNodeElmReader.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 5d1fb48e6..155ba1655 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -38,6 +38,7 @@ void getNumElms(FILE* f, unsigned& elms) {
     if( ! skipLine(line) )
       elms++;
   }
+  delete [] line;
 }
 
 void getNumVerts(FILE* f, unsigned& verts) {
@@ -49,6 +50,7 @@ void getNumVerts(FILE* f, unsigned& verts) {
     if( ! skipLine(line) )
       verts++;
   }
+  delete [] line;
 }
 
 void readCoords(FILE* f, unsigned numvtx, double* coordinates) {

From 5702f6755298da8d00cee87db6ea4f6d60621011 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 4 Jan 2019 16:06:01 -0500
Subject: [PATCH 008/555] remove print stmt

---
 test/matchedNodeElmReader.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 155ba1655..064bbb965 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -199,7 +199,6 @@ int main(int argc, char** argv)
     delete [] m.matches;
   }
   outMap.clear();
-  fprintf(stderr, "verifying mesh...\n");
   mesh->verify();
 
   gmi_write_dmg(model, argv[4]);

From 4e29dc5cd7b6a3af80dc28e02de2e2f7fdb182cd Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 4 Jan 2019 17:37:23 -0500
Subject: [PATCH 009/555] each process reads a portion of the coordinates

---
 test/matchedNodeElmReader.cc | 42 +++++++++++++++++++-----------------
 1 file changed, 22 insertions(+), 20 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 064bbb965..32adc1ff4 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -53,28 +53,30 @@ void getNumVerts(FILE* f, unsigned& verts) {
   delete [] line;
 }
 
-void readCoords(FILE* f, unsigned numvtx, double* coordinates) {
+void readCoords(FILE* f, unsigned numvtx, unsigned& localnumvtx, double** coordinates) {
+  const int self = PCU_Comm_Self();
+  const int peers = PCU_Comm_Peers();
+  localnumvtx = numvtx/peers;
+  if( self == peers-1 ) //last rank
+    if( localnumvtx*peers < numvtx )
+      localnumvtx += numvtx - localnumvtx*peers;
+  const long firstVtx = PCU_Exscan_Long(localnumvtx);
+  const long lastVtx = firstVtx+localnumvtx;
+  fprintf(stderr, "%d localnumvtx %u firstVtx %ld lastVtx %ld\n",
+      self, localnumvtx, firstVtx, lastVtx);
+  *coordinates = new double[localnumvtx*3];
   rewind(f);
-  const double huge = 1024*1024;
-  double min[3] = {huge,huge,huge};
-  double max[3] = {-huge,-huge,-huge};
+  int vidx = 0;
   for(unsigned i=0; i<numvtx; i++) {
     int id;
-    gmi_fscanf(f, 1, "%d", &id);
-    for(unsigned j=0; j<3; j++) {
-      double pos = 0;
-      gmi_fscanf(f, 1, "%lf", &pos);
-      coordinates[i*3+j] = pos;
-      if( pos < min[j] )
-        min[j] = pos;
-      if( pos > max[j] )
-        max[j] = pos;
+    double pos[3];
+    gmi_fscanf(f, 4, "%d %lf %lf %lf", &id, pos+0, pos+1, pos+2);
+    if( i > firstVtx && i < lastVtx ) {
+      for(unsigned j=0; j<3; j++)
+        (*coordinates)[vidx*3+j] = pos[j];
+      vidx++;
     }
   }
-  char d[3] = {'x','y','z'};
-  fprintf(stderr, "mesh bounding box:\n");
-  for(unsigned i=0; i<3; i++)
-    fprintf(stderr, "%c %lf %lf \n", d[i], min[i], max[i]);
 }
 
 void readMatches(FILE* f, unsigned numvtx, int* matches) {
@@ -113,6 +115,7 @@ struct MeshInfo {
   int* matches;
   unsigned elementType;
   unsigned numVerts;
+  unsigned localNumVerts;
   unsigned numElms;
   unsigned numVtxPerElm;
 };
@@ -129,8 +132,7 @@ void readMesh(const char* meshfilename,
   getNumVerts(fc,mesh.numVerts);
   fprintf(stderr, "numElms %u numVerts %u\n",
       mesh.numElms, mesh.numVerts);
-  mesh.coords = new double[mesh.numVerts*3];
-  readCoords(fc, mesh.numVerts, mesh.coords);
+  readCoords(fc, mesh.numVerts, mesh.localNumVerts, &(mesh.coords));
   fclose(fc);
   if( strcmp(matchfilename, "NULL") ) {
     FILE* fm = fopen(matchfilename, "r");
@@ -192,7 +194,7 @@ int main(int argc, char** argv)
   delete [] m.elements;
   apf::alignMdsRemotes(mesh);
   apf::deriveMdsModel(mesh);
-  apf::setCoords(mesh, m.coords, m.numVerts, outMap);
+  apf::setCoords(mesh, m.coords, m.localNumVerts, outMap);
   delete [] m.coords;
   if( isMatched ) {
     setMatches(mesh,m.numVerts,m.matches,outMap);

From aece31a3bdbba26813aca9a65995d675161b76ca Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 4 Jan 2019 19:55:06 -0500
Subject: [PATCH 010/555] helper function for ownership range

---
 test/matchedNodeElmReader.cc | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 32adc1ff4..2c9023660 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -10,6 +10,18 @@
 #include <cstdlib>
 #include <string.h>
 
+void getLocalRange(unsigned total, unsigned& local,
+    long& first, long& last) {
+  const int self = PCU_Comm_Self();
+  const int peers = PCU_Comm_Peers();
+  local = total/peers;
+  if( self == peers-1 ) //last rank
+    if( local*peers < total )
+      local += total - local*peers;
+  first = PCU_Exscan_Long(local);
+  last = first+local;
+}
+
 unsigned getElmType(int numVtxPerElm) {
   if (numVtxPerElm == 4) {
     return apf::Mesh::TET;
@@ -54,16 +66,10 @@ void getNumVerts(FILE* f, unsigned& verts) {
 }
 
 void readCoords(FILE* f, unsigned numvtx, unsigned& localnumvtx, double** coordinates) {
-  const int self = PCU_Comm_Self();
-  const int peers = PCU_Comm_Peers();
-  localnumvtx = numvtx/peers;
-  if( self == peers-1 ) //last rank
-    if( localnumvtx*peers < numvtx )
-      localnumvtx += numvtx - localnumvtx*peers;
-  const long firstVtx = PCU_Exscan_Long(localnumvtx);
-  const long lastVtx = firstVtx+localnumvtx;
+  long firstVtx, lastVtx;
+  getLocalRange(numvtx, localnumvtx,firstVtx,lastVtx);
   fprintf(stderr, "%d localnumvtx %u firstVtx %ld lastVtx %ld\n",
-      self, localnumvtx, firstVtx, lastVtx);
+      PCU_Comm_Self(), localnumvtx, firstVtx, lastVtx);
   *coordinates = new double[localnumvtx*3];
   rewind(f);
   int vidx = 0;

From 9a43859933d917e7cd9c1d95321404679c631b69 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 4 Jan 2019 20:19:40 -0500
Subject: [PATCH 011/555] executes in parallel

problems with coordinates appear in vtu files
---
 test/matchedNodeElmReader.cc | 37 ++++++++++++++++++++++--------------
 1 file changed, 23 insertions(+), 14 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 2c9023660..e623ab346 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -97,22 +97,30 @@ void readMatches(FILE* f, unsigned numvtx, int* matches) {
 }
 
 void readElements(FILE* f, unsigned numelms, unsigned numVtxPerElm,
-    unsigned numVerts, int* elements) {
+    unsigned& localNumElms, int** elements) {
+  long firstElm, lastElm;
+  getLocalRange(numelms, localNumElms, firstElm, lastElm);
+  fprintf(stderr, "%d localNumElms %u firstElm %ld lastElm %ld\n",
+      PCU_Comm_Self(), localNumElms, firstElm, lastElm);
+  *elements = new int[localNumElms*numVtxPerElm];
   rewind(f);
   unsigned i, j;
-  std::map<int, int> count;
+  unsigned elmIdx = 0;
+  int* elmVtx = new int[numVtxPerElm];
   for (i = 0; i < numelms; i++) {
-    int elmid;
-    gmi_fscanf(f, 1, "%u", &elmid);
-    for (j = 0; j < numVtxPerElm; j++) {
-      int elmVtxIdx = i*numVtxPerElm+j;
-      int vtxid;
-      gmi_fscanf(f, 1, "%u", &vtxid);
-      elements[elmVtxIdx] = --vtxid; //export from matlab using 1-based indices
-      count[elements[elmVtxIdx]]++;
+    int ignored;
+    gmi_fscanf(f, 1, "%u", &ignored);
+    for (j = 0; j < numVtxPerElm; j++)
+      gmi_fscanf(f, 1, "%u", elmVtx+j);
+    if (i >= firstElm && i < lastElm) {
+      for (j = 0; j < numVtxPerElm; j++) {
+        const unsigned elmVtxIdx = elmIdx*numVtxPerElm+j;
+        (*elements)[elmVtxIdx] = --(elmVtx[j]); //export from matlab using 1-based indices
+      }
+      elmIdx++;
     }
   }
-  PCU_ALWAYS_ASSERT(count.size() == numVerts);
+  delete [] elmVtx;
 }
 
 struct MeshInfo {
@@ -123,6 +131,7 @@ struct MeshInfo {
   unsigned numVerts;
   unsigned localNumVerts;
   unsigned numElms;
+  unsigned localNumElms;
   unsigned numVtxPerElm;
 };
 
@@ -148,8 +157,8 @@ void readMesh(const char* meshfilename,
     fclose(fm);
   }
   mesh.numVtxPerElm = 8; //hack!
-  mesh.elements = new int [mesh.numElms*mesh.numVtxPerElm];
-  readElements(f, mesh.numElms, mesh.numVtxPerElm, mesh.numVerts, mesh.elements);
+  readElements(f, mesh.numElms, mesh.numVtxPerElm,
+      mesh.localNumElms, &(mesh.elements));
   mesh.elementType = getElmType(mesh.numVtxPerElm);
   fclose(f);
 }
@@ -196,7 +205,7 @@ int main(int argc, char** argv)
   const int dim = 3;
   apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, dim, isMatched);
   apf::GlobalToVert outMap;
-  apf::construct(mesh, m.elements, m.numElms, m.elementType, outMap);
+  apf::construct(mesh, m.elements, m.localNumElms, m.elementType, outMap);
   delete [] m.elements;
   apf::alignMdsRemotes(mesh);
   apf::deriveMdsModel(mesh);

From 7e342a190257b7af0ffeb9748c212e744938bcd9 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 4 Jan 2019 21:57:24 -0500
Subject: [PATCH 012/555] fix coord reader

---
 test/matchedNodeElmReader.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index e623ab346..0650dded9 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -77,7 +77,7 @@ void readCoords(FILE* f, unsigned numvtx, unsigned& localnumvtx, double** coordi
     int id;
     double pos[3];
     gmi_fscanf(f, 4, "%d %lf %lf %lf", &id, pos+0, pos+1, pos+2);
-    if( i > firstVtx && i < lastVtx ) {
+    if( i >= firstVtx && i < lastVtx ) {
       for(unsigned j=0; j<3; j++)
         (*coordinates)[vidx*3+j] = pos[j];
       vidx++;

From fd6b454f52604e10a9246683fae42b486e6fe950 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 4 Jan 2019 21:57:58 -0500
Subject: [PATCH 013/555] remove print statements, add timer

---
 test/matchedNodeElmReader.cc | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 0650dded9..96ad4227f 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -68,8 +68,6 @@ void getNumVerts(FILE* f, unsigned& verts) {
 void readCoords(FILE* f, unsigned numvtx, unsigned& localnumvtx, double** coordinates) {
   long firstVtx, lastVtx;
   getLocalRange(numvtx, localnumvtx,firstVtx,lastVtx);
-  fprintf(stderr, "%d localnumvtx %u firstVtx %ld lastVtx %ld\n",
-      PCU_Comm_Self(), localnumvtx, firstVtx, lastVtx);
   *coordinates = new double[localnumvtx*3];
   rewind(f);
   int vidx = 0;
@@ -100,8 +98,6 @@ void readElements(FILE* f, unsigned numelms, unsigned numVtxPerElm,
     unsigned& localNumElms, int** elements) {
   long firstElm, lastElm;
   getLocalRange(numelms, localNumElms, firstElm, lastElm);
-  fprintf(stderr, "%d localNumElms %u firstElm %ld lastElm %ld\n",
-      PCU_Comm_Self(), localNumElms, firstElm, lastElm);
   *elements = new int[localNumElms*numVtxPerElm];
   rewind(f);
   unsigned i, j;
@@ -145,8 +141,8 @@ void readMesh(const char* meshfilename,
   PCU_ALWAYS_ASSERT(fc);
   getNumElms(f,mesh.numElms);
   getNumVerts(fc,mesh.numVerts);
-  fprintf(stderr, "numElms %u numVerts %u\n",
-      mesh.numElms, mesh.numVerts);
+  if(!PCU_Comm_Self())
+    fprintf(stderr, "numElms %u numVerts %u\n", mesh.numElms, mesh.numVerts);
   readCoords(fc, mesh.numVerts, mesh.localNumVerts, &(mesh.coords));
   fclose(fc);
   if( strcmp(matchfilename, "NULL") ) {
@@ -192,6 +188,7 @@ int main(int argc, char** argv)
 
   gmi_model* model = gmi_load(".null");
 
+  double t0 = PCU_Time();
   MeshInfo m;
   readMesh(argv[1],argv[2],argv[3],m);
 
@@ -216,6 +213,8 @@ int main(int argc, char** argv)
     delete [] m.matches;
   }
   outMap.clear();
+  if(!PCU_Comm_Self())
+    fprintf(stderr, "seconds to create mesh %.3f\n", PCU_Time()-t0);
   mesh->verify();
 
   gmi_write_dmg(model, argv[4]);

From 37cedbd0311aa7a4f62c8c37f47bfc0fe4ec2a06 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 9 Jan 2019 15:38:37 -0500
Subject: [PATCH 014/555] construct: parallel matching compiles

---
 apf/apfConstruct.cc          | 158 +++++++++++++++++++++++++++++++++++
 apf/apfConvert.h             |  13 +++
 test/matchedNodeElmReader.cc |  32 ++++---
 3 files changed, 186 insertions(+), 17 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index 26012d4e1..c9cf91dd7 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -237,6 +237,164 @@ void setCoords(Mesh2* m, const double* coords, int nverts,
   delete [] c;
 }
 
+void setMatches(Mesh2* m, const int* matches, int nverts,
+    GlobalToVert& globalToVert)
+{
+  Gid max = getMax(globalToVert);
+  Gid total = max + 1;
+  int peers = PCU_Comm_Peers();
+  int quotient = total / peers;
+  int remainder = total % peers;
+  int mySize = quotient;
+  int self = PCU_Comm_Self();
+  if (self == (peers - 1))
+    mySize += remainder;
+  int myOffset = self * quotient;
+
+  /* Force each peer to have exactly mySize verts.
+     This means we might need to send and recv some matches */
+  int* c = new int[mySize];
+
+  int start = PCU_Exscan_Int(nverts);
+
+  PCU_Comm_Begin();
+  int to = std::min(peers - 1, start / quotient);
+  int n = std::min((to+1)*quotient-start, nverts);
+  while (nverts > 0) {
+    PCU_COMM_PACK(to, start);
+    PCU_COMM_PACK(to, n);
+    PCU_Comm_Pack(to, matches, n*sizeof(int));
+
+    nverts -= n;
+    start += n;
+    matches += n;
+    to = std::min(peers - 1, to + 1);
+    n = std::min(quotient, nverts);
+  }
+  PCU_Comm_Send();
+  while (PCU_Comm_Receive()) {
+    PCU_COMM_UNPACK(start);
+    PCU_COMM_UNPACK(n);
+    PCU_Comm_Unpack(&c[(start - myOffset)], n*sizeof(int));
+  }
+
+
+  /* Tell all the owners of the matches what we need */
+  typedef std::vector< std::vector<int> > TmpParts;
+  TmpParts tmpParts(mySize);
+  PCU_Comm_Begin();
+  APF_CONST_ITERATE(GlobalToVert, globalToVert, it) {
+    int gid = it->first;
+    int to = std::min(peers - 1, gid / quotient);
+    PCU_COMM_PACK(to, gid);
+  }
+  PCU_Comm_Send();
+  while (PCU_Comm_Receive()) {
+    int gid;
+    PCU_COMM_UNPACK(gid);
+    int from = PCU_Comm_Sender();
+    tmpParts.at(gid - myOffset).push_back(from);
+  }
+ 
+  MeshTag* matchGidTag = m->createIntTag("matchGids", 1);
+  /* Send the matches to everybody who wants them */
+  PCU_Comm_Begin();
+  for (int i = 0; i < mySize; ++i) {
+    std::vector<int>& parts = tmpParts[i];
+    for (size_t j = 0; j < parts.size(); ++j) {
+      int to = parts[j];
+      int gid = i + myOffset;
+      PCU_COMM_PACK(to, gid);
+      PCU_COMM_PACK(to, c[i]);
+    }
+  }
+  PCU_Comm_Send();
+  while (PCU_Comm_Receive()) {
+    int gid;
+    PCU_COMM_UNPACK(gid);
+    int match;
+    PCU_COMM_UNPACK(match);
+    m->setIntTag(globalToVert[gid], matchGidTag, &match);
+  }
+
+  /* Use the 1D partitioning of global ids to distribute the 
+   * entity pointers and their owning ranks. Process 0 will hold
+   * the entity pointers and owners for mesh vertex gid [0..quotient),
+   * process 1 for gids [quotient..2*quotient), ...
+   */
+  typedef std::vector< apf::MeshEntity* > EntPtrs;
+  EntPtrs verts(mySize);
+  int* owners = new int[mySize];
+  APF_CONST_ITERATE(GlobalToVert, globalToVert, it) {
+    MeshEntity* e = it->second;
+    if( m->isOwned(e) ) {
+      int gid = it->first;
+      int to = std::min(peers - 1, gid / quotient);
+      PCU_COMM_PACK(to, gid);
+      PCU_COMM_PACK(to, e);
+      PCU_COMM_PACK(to, self);
+    }
+  }
+  PCU_Comm_Send();
+  while (PCU_Comm_Receive()) {
+    int gid;
+    PCU_COMM_UNPACK(gid);
+    MeshEntity* vert;
+    PCU_COMM_UNPACK(vert);
+    int owner;
+    PCU_COMM_UNPACK(owner);
+    verts[gid-myOffset] = vert;
+    owners[gid-myOffset] = owner;
+  }
+
+  /* Tell all the owners of the matches what we need */
+  typedef std::vector< std::vector<int> > TmpParts;
+  TmpParts matchParts(mySize);
+  PCU_Comm_Begin();
+  APF_CONST_ITERATE(GlobalToVert, globalToVert, it) {
+    int matchGid;
+    m->getIntTag(it->second, matchGidTag, &matchGid);
+    if( matchGid != -1 ) {  // marker for an unmatched vertex
+      int to = std::min(peers - 1, matchGid / quotient);
+      PCU_COMM_PACK(to, matchGid);
+    }
+  }
+  PCU_Comm_Send();
+  while (PCU_Comm_Receive()) {
+    int gid;
+    PCU_COMM_UNPACK(gid);
+    int from = PCU_Comm_Sender();
+    matchParts.at(gid - myOffset).push_back(from);
+  }
+ 
+  /* Send the match pointer and owner process to everybody
+   * who wants them */
+  PCU_Comm_Begin();
+  for (int i = 0; i < mySize; ++i) {
+    std::vector<int>& parts = matchParts[i];
+    for (size_t j = 0; j < parts.size(); ++j) {
+      int to = parts[j];
+      int gid = i + myOffset;
+      PCU_COMM_PACK(to, gid);
+      PCU_COMM_PACK(to, verts[i]);
+      PCU_COMM_PACK(to, owners[i]);
+    }
+  }
+  PCU_Comm_Send();
+  while (PCU_Comm_Receive()) {
+    int gid;
+    PCU_COMM_UNPACK(gid);
+    MeshEntity* match;
+    PCU_COMM_UNPACK(match);
+    int owner;
+    PCU_COMM_UNPACK(owner);
+    m->addMatch(globalToVert[gid], owner, match);
+  }
+
+  delete [] c;
+  delete [] owners;
+}
+
 void destruct(Mesh2* m, int*& conn, int& nelem, int &etype)
 {
   int dim = m->getDimension();
diff --git a/apf/apfConvert.h b/apf/apfConvert.h
index df850d05b..6b55567c2 100644
--- a/apf/apfConvert.h
+++ b/apf/apfConvert.h
@@ -59,6 +59,19 @@ void construct(Mesh2* m, const int* conn, int nelem, int etype,
 void setCoords(Mesh2* m, const double* coords, int nverts,
     GlobalToVert& globalToVert);
 
+/** \brief Assign matching to the mesh
+  * \details
+  * Each peer provides a set of the matched entity global ids. An id set
+  * to -1 indicates that the vertex is not matched.  The ids most be ordered
+  * according to the global ids of the vertices. Peer 0 provides the ids
+  * for vertices 0 to m-1, peer to for m to n-1, ...
+  * After this call, all vertices in the apf::Mesh2 object have correct
+  * coordinates assigned.
+  */
+void setMatches(Mesh2* m, const int* matches, int nverts,
+    GlobalToVert& globalToVert);
+
+
 /** \brief convert an apf::Mesh2 object into a connectivity array
   \details this is useful for debugging the apf::convert function */
 void destruct(Mesh2* m, int*& conn, int& nelem, int &etype);
diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 96ad4227f..9aa5060c8 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -83,14 +83,22 @@ void readCoords(FILE* f, unsigned numvtx, unsigned& localnumvtx, double** coordi
   }
 }
 
-void readMatches(FILE* f, unsigned numvtx, int* matches) {
+void readMatches(FILE* f, unsigned numvtx, int** matches) {
+  long firstVtx, lastVtx;
+  unsigned localnumvtx;
+  getLocalRange(numvtx, localnumvtx, firstVtx, lastVtx);
+  *matches = new int[localnumvtx];
   rewind(f);
+  int vidx = 0;
   for(unsigned i=0; i<numvtx; i++) {
     int ignored, matchedVtx;
     gmi_fscanf(f, 2, "%d %d", &ignored, &matchedVtx); //export from matlab using 1-based indices
-    PCU_ALWAYS_ASSERT( matchedVtx == -1 ||
-        ( matchedVtx > 1 && matchedVtx <= static_cast<int>(numvtx) ));
-    matches[i] = matchedVtx--;
+    if( i >= firstVtx && i < lastVtx ) {
+      PCU_ALWAYS_ASSERT( matchedVtx == -1 ||
+          ( matchedVtx > 1 && matchedVtx <= static_cast<int>(numvtx) ));
+      (*matches)[vidx] = matchedVtx--;
+      vidx++;
+    }
   }
 }
 
@@ -148,8 +156,7 @@ void readMesh(const char* meshfilename,
   if( strcmp(matchfilename, "NULL") ) {
     FILE* fm = fopen(matchfilename, "r");
     PCU_ALWAYS_ASSERT(fm);
-    mesh.matches = new int[mesh.numVerts];
-    readMatches(fm, mesh.numVerts, mesh.matches);
+    readMatches(fm, mesh.numVerts, &(mesh.matches));
     fclose(fm);
   }
   mesh.numVtxPerElm = 8; //hack!
@@ -159,16 +166,6 @@ void readMesh(const char* meshfilename,
   fclose(f);
 }
 
-void setMatches(apf::Mesh2* m, unsigned numVerts, int* matches,
-    apf::GlobalToVert& globToVtx) {
-  int self = PCU_Comm_Self();
-  for(unsigned i=0; i<numVerts; i++) {
-    if( matches[i] != -1 ) {
-      m->addMatch(globToVtx[i], self, globToVtx[matches[i]]);
-    }
-  }
-}
-
 int main(int argc, char** argv)
 {
   if( argc != 6 ) {
@@ -209,7 +206,8 @@ int main(int argc, char** argv)
   apf::setCoords(mesh, m.coords, m.localNumVerts, outMap);
   delete [] m.coords;
   if( isMatched ) {
-    setMatches(mesh,m.numVerts,m.matches,outMap);
+    apf::setMatches(mesh, m.matches, m.localNumVerts, outMap);
+    mesh->acceptChanges();
     delete [] m.matches;
   }
   outMap.clear();

From 5b78e6973145818b6c22393c0b4bf01fb9fc75d1 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 9 Jan 2019 15:43:12 -0500
Subject: [PATCH 015/555] rank 0 prints usage

---
 test/matchedNodeElmReader.cc | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 9aa5060c8..b67d44b6d 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -168,18 +168,20 @@ void readMesh(const char* meshfilename,
 
 int main(int argc, char** argv)
 {
+  MPI_Init(&argc,&argv);
+  PCU_Comm_Init();
+  lion_set_verbosity(1);
   if( argc != 6 ) {
-    printf("Usage: %s <ascii mesh connectivity .cnn> "
-        "<ascii vertex coordinates .crd> "
-        "<ascii vertex matching flag .match> "
-        "<output model .dmg> <output mesh .smb>\n",
-        argv[0]);
+    if( !PCU_Comm_Self() ) {
+      printf("Usage: %s <ascii mesh connectivity .cnn> "
+          "<ascii vertex coordinates .crd> "
+          "<ascii vertex matching flag .match> "
+          "<output model .dmg> <output mesh .smb>\n",
+          argv[0]);
+    }
     return 0;
   }
 
-  MPI_Init(&argc,&argv);
-  PCU_Comm_Init();
-  lion_set_verbosity(1);
   gmi_register_mesh();
   gmi_register_null();
 

From 90025707e8690bc11ad5c13ba523a2f6a2220ee4 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 9 Jan 2019 17:25:06 -0500
Subject: [PATCH 016/555] decrement matched global ids by one

---
 test/matchedNodeElmReader.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index b67d44b6d..824e1210f 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -96,7 +96,9 @@ void readMatches(FILE* f, unsigned numvtx, int** matches) {
     if( i >= firstVtx && i < lastVtx ) {
       PCU_ALWAYS_ASSERT( matchedVtx == -1 ||
           ( matchedVtx > 1 && matchedVtx <= static_cast<int>(numvtx) ));
-      (*matches)[vidx] = matchedVtx--;
+      if( matchedVtx != -1 )
+        --matchedVtx;
+      (*matches)[vidx] = matchedVtx;
       vidx++;
     }
   }

From 3766f07e28350101473c4ab692737a484d4efbef Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 9 Jan 2019 17:26:44 -0500
Subject: [PATCH 017/555] missing pcu_begin

---
 apf/apfConstruct.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index c9cf91dd7..f9111825d 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -325,6 +325,7 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
   typedef std::vector< apf::MeshEntity* > EntPtrs;
   EntPtrs verts(mySize);
   int* owners = new int[mySize];
+  PCU_Comm_Begin();
   APF_CONST_ITERATE(GlobalToVert, globalToVert, it) {
     MeshEntity* e = it->second;
     if( m->isOwned(e) ) {

From 51e61a2bc60fdbec76e2d698c445770cc7f42e86 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 9 Jan 2019 17:27:03 -0500
Subject: [PATCH 018/555] safety checks on matches

---
 apf/apfConstruct.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index f9111825d..478b06ebc 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -1,4 +1,5 @@
 #include <PCU.h>
+#include "pcu_util.h"
 #include "apfConvert.h"
 #include "apfMesh2.h"
 #include "apf.h"
@@ -314,6 +315,7 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
     PCU_COMM_UNPACK(gid);
     int match;
     PCU_COMM_UNPACK(match);
+    PCU_ALWAYS_ASSERT(gid != match);
     m->setIntTag(globalToVert[gid], matchGidTag, &match);
   }
 
@@ -389,6 +391,7 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
     PCU_COMM_UNPACK(match);
     int owner;
     PCU_COMM_UNPACK(owner);
+    PCU_ALWAYS_ASSERT(match != globalToVert[gid] && owner != self);
     m->addMatch(globalToVert[gid], owner, match);
   }
 

From fbb6efa858f8bf62c3e83c985ab0fc71aed2c923 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 9 Jan 2019 22:43:26 -0500
Subject: [PATCH 019/555] works for a single matched pair of verts

---
 apf/apfConstruct.cc          | 42 +++++++++++++++++++++++-------------
 test/matchedNodeElmReader.cc |  2 +-
 2 files changed, 28 insertions(+), 16 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index 478b06ebc..0edce6c87 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -279,7 +279,6 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
     PCU_Comm_Unpack(&c[(start - myOffset)], n*sizeof(int));
   }
 
-
   /* Tell all the owners of the matches what we need */
   typedef std::vector< std::vector<int> > TmpParts;
   TmpParts tmpParts(mySize);
@@ -351,14 +350,17 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
   }
 
   /* Tell all the owners of the matches what we need */
-  typedef std::vector< std::vector<int> > TmpParts;
-  TmpParts matchParts(mySize);
+  typedef std::pair<int,int> MatchingPair;
+  typedef std::map< MatchingPair, int > MatchMap;
+  MatchMap matchParts;
   PCU_Comm_Begin();
   APF_CONST_ITERATE(GlobalToVert, globalToVert, it) {
+    int gid = it->first;
     int matchGid;
     m->getIntTag(it->second, matchGidTag, &matchGid);
     if( matchGid != -1 ) {  // marker for an unmatched vertex
       int to = std::min(peers - 1, matchGid / quotient);
+      PCU_COMM_PACK(to, gid);
       PCU_COMM_PACK(to, matchGid);
     }
   }
@@ -366,22 +368,27 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
   while (PCU_Comm_Receive()) {
     int gid;
     PCU_COMM_UNPACK(gid);
+    int matchGid;
+    PCU_COMM_UNPACK(matchGid);
+    MatchingPair mp(gid,matchGid);
     int from = PCU_Comm_Sender();
-    matchParts.at(gid - myOffset).push_back(from);
+    fprintf(stderr, "%d received gid %d from %d\n", self, gid, from);
+    PCU_ALWAYS_ASSERT( ! matchParts.count(mp) ); // only allow single matching
+    matchParts[mp] = from;
   }
  
   /* Send the match pointer and owner process to everybody
    * who wants them */
   PCU_Comm_Begin();
-  for (int i = 0; i < mySize; ++i) {
-    std::vector<int>& parts = matchParts[i];
-    for (size_t j = 0; j < parts.size(); ++j) {
-      int to = parts[j];
-      int gid = i + myOffset;
-      PCU_COMM_PACK(to, gid);
-      PCU_COMM_PACK(to, verts[i]);
-      PCU_COMM_PACK(to, owners[i]);
-    }
+  APF_ITERATE(MatchMap,matchParts,it) {
+    MatchingPair mp = it->first;
+    int to = it->second;
+    int gid = mp.first;
+    int matchGid = mp.second;
+    fprintf(stderr, "%d %d packing gid %d matchGid %d to %d\n", self, gid, matchGid, gid, to);
+    PCU_COMM_PACK(to, gid);
+    PCU_COMM_PACK(to, verts[matchGid-myOffset]);
+    PCU_COMM_PACK(to, owners[matchGid-myOffset]);
   }
   PCU_Comm_Send();
   while (PCU_Comm_Receive()) {
@@ -391,8 +398,13 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
     PCU_COMM_UNPACK(match);
     int owner;
     PCU_COMM_UNPACK(owner);
-    PCU_ALWAYS_ASSERT(match != globalToVert[gid] && owner != self);
-    m->addMatch(globalToVert[gid], owner, match);
+    MeshEntity* partner = globalToVert[gid];
+    if(match == partner && owner == self) {
+      fprintf(stderr, "%d match %p globalToVert[gid] %p owner %d\n",
+          self, match, partner, owner);
+    }
+    PCU_ALWAYS_ASSERT(! (match == partner && owner == self) );
+    m->addMatch(partner, owner, match);
   }
 
   delete [] c;
diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 824e1210f..f8ec64a04 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -95,7 +95,7 @@ void readMatches(FILE* f, unsigned numvtx, int** matches) {
     gmi_fscanf(f, 2, "%d %d", &ignored, &matchedVtx); //export from matlab using 1-based indices
     if( i >= firstVtx && i < lastVtx ) {
       PCU_ALWAYS_ASSERT( matchedVtx == -1 ||
-          ( matchedVtx > 1 && matchedVtx <= static_cast<int>(numvtx) ));
+          ( matchedVtx >= 1 && matchedVtx <= static_cast<int>(numvtx) ));
       if( matchedVtx != -1 )
         --matchedVtx;
       (*matches)[vidx] = matchedVtx;

From bdfaefe21ce815d84269f296516b0b9618c856fb Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 9 Jan 2019 23:18:59 -0500
Subject: [PATCH 020/555] phasta: use strcpy

---
 phasta/phIO.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/phasta/phIO.c b/phasta/phIO.c
index 4352dadbb..7c65a2645 100644
--- a/phasta/phIO.c
+++ b/phasta/phIO.c
@@ -81,8 +81,7 @@ static int find_header(FILE* f, const char* name, char* found, char header[PH_LI
     tmp[PH_LINE-1] = '\0';
     parse_header(tmp, &hname, &bytes, 0, NULL);
     if (!strncmp(name, hname, strlen(name))) {
-      strncpy(found, hname, strlen(hname));
-      found[strlen(hname)] = '\0';
+      strcpy(found, hname);
       return 1;
     }
     fseek(f, bytes, SEEK_CUR);

From a7ff15846e075d31ecd393913c2ca44725b75e0f Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 9 Jan 2019 23:20:59 -0500
Subject: [PATCH 021/555] xgc_split: avoid string overflow

detected by gcc 8.2.1
---
 test/xgc_split.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/xgc_split.cc b/test/xgc_split.cc
index deaaddc75..2417362c2 100644
--- a/test/xgc_split.cc
+++ b/test/xgc_split.cc
@@ -76,7 +76,7 @@ int main(int argc, char** argv)
   char without_extension[256];
   snprintf(without_extension,strlen(argv[3])-3,"%s",argv[3]);
 
-  char vtk_fname[32];
+  char vtk_fname[512];
   sprintf(vtk_fname,"%s",without_extension); 
   pumi_mesh_write(m, vtk_fname, "vtk");
 

From 7c67774b0cc0dde9b26e8ccc8059a4f933020edb Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Thu, 10 Jan 2019 00:11:48 -0500
Subject: [PATCH 022/555] something broken with the matches file reader

---
 apf/apfConstruct.cc          | 34 +++++++++++++++++++++++++++++++---
 test/matchedNodeElmReader.cc | 12 ++++++++++--
 2 files changed, 41 insertions(+), 5 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index 0edce6c87..a426d1d8e 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -255,6 +255,7 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
   /* Force each peer to have exactly mySize verts.
      This means we might need to send and recv some matches */
   int* c = new int[mySize];
+  fprintf(stderr, "%d mysize %d\n", self, mySize);
 
   int start = PCU_Exscan_Int(nverts);
 
@@ -265,6 +266,8 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
     PCU_COMM_PACK(to, start);
     PCU_COMM_PACK(to, n);
     PCU_Comm_Pack(to, matches, n*sizeof(int));
+    fprintf(stderr, "%d sending start %d n %d to %d\n",
+        self, start, n, to);
 
     nverts -= n;
     start += n;
@@ -277,6 +280,16 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
     PCU_COMM_UNPACK(start);
     PCU_COMM_UNPACK(n);
     PCU_Comm_Unpack(&c[(start - myOffset)], n*sizeof(int));
+    fprintf(stderr, "%d reciving start %d n %d from %d\n",
+        self, start, n, PCU_Comm_Sender());
+  }
+
+  for (int i = 0; i < mySize; ++i) {
+    int match = c[i];
+    if( match == 66350 || match == 65075 ) {
+      fprintf(stderr, "%d found match %d at gid %d\n",
+          self, match, i+myOffset);
+    }
   }
 
   /* Tell all the owners of the matches what we need */
@@ -304,8 +317,13 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
     for (size_t j = 0; j < parts.size(); ++j) {
       int to = parts[j];
       int gid = i + myOffset;
+      int matchGid = c[i];
       PCU_COMM_PACK(to, gid);
-      PCU_COMM_PACK(to, c[i]);
+      PCU_COMM_PACK(to, matchGid);
+      if( matchGid == 66350 || matchGid == 65075 ) {
+        fprintf(stderr, "%d packing i %d gid %d matchGid %d to %d\n",
+            self, i, gid, matchGid, to);
+      }
     }
   }
   PCU_Comm_Send();
@@ -316,6 +334,10 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
     PCU_COMM_UNPACK(match);
     PCU_ALWAYS_ASSERT(gid != match);
     m->setIntTag(globalToVert[gid], matchGidTag, &match);
+    if( match == 66350 || match == 65075 ) {
+      fprintf(stderr, "%d attaching match %d to gid %d\n",
+          self, match, gid);
+    }
   }
 
   /* Use the 1D partitioning of global ids to distribute the 
@@ -361,6 +383,10 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
     if( matchGid != -1 ) {  // marker for an unmatched vertex
       int to = std::min(peers - 1, matchGid / quotient);
       PCU_COMM_PACK(to, gid);
+      if( matchGid == 66350 || matchGid == 65075 ) {
+        fprintf(stderr, "%d packing gid %d matchGid %d to %d\n",
+            self, gid, matchGid, to);
+      }
       PCU_COMM_PACK(to, matchGid);
     }
   }
@@ -372,7 +398,10 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
     PCU_COMM_UNPACK(matchGid);
     MatchingPair mp(gid,matchGid);
     int from = PCU_Comm_Sender();
-    fprintf(stderr, "%d received gid %d from %d\n", self, gid, from);
+    if( matchParts.count(mp) ) {
+      fprintf(stderr, "%d received gid %d matchGid %d from %d matchParts.count(mp) %lu\n",
+          self, gid, matchGid, from, matchParts.count(mp));
+    }
     PCU_ALWAYS_ASSERT( ! matchParts.count(mp) ); // only allow single matching
     matchParts[mp] = from;
   }
@@ -385,7 +414,6 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
     int to = it->second;
     int gid = mp.first;
     int matchGid = mp.second;
-    fprintf(stderr, "%d %d packing gid %d matchGid %d to %d\n", self, gid, matchGid, gid, to);
     PCU_COMM_PACK(to, gid);
     PCU_COMM_PACK(to, verts[matchGid-myOffset]);
     PCU_COMM_PACK(to, owners[matchGid-myOffset]);
diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index f8ec64a04..8482b92fc 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -9,6 +9,7 @@
 #include <pcu_util.h>
 #include <cstdlib>
 #include <string.h>
+#include <cassert>
 
 void getLocalRange(unsigned total, unsigned& local,
     long& first, long& last) {
@@ -87,17 +88,24 @@ void readMatches(FILE* f, unsigned numvtx, int** matches) {
   long firstVtx, lastVtx;
   unsigned localnumvtx;
   getLocalRange(numvtx, localnumvtx, firstVtx, lastVtx);
+  fprintf(stderr, "%d readMatches numvtx %d localnumvtx %u firstVtx %ld lastVtx %ld\n",
+      PCU_Comm_Self(), numvtx, localnumvtx, firstVtx, lastVtx);
   *matches = new int[localnumvtx];
   rewind(f);
   int vidx = 0;
   for(unsigned i=0; i<numvtx; i++) {
-    int ignored, matchedVtx;
-    gmi_fscanf(f, 2, "%d %d", &ignored, &matchedVtx); //export from matlab using 1-based indices
+    int gid, matchedVtx;
+    int cnt = fscanf(f, "%d %d", &gid, &matchedVtx); //export from matlab using 1-based indices
+    assert(cnt == 2);
     if( i >= firstVtx && i < lastVtx ) {
       PCU_ALWAYS_ASSERT( matchedVtx == -1 ||
           ( matchedVtx >= 1 && matchedVtx <= static_cast<int>(numvtx) ));
       if( matchedVtx != -1 )
         --matchedVtx;
+      if( matchedVtx == 66350 || matchedVtx == 65075 ) {
+        fprintf(stderr, "%d reader found match %d at gid %d i %d vidx %d\n",
+            PCU_Comm_Self(), matchedVtx, gid, i, vidx);
+      }
       (*matches)[vidx] = matchedVtx;
       vidx++;
     }

From 9c1ac920dd9e694ebaafc552a979834f07985d11 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 11 Jan 2019 10:34:49 -0500
Subject: [PATCH 023/555] debugging matches

---
 apf/apfConstruct.cc          | 101 ++++++++++++++++++++++++++---------
 test/matchedNodeElmReader.cc |  10 ++--
 2 files changed, 80 insertions(+), 31 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index a426d1d8e..ef5430767 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -286,10 +286,9 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
 
   for (int i = 0; i < mySize; ++i) {
     int match = c[i];
-    if( match == 66350 || match == 65075 ) {
+    if( match != -1 )
       fprintf(stderr, "%d found match %d at gid %d\n",
           self, match, i+myOffset);
-    }
   }
 
   /* Tell all the owners of the matches what we need */
@@ -300,6 +299,10 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
     int gid = it->first;
     int to = std::min(peers - 1, gid / quotient);
     PCU_COMM_PACK(to, gid);
+    if( gid == 66300 || gid == 66350 ) {
+      fprintf(stderr, "%d requesting matches of gid %d from %d\n",
+          self, gid, to);
+    }
   }
   PCU_Comm_Send();
   while (PCU_Comm_Receive()) {
@@ -320,7 +323,7 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
       int matchGid = c[i];
       PCU_COMM_PACK(to, gid);
       PCU_COMM_PACK(to, matchGid);
-      if( matchGid == 66350 || matchGid == 65075 ) {
+      if( matchGid != -1 ) {
         fprintf(stderr, "%d packing i %d gid %d matchGid %d to %d\n",
             self, i, gid, matchGid, to);
       }
@@ -333,8 +336,9 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
     int match;
     PCU_COMM_UNPACK(match);
     PCU_ALWAYS_ASSERT(gid != match);
+    PCU_ALWAYS_ASSERT(globalToVert.count(gid));
     m->setIntTag(globalToVert[gid], matchGidTag, &match);
-    if( match == 66350 || match == 65075 ) {
+    if( match != -1 ) {
       fprintf(stderr, "%d attaching match %d to gid %d\n",
           self, match, gid);
     }
@@ -356,7 +360,10 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
       int to = std::min(peers - 1, gid / quotient);
       PCU_COMM_PACK(to, gid);
       PCU_COMM_PACK(to, e);
-      PCU_COMM_PACK(to, self);
+      if( gid == 50 || gid == 0 || gid == 66300 || gid == 66350 ) {
+        fprintf(stderr, "%d packing pointer to %d gid %d vert %p\n",
+            self, to, gid, (void*)e);
+      }
     }
   }
   PCU_Comm_Send();
@@ -365,15 +372,18 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
     PCU_COMM_UNPACK(gid);
     MeshEntity* vert;
     PCU_COMM_UNPACK(vert);
-    int owner;
-    PCU_COMM_UNPACK(owner);
+    int owner = PCU_Comm_Sender();
     verts[gid-myOffset] = vert;
     owners[gid-myOffset] = owner;
+    if( gid == 50 || gid == 0 || gid == 66300 || gid == 66350 ) {
+      fprintf(stderr, "%d unpacking pointer from %d gid %d vert %p\n",
+          self, owner, gid, (void*)vert);
+    }
   }
 
-  /* Tell all the owners of the matches what we need */
+  /* Tell the brokers of the matches we need */
   typedef std::pair<int,int> MatchingPair;
-  typedef std::map< MatchingPair, int > MatchMap;
+  typedef std::map< MatchingPair, std::vector<int> > MatchMap;
   MatchMap matchParts;
   PCU_Comm_Begin();
   APF_CONST_ITERATE(GlobalToVert, globalToVert, it) {
@@ -383,11 +393,9 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
     if( matchGid != -1 ) {  // marker for an unmatched vertex
       int to = std::min(peers - 1, matchGid / quotient);
       PCU_COMM_PACK(to, gid);
-      if( matchGid == 66350 || matchGid == 65075 ) {
-        fprintf(stderr, "%d packing gid %d matchGid %d to %d\n",
-            self, gid, matchGid, to);
-      }
       PCU_COMM_PACK(to, matchGid);
+      fprintf(stderr, "%d packing req ptr to %d gid %d matchGid %d\n",
+          self, to, gid, matchGid);
     }
   }
   PCU_Comm_Send();
@@ -398,45 +406,86 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
     PCU_COMM_UNPACK(matchGid);
     MatchingPair mp(gid,matchGid);
     int from = PCU_Comm_Sender();
-    if( matchParts.count(mp) ) {
-      fprintf(stderr, "%d received gid %d matchGid %d from %d matchParts.count(mp) %lu\n",
-          self, gid, matchGid, from, matchParts.count(mp));
-    }
-    PCU_ALWAYS_ASSERT( ! matchParts.count(mp) ); // only allow single matching
-    matchParts[mp] = from;
+    matchParts[mp].push_back(from);
+    fprintf(stderr, "%d unpacking ptr req from %d gid %d matchGid %d\n",
+        self, from, gid, matchGid);
   }
- 
+
   /* Send the match pointer and owner process to everybody
    * who wants them */
   PCU_Comm_Begin();
   APF_ITERATE(MatchMap,matchParts,it) {
     MatchingPair mp = it->first;
-    int to = it->second;
     int gid = mp.first;
     int matchGid = mp.second;
-    PCU_COMM_PACK(to, gid);
-    PCU_COMM_PACK(to, verts[matchGid-myOffset]);
-    PCU_COMM_PACK(to, owners[matchGid-myOffset]);
+    std::vector<int> parts = it->second;
+    for(size_t i=0; i<parts.size(); i++) {
+      const int to = parts[i];
+      PCU_COMM_PACK(to, gid);
+      PCU_COMM_PACK(to, matchGid);
+      PCU_COMM_PACK(to, verts[matchGid-myOffset]);
+      PCU_COMM_PACK(to, owners[matchGid-myOffset]);
+      fprintf(stderr, "%d packing match ptr to %d gid %d matchGid %d vert %p owner %d\n",
+          self, to, gid, matchGid,
+          (void*)verts[matchGid-myOffset], owners[matchGid-myOffset]);
+    }
   }
   PCU_Comm_Send();
   while (PCU_Comm_Receive()) {
     int gid;
     PCU_COMM_UNPACK(gid);
+    int matchGid;
+    PCU_COMM_UNPACK(matchGid);
     MeshEntity* match;
     PCU_COMM_UNPACK(match);
     int owner;
     PCU_COMM_UNPACK(owner);
+    fprintf(stderr, "%d unpacked match ptr from %d gid %d matchGid %d matchPtr %p owner %d\n",
+        self, PCU_Comm_Sender(), gid, matchGid, (void*)match, owner);
+    PCU_ALWAYS_ASSERT(globalToVert.count(gid));
     MeshEntity* partner = globalToVert[gid];
     if(match == partner && owner == self) {
-      fprintf(stderr, "%d match %p globalToVert[gid] %p owner %d\n",
-          self, match, partner, owner);
+      fprintf(stderr, "%d match == partner owner == self match %p partner %p owner %d\n",
+          self, (void*)match, (void*)partner, owner);
     }
     PCU_ALWAYS_ASSERT(! (match == partner && owner == self) );
     m->addMatch(partner, owner, match);
   }
 
+
   delete [] c;
   delete [] owners;
+
+  /* Owners send match info to copies */
+  /*
+  PCU_Comm_Begin();
+  APF_CONST_ITERATE(GlobalToVert, globalToVert, it) {
+    // send matches to remotes
+    int vtx = it->second;
+    apf::Matches matches;
+    m->getMatches(vtx, matches);
+    if( matches.getSize() ) {
+      apf::Copies remotes;
+      m->getRemotes(vtx, remotes);
+      APF_ITERATE(apf::Copies, remotes, r) {
+        int peer = r->first;
+        apf::MeshEntity* rmtVtx = r->second;
+        for (size_t j=0; j < matches.getSize(); ++j)
+          PCU_COMM_PACK(matches[j].peer,matches[j].entity);
+        fprintf(stderr, "%d packing req ptr to %d gid %d matchGid %d\n",
+            self, to, gid, matchGid);
+      }
+    }
+  }
+  PCU_Comm_Send();
+  while (PCU_Comm_Receive()) {
+    //unpack and create matches
+  }
+  */
+
+
+  apf::removeTagFromDimension(m, matchGidTag, 0);
+  m->destroyTag(matchGidTag);
 }
 
 void destruct(Mesh2* m, int*& conn, int& nelem, int &etype)
diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 8482b92fc..48bae7e6e 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -93,10 +93,9 @@ void readMatches(FILE* f, unsigned numvtx, int** matches) {
   *matches = new int[localnumvtx];
   rewind(f);
   int vidx = 0;
-  for(unsigned i=0; i<numvtx; i++) {
-    int gid, matchedVtx;
-    int cnt = fscanf(f, "%d %d", &gid, &matchedVtx); //export from matlab using 1-based indices
-    assert(cnt == 2);
+  int gid, matchedVtx;
+  int i = 0;
+  while( 2 == fscanf(f, "%d %d", &gid, &matchedVtx) ) {
     if( i >= firstVtx && i < lastVtx ) {
       PCU_ALWAYS_ASSERT( matchedVtx == -1 ||
           ( matchedVtx >= 1 && matchedVtx <= static_cast<int>(numvtx) ));
@@ -109,6 +108,7 @@ void readMatches(FILE* f, unsigned numvtx, int** matches) {
       (*matches)[vidx] = matchedVtx;
       vidx++;
     }
+    i++;
   }
 }
 
@@ -195,7 +195,6 @@ int main(int argc, char** argv)
   gmi_register_mesh();
   gmi_register_null();
 
-  gmi_model* model = gmi_load(".null");
 
   double t0 = PCU_Time();
   MeshInfo m;
@@ -209,6 +208,7 @@ int main(int argc, char** argv)
     fprintf(stderr, "isMatched %d\n", isMatched);
 
   const int dim = 3;
+  gmi_model* model = gmi_load(".null");
   apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, dim, isMatched);
   apf::GlobalToVert outMap;
   apf::construct(mesh, m.elements, m.localNumElms, m.elementType, outMap);

From 15d059966be5b014972555579c5cdbcf6295fa50 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 16 Jan 2019 21:12:46 -0500
Subject: [PATCH 024/555] debugging

---
 apf/apfConstruct.cc | 52 ++++++++++++++++++++++-----------------------
 1 file changed, 26 insertions(+), 26 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index ef5430767..fcf7d37f5 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -241,6 +241,7 @@ void setCoords(Mesh2* m, const double* coords, int nverts,
 void setMatches(Mesh2* m, const int* matches, int nverts,
     GlobalToVert& globalToVert)
 {
+  PCU_Debug_Open();
   Gid max = getMax(globalToVert);
   Gid total = max + 1;
   int peers = PCU_Comm_Peers();
@@ -255,7 +256,7 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
   /* Force each peer to have exactly mySize verts.
      This means we might need to send and recv some matches */
   int* c = new int[mySize];
-  fprintf(stderr, "%d mysize %d\n", self, mySize);
+  PCU_Debug_Print("%d mysize %d\n", self, mySize);
 
   int start = PCU_Exscan_Int(nverts);
 
@@ -266,7 +267,7 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
     PCU_COMM_PACK(to, start);
     PCU_COMM_PACK(to, n);
     PCU_Comm_Pack(to, matches, n*sizeof(int));
-    fprintf(stderr, "%d sending start %d n %d to %d\n",
+    PCU_Debug_Print("%d sending start %d n %d to %d\n",
         self, start, n, to);
 
     nverts -= n;
@@ -280,14 +281,14 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
     PCU_COMM_UNPACK(start);
     PCU_COMM_UNPACK(n);
     PCU_Comm_Unpack(&c[(start - myOffset)], n*sizeof(int));
-    fprintf(stderr, "%d reciving start %d n %d from %d\n",
+    PCU_Debug_Print("%d reciving start %d n %d from %d\n",
         self, start, n, PCU_Comm_Sender());
   }
 
   for (int i = 0; i < mySize; ++i) {
     int match = c[i];
     if( match != -1 )
-      fprintf(stderr, "%d found match %d at gid %d\n",
+      PCU_Debug_Print("%d found match %d at gid %d\n",
           self, match, i+myOffset);
   }
 
@@ -300,7 +301,7 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
     int to = std::min(peers - 1, gid / quotient);
     PCU_COMM_PACK(to, gid);
     if( gid == 66300 || gid == 66350 ) {
-      fprintf(stderr, "%d requesting matches of gid %d from %d\n",
+      PCU_Debug_Print("%d requesting matches of gid %d from %d\n",
           self, gid, to);
     }
   }
@@ -324,7 +325,7 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
       PCU_COMM_PACK(to, gid);
       PCU_COMM_PACK(to, matchGid);
       if( matchGid != -1 ) {
-        fprintf(stderr, "%d packing i %d gid %d matchGid %d to %d\n",
+        PCU_Debug_Print("%d packing i %d gid %d matchGid %d to %d\n",
             self, i, gid, matchGid, to);
       }
     }
@@ -339,7 +340,7 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
     PCU_ALWAYS_ASSERT(globalToVert.count(gid));
     m->setIntTag(globalToVert[gid], matchGidTag, &match);
     if( match != -1 ) {
-      fprintf(stderr, "%d attaching match %d to gid %d\n",
+      PCU_Debug_Print("%d attaching match %d to gid %d\n",
           self, match, gid);
     }
   }
@@ -347,7 +348,7 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
   /* Use the 1D partitioning of global ids to distribute the 
    * entity pointers and their owning ranks. Process 0 will hold
    * the entity pointers and owners for mesh vertex gid [0..quotient),
-   * process 1 for gids [quotient..2*quotient), ...
+   * process 1 holds gids [quotient..2*quotient), ...
    */
   typedef std::vector< apf::MeshEntity* > EntPtrs;
   EntPtrs verts(mySize);
@@ -361,7 +362,7 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
       PCU_COMM_PACK(to, gid);
       PCU_COMM_PACK(to, e);
       if( gid == 50 || gid == 0 || gid == 66300 || gid == 66350 ) {
-        fprintf(stderr, "%d packing pointer to %d gid %d vert %p\n",
+        PCU_Debug_Print("%d packing pointer to %d gid %d vert %p\n",
             self, to, gid, (void*)e);
       }
     }
@@ -376,7 +377,7 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
     verts[gid-myOffset] = vert;
     owners[gid-myOffset] = owner;
     if( gid == 50 || gid == 0 || gid == 66300 || gid == 66350 ) {
-      fprintf(stderr, "%d unpacking pointer from %d gid %d vert %p\n",
+      PCU_Debug_Print("%d unpacking pointer from %d gid %d vert %p\n",
           self, owner, gid, (void*)vert);
     }
   }
@@ -386,28 +387,28 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
   typedef std::map< MatchingPair, std::vector<int> > MatchMap;
   MatchMap matchParts;
   PCU_Comm_Begin();
-  APF_CONST_ITERATE(GlobalToVert, globalToVert, it) {
+  APF_CONST_ITERATE(GlobalToVert, globalToVert, it) { //loop over local verts
     int gid = it->first;
     int matchGid;
-    m->getIntTag(it->second, matchGidTag, &matchGid);
+    m->getIntTag(it->second, matchGidTag, &matchGid); //get the matched ent gid
     if( matchGid != -1 ) {  // marker for an unmatched vertex
-      int to = std::min(peers - 1, matchGid / quotient);
-      PCU_COMM_PACK(to, gid);
-      PCU_COMM_PACK(to, matchGid);
-      fprintf(stderr, "%d packing req ptr to %d gid %d matchGid %d\n",
+      int to = std::min(peers - 1, matchGid / quotient); // broker
+      PCU_COMM_PACK(to, gid); // send the local vert gid
+      PCU_COMM_PACK(to, matchGid); // and the match gid needed
+      PCU_Debug_Print("%d packing req ptr to %d gid %d matchGid %d\n",
           self, to, gid, matchGid);
     }
   }
   PCU_Comm_Send();
   while (PCU_Comm_Receive()) {
     int gid;
-    PCU_COMM_UNPACK(gid);
+    PCU_COMM_UNPACK(gid); // request from entity gid
     int matchGid;
-    PCU_COMM_UNPACK(matchGid);
+    PCU_COMM_UNPACK(matchGid); // requesting matched entity gid 
     MatchingPair mp(gid,matchGid);
     int from = PCU_Comm_Sender();
-    matchParts[mp].push_back(from);
-    fprintf(stderr, "%d unpacking ptr req from %d gid %d matchGid %d\n",
+    matchParts[mp].push_back(from); // store a list of the proceses that need the pair (entity gid, match gid)
+    PCU_Debug_Print("%d unpacking ptr req from %d gid %d matchGid %d\n",
         self, from, gid, matchGid);
   }
 
@@ -425,7 +426,7 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
       PCU_COMM_PACK(to, matchGid);
       PCU_COMM_PACK(to, verts[matchGid-myOffset]);
       PCU_COMM_PACK(to, owners[matchGid-myOffset]);
-      fprintf(stderr, "%d packing match ptr to %d gid %d matchGid %d vert %p owner %d\n",
+      PCU_Debug_Print("%d packing match ptr to %d gid %d matchGid %d vert %p owner %d\n",
           self, to, gid, matchGid,
           (void*)verts[matchGid-myOffset], owners[matchGid-myOffset]);
     }
@@ -440,12 +441,12 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
     PCU_COMM_UNPACK(match);
     int owner;
     PCU_COMM_UNPACK(owner);
-    fprintf(stderr, "%d unpacked match ptr from %d gid %d matchGid %d matchPtr %p owner %d\n",
+    PCU_Debug_Print("%d unpacked match ptr from %d gid %d matchGid %d matchPtr %p owner %d\n",
         self, PCU_Comm_Sender(), gid, matchGid, (void*)match, owner);
     PCU_ALWAYS_ASSERT(globalToVert.count(gid));
     MeshEntity* partner = globalToVert[gid];
     if(match == partner && owner == self) {
-      fprintf(stderr, "%d match == partner owner == self match %p partner %p owner %d\n",
+      PCU_Debug_Print("%d match == partner owner == self match %p partner %p owner %d\n",
           self, (void*)match, (void*)partner, owner);
     }
     PCU_ALWAYS_ASSERT(! (match == partner && owner == self) );
@@ -461,7 +462,7 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
   PCU_Comm_Begin();
   APF_CONST_ITERATE(GlobalToVert, globalToVert, it) {
     // send matches to remotes
-    int vtx = it->second;
+    apf::MeshEntity* vtx = it->second;
     apf::Matches matches;
     m->getMatches(vtx, matches);
     if( matches.getSize() ) {
@@ -472,7 +473,7 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
         apf::MeshEntity* rmtVtx = r->second;
         for (size_t j=0; j < matches.getSize(); ++j)
           PCU_COMM_PACK(matches[j].peer,matches[j].entity);
-        fprintf(stderr, "%d packing req ptr to %d gid %d matchGid %d\n",
+        PCU_Debug_Print("%d packing req ptr to %d gid %d matchGid %d\n",
             self, to, gid, matchGid);
       }
     }
@@ -483,7 +484,6 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
   }
   */
 
-
   apf::removeTagFromDimension(m, matchGidTag, 0);
   m->destroyTag(matchGidTag);
 }

From bd66e952d1092b9712711e9d1628636609a53e96 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 16 Jan 2019 22:04:46 -0500
Subject: [PATCH 025/555] brokers of vtx gids store all <owner,ptr> pairs

---
 apf/apfConstruct.cc | 76 ++++++++++++++++++++++++---------------------
 1 file changed, 41 insertions(+), 35 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index fcf7d37f5..efa445946 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -301,8 +301,8 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
     int to = std::min(peers - 1, gid / quotient);
     PCU_COMM_PACK(to, gid);
     if( gid == 66300 || gid == 66350 ) {
-      PCU_Debug_Print("%d requesting matches of gid %d from %d\n",
-          self, gid, to);
+      PCU_Debug_Print("%d requesting matches of gid %d isShared %d isOwned %d from %d\n",
+          self, gid, m->isShared(it->second), m->isOwned(it->second), to);
     }
   }
   PCU_Comm_Send();
@@ -350,32 +350,29 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
    * the entity pointers and owners for mesh vertex gid [0..quotient),
    * process 1 holds gids [quotient..2*quotient), ...
    */
-  typedef std::vector< apf::MeshEntity* > EntPtrs;
-  EntPtrs verts(mySize);
-  int* owners = new int[mySize];
   PCU_Comm_Begin();
   APF_CONST_ITERATE(GlobalToVert, globalToVert, it) {
     MeshEntity* e = it->second;
-    if( m->isOwned(e) ) {
-      int gid = it->first;
-      int to = std::min(peers - 1, gid / quotient);
-      PCU_COMM_PACK(to, gid);
-      PCU_COMM_PACK(to, e);
-      if( gid == 50 || gid == 0 || gid == 66300 || gid == 66350 ) {
-        PCU_Debug_Print("%d packing pointer to %d gid %d vert %p\n",
-            self, to, gid, (void*)e);
-      }
+    int gid = it->first;
+    int to = std::min(peers - 1, gid / quotient);
+    PCU_COMM_PACK(to, gid);
+    PCU_COMM_PACK(to, e);
+    if( gid == 50 || gid == 0 || gid == 66300 || gid == 66350 ) {
+      PCU_Debug_Print("%d packing pointer to %d gid %d vert %p\n",
+          self, to, gid, (void*)e);
     }
   }
   PCU_Comm_Send();
+  typedef std::pair< int, apf::MeshEntity* > EntOwnerPtrs;
+  typedef std::map< int, std::vector< EntOwnerPtrs > > GidPtrs;
+  GidPtrs gidPtrs;
   while (PCU_Comm_Receive()) {
     int gid;
     PCU_COMM_UNPACK(gid);
     MeshEntity* vert;
     PCU_COMM_UNPACK(vert);
     int owner = PCU_Comm_Sender();
-    verts[gid-myOffset] = vert;
-    owners[gid-myOffset] = owner;
+    gidPtrs[gid-myOffset].push_back(EntOwnerPtrs(owner,vert));
     if( gid == 50 || gid == 0 || gid == 66300 || gid == 66350 ) {
       PCU_Debug_Print("%d unpacking pointer from %d gid %d vert %p\n",
           self, owner, gid, (void*)vert);
@@ -424,11 +421,17 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
       const int to = parts[i];
       PCU_COMM_PACK(to, gid);
       PCU_COMM_PACK(to, matchGid);
-      PCU_COMM_PACK(to, verts[matchGid-myOffset]);
-      PCU_COMM_PACK(to, owners[matchGid-myOffset]);
-      PCU_Debug_Print("%d packing match ptr to %d gid %d matchGid %d vert %p owner %d\n",
-          self, to, gid, matchGid,
-          (void*)verts[matchGid-myOffset], owners[matchGid-myOffset]);
+      size_t numMatches = gidPtrs[matchGid-myOffset].size();
+      PCU_COMM_PACK(to, numMatches);
+      for( size_t i=0; i<gidPtrs[matchGid-myOffset].size(); i++) {
+        EntOwnerPtrs eop = gidPtrs[matchGid-myOffset][i];
+        int owner = eop.first;
+        PCU_COMM_PACK(to, owner);
+        apf::MeshEntity* ent = eop.second;
+        PCU_COMM_PACK(to, ent);
+        PCU_Debug_Print("%d packing match ptr to %d gid %d matchGid %d vert %p owner %d\n",
+            self, to, gid, matchGid, (void*)ent, owner);
+      }
     }
   }
   PCU_Comm_Send();
@@ -437,25 +440,28 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
     PCU_COMM_UNPACK(gid);
     int matchGid;
     PCU_COMM_UNPACK(matchGid);
-    MeshEntity* match;
-    PCU_COMM_UNPACK(match);
-    int owner;
-    PCU_COMM_UNPACK(owner);
-    PCU_Debug_Print("%d unpacked match ptr from %d gid %d matchGid %d matchPtr %p owner %d\n",
-        self, PCU_Comm_Sender(), gid, matchGid, (void*)match, owner);
-    PCU_ALWAYS_ASSERT(globalToVert.count(gid));
-    MeshEntity* partner = globalToVert[gid];
-    if(match == partner && owner == self) {
-      PCU_Debug_Print("%d match == partner owner == self match %p partner %p owner %d\n",
-          self, (void*)match, (void*)partner, owner);
+    size_t numMatches;
+    PCU_COMM_UNPACK(numMatches);
+    for(size_t i=0; i<numMatches; i++) {
+      int owner;
+      PCU_COMM_UNPACK(owner);
+      MeshEntity* match;
+      PCU_COMM_UNPACK(match);
+      PCU_Debug_Print("%d unpacked match ptr from %d gid %d matchGid %d matchPtr %p owner %d\n",
+          self, PCU_Comm_Sender(), gid, matchGid, (void*)match, owner);
+      PCU_ALWAYS_ASSERT(globalToVert.count(gid));
+      MeshEntity* partner = globalToVert[gid];
+      if(match == partner && owner == self) {
+        PCU_Debug_Print("%d match == partner owner == self match %p partner %p owner %d\n",
+            self, (void*)match, (void*)partner, owner);
+      }
+      PCU_ALWAYS_ASSERT(! (match == partner && owner == self) );
+      m->addMatch(partner, owner, match);
     }
-    PCU_ALWAYS_ASSERT(! (match == partner && owner == self) );
-    m->addMatch(partner, owner, match);
   }
 
 
   delete [] c;
-  delete [] owners;
 
   /* Owners send match info to copies */
   /*

From 700880ff19c31bdc6aa62a1870906adad28114f7 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Thu, 17 Jan 2019 10:31:02 -0500
Subject: [PATCH 026/555] support 2d meshes, read dim and topo blocks from
 connectivity file

---
 test/matchedNodeElmReader.cc | 75 +++++++++++++++++++++---------------
 1 file changed, 43 insertions(+), 32 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 48bae7e6e..bf56a4688 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -23,15 +23,35 @@ void getLocalRange(unsigned total, unsigned& local,
   last = first+local;
 }
 
-unsigned getElmType(int numVtxPerElm) {
-  if (numVtxPerElm == 4) {
-    return apf::Mesh::TET;
-  } else if (numVtxPerElm == 6) {
-    return apf::Mesh::PRISM;
-  } else if (numVtxPerElm == 8) {
-    return apf::Mesh::HEX;
+void printElmTypeError(int dim, int numVtxPerElm) {
+  fprintf(stderr, "unknown element type for"
+      "dim %d and numVtxPerElm %d in %s\n",
+      dim, numVtxPerElm, __func__);
+}
+
+unsigned getElmType(int dim, int numVtxPerElm) {
+  if (dim == 2) {
+    if (numVtxPerElm == 3)
+      return apf::Mesh::TRIANGLE;
+    if (numVtxPerElm == 4)
+      return apf::Mesh::QUAD;
+    else {
+      printElmTypeError(dim, numVtxPerElm);
+      exit(EXIT_FAILURE);
+    }
+  } else if (dim == 3) {
+    if (numVtxPerElm == 4)
+      return apf::Mesh::TET;
+    else if (numVtxPerElm == 6)
+      return apf::Mesh::PRISM;
+    else if (numVtxPerElm == 8)
+      return apf::Mesh::HEX;
+    else {
+      printElmTypeError(dim, numVtxPerElm);
+      exit(EXIT_FAILURE);
+    }
   } else {
-    fprintf(stderr, "unknown element type in %s\n", __func__);
+    printElmTypeError(dim, numVtxPerElm);
     exit(EXIT_FAILURE);
   }
 }
@@ -42,18 +62,6 @@ bool skipLine(char* line) {
   return (line[0] == '#' || line[0] == ' ' );
 }
 
-void getNumElms(FILE* f, unsigned& elms) {
-  rewind(f);
-  elms = 0;
-  size_t linelimit = 1024;
-  char* line = new char[linelimit];
-  while( gmi_getline(&line,&linelimit,f) != -1 ) {
-    if( ! skipLine(line) )
-      elms++;
-  }
-  delete [] line;
-}
-
 void getNumVerts(FILE* f, unsigned& verts) {
   rewind(f);
   verts = 0;
@@ -112,16 +120,21 @@ void readMatches(FILE* f, unsigned numvtx, int** matches) {
   }
 }
 
-void readElements(FILE* f, unsigned numelms, unsigned numVtxPerElm,
-    unsigned& localNumElms, int** elements) {
+void readElements(FILE* f, unsigned &dim, unsigned& numElms,
+    unsigned& numVtxPerElm, unsigned& localNumElms, int** elements) {
+  rewind(f);
+  int dimHeader[2];
+  gmi_fscanf(f, 2, "%u %u", dimHeader, dimHeader+1);
+  assert( dimHeader[0] == 1 && dimHeader[1] == 1);
+  gmi_fscanf(f, 1, "%u", &dim);
+  gmi_fscanf(f, 2, "%u %u", &numElms, &numVtxPerElm);
   long firstElm, lastElm;
-  getLocalRange(numelms, localNumElms, firstElm, lastElm);
+  getLocalRange(numElms, localNumElms, firstElm, lastElm);
   *elements = new int[localNumElms*numVtxPerElm];
-  rewind(f);
   unsigned i, j;
   unsigned elmIdx = 0;
   int* elmVtx = new int[numVtxPerElm];
-  for (i = 0; i < numelms; i++) {
+  for (i = 0; i < numElms; i++) {
     int ignored;
     gmi_fscanf(f, 1, "%u", &ignored);
     for (j = 0; j < numVtxPerElm; j++)
@@ -141,6 +154,7 @@ struct MeshInfo {
   double* coords;
   int* elements;
   int* matches;
+  unsigned dim;
   unsigned elementType;
   unsigned numVerts;
   unsigned localNumVerts;
@@ -157,10 +171,9 @@ void readMesh(const char* meshfilename,
   PCU_ALWAYS_ASSERT(f);
   FILE* fc = fopen(coordfilename, "r");
   PCU_ALWAYS_ASSERT(fc);
-  getNumElms(f,mesh.numElms);
   getNumVerts(fc,mesh.numVerts);
   if(!PCU_Comm_Self())
-    fprintf(stderr, "numElms %u numVerts %u\n", mesh.numElms, mesh.numVerts);
+    fprintf(stderr, "numVerts %u\n", mesh.numVerts);
   readCoords(fc, mesh.numVerts, mesh.localNumVerts, &(mesh.coords));
   fclose(fc);
   if( strcmp(matchfilename, "NULL") ) {
@@ -169,10 +182,9 @@ void readMesh(const char* meshfilename,
     readMatches(fm, mesh.numVerts, &(mesh.matches));
     fclose(fm);
   }
-  mesh.numVtxPerElm = 8; //hack!
-  readElements(f, mesh.numElms, mesh.numVtxPerElm,
+  readElements(f, mesh.dim, mesh.numElms, mesh.numVtxPerElm,
       mesh.localNumElms, &(mesh.elements));
-  mesh.elementType = getElmType(mesh.numVtxPerElm);
+  mesh.elementType = getElmType(mesh.dim, mesh.numVtxPerElm);
   fclose(f);
 }
 
@@ -207,9 +219,8 @@ int main(int argc, char** argv)
   if(!PCU_Comm_Self())
     fprintf(stderr, "isMatched %d\n", isMatched);
 
-  const int dim = 3;
   gmi_model* model = gmi_load(".null");
-  apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, dim, isMatched);
+  apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, m.dim, isMatched);
   apf::GlobalToVert outMap;
   apf::construct(mesh, m.elements, m.localNumElms, m.elementType, outMap);
   delete [] m.elements;

From 75db2593d0d495b96ca0b3148c7d98aa01c75295 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Thu, 17 Jan 2019 11:45:34 -0500
Subject: [PATCH 027/555] create a match for each remote copy

---
 apf/apfConstruct.cc | 59 ++++++++++++++++-----------------------------
 1 file changed, 21 insertions(+), 38 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index efa445946..59be547df 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -300,10 +300,8 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
     int gid = it->first;
     int to = std::min(peers - 1, gid / quotient);
     PCU_COMM_PACK(to, gid);
-    if( gid == 66300 || gid == 66350 ) {
-      PCU_Debug_Print("%d requesting matches of gid %d isShared %d isOwned %d from %d\n",
-          self, gid, m->isShared(it->second), m->isOwned(it->second), to);
-    }
+    PCU_Debug_Print("%d requesting matches of gid %d isShared %d isOwned %d from %d\n",
+        self, gid, m->isShared(it->second), m->isOwned(it->second), to);
   }
   PCU_Comm_Send();
   while (PCU_Comm_Receive()) {
@@ -357,10 +355,8 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
     int to = std::min(peers - 1, gid / quotient);
     PCU_COMM_PACK(to, gid);
     PCU_COMM_PACK(to, e);
-    if( gid == 50 || gid == 0 || gid == 66300 || gid == 66350 ) {
-      PCU_Debug_Print("%d packing pointer to %d gid %d vert %p\n",
-          self, to, gid, (void*)e);
-    }
+    PCU_Debug_Print("%d packing pointer to %d gid %d vert %p\n",
+        self, to, gid, (void*)e);
   }
   PCU_Comm_Send();
   typedef std::pair< int, apf::MeshEntity* > EntOwnerPtrs;
@@ -373,10 +369,8 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
     PCU_COMM_UNPACK(vert);
     int owner = PCU_Comm_Sender();
     gidPtrs[gid-myOffset].push_back(EntOwnerPtrs(owner,vert));
-    if( gid == 50 || gid == 0 || gid == 66300 || gid == 66350 ) {
-      PCU_Debug_Print("%d unpacking pointer from %d gid %d vert %p\n",
-          self, owner, gid, (void*)vert);
-    }
+    PCU_Debug_Print("%d unpacking pointer from %d gid %d vert %p\n",
+        self, owner, gid, (void*)vert);
   }
 
   /* Tell the brokers of the matches we need */
@@ -460,35 +454,24 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
     }
   }
 
-
-  delete [] c;
-
-  /* Owners send match info to copies */
-  /*
-  PCU_Comm_Begin();
-  APF_CONST_ITERATE(GlobalToVert, globalToVert, it) {
-    // send matches to remotes
-    apf::MeshEntity* vtx = it->second;
-    apf::Matches matches;
-    m->getMatches(vtx, matches);
-    if( matches.getSize() ) {
-      apf::Copies remotes;
-      m->getRemotes(vtx, remotes);
-      APF_ITERATE(apf::Copies, remotes, r) {
-        int peer = r->first;
-        apf::MeshEntity* rmtVtx = r->second;
-        for (size_t j=0; j < matches.getSize(); ++j)
-          PCU_COMM_PACK(matches[j].peer,matches[j].entity);
-        PCU_Debug_Print("%d packing req ptr to %d gid %d matchGid %d\n",
-            self, to, gid, matchGid);
+  APF_CONST_ITERATE(GlobalToVert, globalToVert, it) { //loop over local verts
+    apf::MeshEntity* left = it->second;
+    int matchGid;
+    m->getIntTag(left, matchGidTag, &matchGid); //get the matched ent gid
+    if( matchGid != -1 ) {  // a matched vtx
+      apf::Copies copies;
+      m->getRemotes(left,copies);
+      APF_ITERATE(apf::Copies, copies, cp) {
+        int rightPart = cp->first;
+        apf::MeshEntity* right = cp->second;
+        m->addMatch(left, rightPart, right);
+        PCU_Debug_Print("%d add remote copy match ptr to %d gid %d\n",
+          self, rightPart, it->first);
       }
     }
   }
-  PCU_Comm_Send();
-  while (PCU_Comm_Receive()) {
-    //unpack and create matches
-  }
-  */
+
+  delete [] c;
 
   apf::removeTagFromDimension(m, matchGidTag, 0);
   m->destroyTag(matchGidTag);

From 04f4a94ac1e6f0c9ce16efefe5db10bcdb5f4879 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Thu, 17 Jan 2019 19:17:53 -0500
Subject: [PATCH 028/555] helper fn to read user ints

---
 apf/apfConstruct.cc | 85 ++++++++++++++++++++++++++++++++++++++++++++-
 apf/apfConvert.h    | 13 +++++++
 2 files changed, 97 insertions(+), 1 deletion(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index 59be547df..daee252b2 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -213,7 +213,7 @@ void setCoords(Mesh2* m, const double* coords, int nverts,
     int from = PCU_Comm_Sender();
     tmpParts.at(gid - myOffset).push_back(from);
   }
-  
+
   /* Send the coords to everybody who want them */
   PCU_Comm_Begin();
   for (int i = 0; i < mySize; ++i) {
@@ -238,6 +238,89 @@ void setCoords(Mesh2* m, const double* coords, int nverts,
   delete [] c;
 }
 
+apf::MeshTag* setIntTag(Mesh2* m, const int* vals, const int entries,
+    int nverts, GlobalToVert& globalToVert)
+{
+  Gid max = getMax(globalToVert);
+  Gid total = max + 1;
+  int peers = PCU_Comm_Peers();
+  int quotient = total / peers;
+  int remainder = total % peers;
+  int mySize = quotient;
+  int self = PCU_Comm_Self();
+  if (self == (peers - 1))
+    mySize += remainder;
+  int myOffset = self * quotient;
+
+  /* Force each peer to have exactly mySize verts.
+     This means we might need to send and recv some coords */
+  int* c = new int[mySize*entries];
+
+  int start = PCU_Exscan_Int(nverts);
+
+  PCU_Comm_Begin();
+  int to = std::min(peers - 1, start / quotient);
+  int n = std::min((to+1)*quotient-start, nverts);
+  while (nverts > 0) {
+    PCU_COMM_PACK(to, start);
+    PCU_COMM_PACK(to, n);
+    PCU_Comm_Pack(to, vals, n*entries*sizeof(int));
+
+    nverts -= n;
+    start += n;
+    vals += n*entries;
+    to = std::min(peers - 1, to + 1);
+    n = std::min(quotient, nverts);
+  }
+  PCU_Comm_Send();
+  while (PCU_Comm_Receive()) {
+    PCU_COMM_UNPACK(start);
+    PCU_COMM_UNPACK(n);
+    PCU_Comm_Unpack(&c[(start - myOffset) * entries], n*entries*sizeof(int));
+  }
+
+  /* Tell all the owners of the data what we need */
+  typedef std::vector< std::vector<int> > TmpParts;
+  TmpParts tmpParts(mySize);
+  PCU_Comm_Begin();
+  APF_CONST_ITERATE(GlobalToVert, globalToVert, it) {
+    int gid = it->first;
+    int to = std::min(peers - 1, gid / quotient);
+    PCU_COMM_PACK(to, gid);
+  }
+  PCU_Comm_Send();
+  while (PCU_Comm_Receive()) {
+    int gid;
+    PCU_COMM_UNPACK(gid);
+    int from = PCU_Comm_Sender();
+    tmpParts.at(gid - myOffset).push_back(from);
+  }
+
+  /* Send the data to everybody who want them */
+  PCU_Comm_Begin();
+  for (int i = 0; i < mySize; ++i) {
+    std::vector<int>& parts = tmpParts[i];
+    for (size_t j = 0; j < parts.size(); ++j) {
+      int to = parts[j];
+      int gid = i + myOffset;
+      PCU_COMM_PACK(to, gid);
+      PCU_Comm_Pack(to, &c[i*entries], entries*sizeof(int));
+    }
+  }
+  PCU_Comm_Send();
+  apf::MeshTag* t = m->createIntTag("userInts",entries);
+  int* v = new int[entries];
+  while (PCU_Comm_Receive()) {
+    int gid;
+    PCU_COMM_UNPACK(gid);
+    PCU_Comm_Unpack(v, entries*sizeof(int));
+    m->setIntTag(globalToVert[gid],t,v);
+  }
+  delete [] v;
+  delete [] c;
+  return t;
+}
+
 void setMatches(Mesh2* m, const int* matches, int nverts,
     GlobalToVert& globalToVert)
 {
diff --git a/apf/apfConvert.h b/apf/apfConvert.h
index 6b55567c2..2bb27282b 100644
--- a/apf/apfConvert.h
+++ b/apf/apfConvert.h
@@ -17,6 +17,7 @@ namespace apf {
 
 class Mesh;
 class Mesh2;
+class MeshTag;
 class ModelEntity;
 class MeshEntity;
 
@@ -59,6 +60,18 @@ void construct(Mesh2* m, const int* conn, int nelem, int etype,
 void setCoords(Mesh2* m, const double* coords, int nverts,
     GlobalToVert& globalToVert);
 
+/** \brief Assign coordinates to the mesh
+  * \param vals (in) array of nverts ints
+  * \param entries (in) number of ints per vertex
+  * \param nverts (in) number of vertices for this process
+  * \param globalToVert (in) map from global mesh vertex ids
+  *                          to local vertex * pointers
+  * \details
+  * See 'setCoords' for distribution details
+  */
+MeshTag* setIntTag(Mesh2* m, const int* vals, const int entries,
+    int nverts, GlobalToVert& globalToVert);
+
 /** \brief Assign matching to the mesh
   * \details
   * Each peer provides a set of the matched entity global ids. An id set

From be9dbe1d1878bc2749700db168d5f5694ce59da5 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Thu, 17 Jan 2019 19:18:08 -0500
Subject: [PATCH 029/555] start classification

---
 test/matchedNodeElmReader.cc | 117 +++++++++++++++++++++++++++++++++--
 1 file changed, 113 insertions(+), 4 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index bf56a4688..e8c8e95af 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -11,6 +11,82 @@
 #include <string.h>
 #include <cassert>
 
+/* from https://github.com/SCOREC/core/issues/205
+0=fully interior of the volume
+1-6 =classified on face (not edge or vertex)
+11-22 = classified on model edge (not end points which are model vertices)
+31-38 = classified on a model vertex.
+*/
+
+/* tags on vertices */
+#define INTERIORTAG  0
+#define FACE 1
+#define FACE_LAST 1
+#define EDGE 11
+#define EDGE_LAST 22
+#define VERTEX 31
+#define VERTEX_LAST 38
+
+/* model entity ids */
+#define INTERIOR_REGION 1
+
+apf::ModelEntity* getMdlRgn(gmi_model* model) {
+  apf::ModelEntity* rgn = reinterpret_cast<apf::ModelEntity*>(
+      gmi_find(model, 3, INTERIOR_REGION));
+  PCU_ALWAYS_ASSERT(rgn);
+  return rgn;
+}
+
+apf::ModelEntity* getMdlEdge(apf::Mesh2* mesh, int tag) {
+  apf::ModelEntity* edge = mesh->findModelEntity(1,tag);
+  PCU_ALWAYS_ASSERT(edge);
+  return edge;
+}
+
+apf::ModelEntity* getMdlFace(apf::Mesh2* mesh, int tag) {
+  apf::ModelEntity* face = mesh->findModelEntity(2,tag);
+  PCU_ALWAYS_ASSERT(face);
+  return face;
+}
+
+void setVtxClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* t) {
+  (void)model;
+  (void)mesh;
+  (void)t;
+}
+
+void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh) {
+  (void)model;
+  (void)mesh;
+}
+
+void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtxType) {
+  (void)model;
+  (void)mesh;
+  (void)vtxType;
+}
+
+/** \brief set the mesh region classification
+  \details hacked to set the classification to the same geometric model region
+*/
+void setRgnClassification(gmi_model* model, apf::Mesh2* mesh) {
+  apf::ModelEntity* mdlRgn = getMdlRgn(model);
+  apf::MeshIterator* it = mesh->begin(3);
+  apf::MeshEntity* rgn;
+  while( (rgn = mesh->iterate(it)) )
+    mesh->setModelEntity(rgn,mdlRgn);
+  mesh->end(it);
+}
+
+void setClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* t) {
+  setRgnClassification(model,mesh);
+  setFaceClassification(model,mesh,t);
+  setEdgeClassification(model,mesh);
+  setVtxClassification(model,mesh,t);
+  mesh->acceptChanges();
+}
+
+
 void getLocalRange(unsigned total, unsigned& local,
     long& first, long& last) {
   const int self = PCU_Comm_Self();
@@ -74,6 +150,24 @@ void getNumVerts(FILE* f, unsigned& verts) {
   delete [] line;
 }
 
+void readClassification(FILE* f, unsigned numVtx, int** classification) {
+  long firstVtx, lastVtx;
+  unsigned localNumVtx;
+  getLocalRange(numVtx,localNumVtx,firstVtx,lastVtx);
+  *classification = new int[localNumVtx];
+  rewind(f);
+  int vidx = 0;
+  for(unsigned i=0; i<numVtx; i++) {
+    int id;
+    int mdlId;
+    gmi_fscanf(f, 2, "%d %d", &id, &mdlId);
+    if( i >= firstVtx && i < lastVtx ) {
+      (*classification)[vidx] = mdlId;
+      vidx++;
+    }
+  }
+}
+
 void readCoords(FILE* f, unsigned numvtx, unsigned& localnumvtx, double** coordinates) {
   long firstVtx, lastVtx;
   getLocalRange(numvtx, localnumvtx,firstVtx,lastVtx);
@@ -154,6 +248,7 @@ struct MeshInfo {
   double* coords;
   int* elements;
   int* matches;
+  int* classification;
   unsigned dim;
   unsigned elementType;
   unsigned numVerts;
@@ -166,9 +261,8 @@ struct MeshInfo {
 void readMesh(const char* meshfilename,
     const char* coordfilename,
     const char* matchfilename,
+    const char* classfilename,
     MeshInfo& mesh) {
-  FILE* f = fopen(meshfilename, "r");
-  PCU_ALWAYS_ASSERT(f);
   FILE* fc = fopen(coordfilename, "r");
   PCU_ALWAYS_ASSERT(fc);
   getNumVerts(fc,mesh.numVerts);
@@ -176,12 +270,21 @@ void readMesh(const char* meshfilename,
     fprintf(stderr, "numVerts %u\n", mesh.numVerts);
   readCoords(fc, mesh.numVerts, mesh.localNumVerts, &(mesh.coords));
   fclose(fc);
+
+  FILE* ff = fopen(classfilename, "r");
+  PCU_ALWAYS_ASSERT(ff);
+  readClassification(ff, mesh.numVerts, &(mesh.classification));
+  fclose(ff);
+
   if( strcmp(matchfilename, "NULL") ) {
     FILE* fm = fopen(matchfilename, "r");
     PCU_ALWAYS_ASSERT(fm);
     readMatches(fm, mesh.numVerts, &(mesh.matches));
     fclose(fm);
   }
+
+  FILE* f = fopen(meshfilename, "r");
+  PCU_ALWAYS_ASSERT(f);
   readElements(f, mesh.dim, mesh.numElms, mesh.numVtxPerElm,
       mesh.localNumElms, &(mesh.elements));
   mesh.elementType = getElmType(mesh.dim, mesh.numVtxPerElm);
@@ -193,11 +296,12 @@ int main(int argc, char** argv)
   MPI_Init(&argc,&argv);
   PCU_Comm_Init();
   lion_set_verbosity(1);
-  if( argc != 6 ) {
+  if( argc != 7 ) {
     if( !PCU_Comm_Self() ) {
       printf("Usage: %s <ascii mesh connectivity .cnn> "
           "<ascii vertex coordinates .crd> "
           "<ascii vertex matching flag .match> "
+          "<ascii vertex classification flag .class> "
           "<output model .dmg> <output mesh .smb>\n",
           argv[0]);
     }
@@ -210,7 +314,7 @@ int main(int argc, char** argv)
 
   double t0 = PCU_Time();
   MeshInfo m;
-  readMesh(argv[1],argv[2],argv[3],m);
+  readMesh(argv[1],argv[2],argv[3],argv[4],m);
 
   bool isMatched = true;
   if( !strcmp(argv[3], "NULL") )
@@ -233,7 +337,12 @@ int main(int argc, char** argv)
     mesh->acceptChanges();
     delete [] m.matches;
   }
+  apf::MeshTag* t = setIntTag(mesh, m.classification, 1,
+      m.localNumVerts, outMap);
   outMap.clear();
+  setClassification(model,mesh,t);
+  apf::removeTagFromDimension(mesh, t, 0);
+  mesh->destroyTag(t);
   if(!PCU_Comm_Self())
     fprintf(stderr, "seconds to create mesh %.3f\n", PCU_Time()-t0);
   mesh->verify();

From d0132907a9bd96263167551c39285c720c97b25c Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Mon, 28 Jan 2019 13:28:55 -0500
Subject: [PATCH 030/555] WIP: set face classification

probably does not compile
---
 test/matchedNodeElmReader.cc | 26 +++++++++++++++++++++++---
 1 file changed, 23 insertions(+), 3 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index e8c8e95af..b2c2de83f 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -60,10 +60,31 @@ void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh) {
   (void)mesh;
 }
 
-void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtxType) {
+/* if any of four vertices are classified on region -> region
+ * else on model face and it is impossible to have more than one face in the 4
+ * vertices classification
+ * */
+void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtxClass) {
   (void)model;
   (void)mesh;
-  (void)vtxType;
+
+  apf::MeshIterator* it = mesh->begin(2);
+  apf::MeshEntity* f;
+  int c;
+
+  apf::Adjacent verts;
+  while( (f = mesh->iterate(it)) ) {
+    m->getAdjacent(f, 0, verts) = 0;
+    bool hasRgClass = false;
+    for(int i=0; i<verts.size(); i++) {
+      m->getIntTag(f,vtxClass,&c);
+      if( c == INTERIORTAG )
+        mesh->setModelEntity(f,mdlRgn);
+      }
+      
+    }
+  }
+  mesh->end(it);
 }
 
 /** \brief set the mesh region classification
@@ -86,7 +107,6 @@ void setClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* t) {
   mesh->acceptChanges();
 }
 
-
 void getLocalRange(unsigned total, unsigned& local,
     long& first, long& last) {
   const int self = PCU_Comm_Self();

From 16f79b467d9373beef87f17d4bdd0e447db5577b Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Wed, 30 Jan 2019 14:58:20 -0700
Subject: [PATCH 031/555] copied over Riccardo's current

---
 cmake/FindSimModSuite.cmake  |   4 +-
 test/matchedNodeElmReader.cc | 113 +++++++++++++++++++++++++++++++----
 2 files changed, 102 insertions(+), 15 deletions(-)

diff --git a/cmake/FindSimModSuite.cmake b/cmake/FindSimModSuite.cmake
index 9129f5051..4ddfb8632 100644
--- a/cmake/FindSimModSuite.cmake
+++ b/cmake/FindSimModSuite.cmake
@@ -74,7 +74,7 @@ string(REGEX REPLACE
   "${SIM_VERSION}")
 
 set(MIN_VALID_SIM_VERSION 11.0.170826)
-set(MAX_VALID_SIM_VERSION 12.0.181124)
+set(MAX_VALID_SIM_VERSION 18.0.181124)
 if( (SIM_DOT_VERSION VERSION_LESS MIN_VALID_SIM_VERSION) OR
     (SIM_DOT_VERSION VERSION_GREATER MAX_VALID_SIM_VERSION) )
   MESSAGE(FATAL_ERROR 
@@ -97,7 +97,7 @@ math(EXPR len "${archEnd}-${archStart}")
 string(SUBSTRING "${SIMMODSUITE_LIBS}" "${archStart}" "${len}" SIM_ARCHOS)
 message(STATUS "SIM_ARCHOS ${SIM_ARCHOS}")
 
-set(SIM_PARASOLID_VERSION 290)
+set(SIM_PARASOLID_VERSION 300)
 option(SIM_PARASOLID "Use Parasolid through Simmetrix" OFF)
 if (SIM_PARASOLID)
   getSimCadLib("${SIMMODSUITE_INSTALL_DIR}/lib/${SIM_ARCHOS}" 
diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index b2c2de83f..de76d0dd2 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -10,6 +10,7 @@
 #include <cstdlib>
 #include <string.h>
 #include <cassert>
+#include <algorithm>
 
 /* from https://github.com/SCOREC/core/issues/205
 0=fully interior of the volume
@@ -21,7 +22,7 @@
 /* tags on vertices */
 #define INTERIORTAG  0
 #define FACE 1
-#define FACE_LAST 1
+#define FACE_LAST 6
 #define EDGE 11
 #define EDGE_LAST 22
 #define VERTEX 31
@@ -37,27 +38,84 @@ apf::ModelEntity* getMdlRgn(gmi_model* model) {
   return rgn;
 }
 
+apf::ModelEntity* getMdlFace(apf::Mesh2* mesh, int tag) {
+  apf::ModelEntity* face = mesh->findModelEntity(2,tag);
+  PCU_ALWAYS_ASSERT(face);
+  return face;
+}
+
 apf::ModelEntity* getMdlEdge(apf::Mesh2* mesh, int tag) {
   apf::ModelEntity* edge = mesh->findModelEntity(1,tag);
   PCU_ALWAYS_ASSERT(edge);
   return edge;
 }
 
-apf::ModelEntity* getMdlFace(apf::Mesh2* mesh, int tag) {
-  apf::ModelEntity* face = mesh->findModelEntity(2,tag);
-  PCU_ALWAYS_ASSERT(face);
-  return face;
+apf::ModelEntity* getMdlVtx(apf::Mesh2* mesh, int tag) {
+  apf::ModelEntity* vertex = mesh->findModelEntity(0,tag);
+  PCU_ALWAYS_ASSERT(vertex);
+  return vertex;
 }
 
-void setVtxClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* t) {
+void setVtxClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtxClass) {
   (void)model;
   (void)mesh;
-  (void)t;
+  (void)vtxClass;
+  apf::MeshIterator* it = mesh->begin(0);
+  apf::MeshEntity* v;
+  apf::Vector3 vCoord;
+  int c;
+  //int count=0,cint=0,cface=0,cedge=0,cvtx=0;
+  while( (v = mesh->iterate(it)) ) {
+    mesh->getPoint(v, 0, vCoord);
+    //std::cout<<"Coordinates: "<<vCoord[0]<<" "<<vCoord[1]<<" "<<vCoord[2]<<std::endl;
+    mesh->getIntTag(v,vtxClass,&c);
+    //std::cout<<"Returned tag is c= "<<c<<std::endl;
+    //std::cout<<" "<<std::endl;
+    //count++;
+    if (c == INTERIORTAG) {
+       mesh->setModelEntity(v,getMdlRgn(model));
+       //cint++;
+    } else if (c >= FACE && c <= FACE_LAST) {
+       mesh->setModelEntity(v,getMdlFace(mesh,c));
+       //cface++;
+    } else if (c >= EDGE && c <= EDGE_LAST) {
+       mesh->setModelEntity(v,getMdlEdge(mesh,c));
+       //cedge++;
+    } else if (c >= VERTEX && c <= VERTEX_LAST) {
+       mesh->setModelEntity(v,getMdlVtx(mesh,c));
+       //cvtx++;
+    }
+  }
+  //std::cout<<"count is "<<cvtx<<std::endl;
+  mesh->end(it);
 }
 
-void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh) {
+void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh,apf::MeshTag* vtxClass) {
   (void)model;
   (void)mesh;
+  (void)vtxClass;
+  apf::MeshIterator* it = mesh->begin(1);
+  apf::MeshEntity* e;
+  int c;
+  apf::Adjacent verts;
+  while( (e = mesh->iterate(it)) ) {
+    mesh->getAdjacent(e, 0, verts);
+    int cmin=100;
+    for(int i=0; i<verts.size(); i++) {
+      mesh->getIntTag(verts[i],vtxClass,&c);
+      cmin=std::min(cmin,c);
+    }
+    if (cmin == INTERIORTAG) {
+       mesh->setModelEntity(e,getMdlRgn(model));
+    } else if (cmin >= FACE && cmin <= FACE_LAST) {
+       mesh->setModelEntity(e,getMdlFace(mesh,cmin));
+    } else if (cmin >= EDGE && cmin <= EDGE_LAST) {
+       mesh->setModelEntity(e,getMdlEdge(mesh,cmin));
+    } else if (cmin >= VERTEX && cmin <= VERTEX_LAST) {
+       mesh->setModelEntity(e,getMdlVtx(mesh,cmin));
+    }
+  }
+  mesh->end(it);
 }
 
 /* if any of four vertices are classified on region -> region
@@ -67,23 +125,43 @@ void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh) {
 void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtxClass) {
   (void)model;
   (void)mesh;
+  (void)vtxClass;
 
   apf::MeshIterator* it = mesh->begin(2);
   apf::MeshEntity* f;
   int c;
-
+/* What was here before was commented when we went to simple min rule
   apf::Adjacent verts;
   while( (f = mesh->iterate(it)) ) {
     m->getAdjacent(f, 0, verts) = 0;
     bool hasRgClass = false;
     for(int i=0; i<verts.size(); i++) {
-      m->getIntTag(f,vtxClass,&c);
+      m->getIntTag(i,vtxClass,&c);
       if( c == INTERIORTAG )
         mesh->setModelEntity(f,mdlRgn);
       }
       
     }
   }
+*/
+  apf::Adjacent verts;
+  while( (f = mesh->iterate(it)) ) {
+    mesh->getAdjacent(f, 0, verts);
+    int cmin=100;
+    for(int i=0; i<verts.size(); i++) {
+      mesh->getIntTag(verts[i],vtxClass,&c);
+      cmin=std::min(cmin,c);
+    }
+    if (cmin == INTERIORTAG) {
+       mesh->setModelEntity(f,getMdlRgn(model));
+    } else if (cmin >= FACE && cmin <= FACE_LAST) {
+       mesh->setModelEntity(f,getMdlFace(mesh,cmin));
+    } else if (cmin >= EDGE && cmin <= EDGE_LAST) {
+       mesh->setModelEntity(f,getMdlEdge(mesh,cmin));
+    } else if (cmin >= VERTEX && cmin <= VERTEX_LAST) {
+       mesh->setModelEntity(f,getMdlVtx(mesh,cmin));
+    }
+  }
   mesh->end(it);
 }
 
@@ -102,7 +180,7 @@ void setRgnClassification(gmi_model* model, apf::Mesh2* mesh) {
 void setClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* t) {
   setRgnClassification(model,mesh);
   setFaceClassification(model,mesh,t);
-  setEdgeClassification(model,mesh);
+  setEdgeClassification(model,mesh,t);
   setVtxClassification(model,mesh,t);
   mesh->acceptChanges();
 }
@@ -181,11 +259,17 @@ void readClassification(FILE* f, unsigned numVtx, int** classification) {
     int id;
     int mdlId;
     gmi_fscanf(f, 2, "%d %d", &id, &mdlId);
+    //std::cout<<"read id is "<<id<<std::endl;
+    //std::cout<<"read model id is "<<mdlId<<std::endl;
     if( i >= firstVtx && i < lastVtx ) {
       (*classification)[vidx] = mdlId;
       vidx++;
     }
   }
+  /*std::cout<<"numVtx is "<<numVtx<<std::endl;
+  for (int i=0;i<numVtx;i++) {
+      std::cout<<"vtx num "<<i<<" has class "<<(*classification)[i]<<std::endl;
+  }*/
 }
 
 void readCoords(FILE* f, unsigned numvtx, unsigned& localnumvtx, double** coordinates) {
@@ -350,6 +434,9 @@ int main(int argc, char** argv)
   delete [] m.elements;
   apf::alignMdsRemotes(mesh);
   apf::deriveMdsModel(mesh);
+  /*for (int i=0; i<81; i++) {
+  std::cout<<m.coords[i]<<std::endl;
+  }*/
   apf::setCoords(mesh, m.coords, m.localNumVerts, outMap);
   delete [] m.coords;
   if( isMatched ) {
@@ -367,8 +454,8 @@ int main(int argc, char** argv)
     fprintf(stderr, "seconds to create mesh %.3f\n", PCU_Time()-t0);
   mesh->verify();
 
-  gmi_write_dmg(model, argv[4]);
-  mesh->writeNative(argv[5]);
+  gmi_write_dmg(model, argv[5]);
+  mesh->writeNative(argv[6]);
   apf::writeVtkFiles("rendered",mesh);
 
   mesh->destroyNative();

From e24aee03b2663c75bca2d3a5be104cf924b09979 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Thu, 14 Feb 2019 11:43:35 -0700
Subject: [PATCH 032/555] stale development -- unsure but committing before
 merging with Riccardos dev

---
 mds/apfBox.cc                |  3 +++
 test/matchedNodeElmReader.cc | 45 ++++++++++++++++++++++++++----------
 2 files changed, 36 insertions(+), 12 deletions(-)

diff --git a/mds/apfBox.cc b/mds/apfBox.cc
index 0e8ecf88d..8891aca9b 100644
--- a/mds/apfBox.cc
+++ b/mds/apfBox.cc
@@ -86,6 +86,7 @@ BoxBuilder::BoxBuilder(int nx, int ny, int nz,
 void BoxBuilder::formModelTable()
 {
   int nd[4] = {0,0,0,0};
+//  int nds[4] = {31,11,1,0};
   for (int i = 0; i < mgrid.total(); ++i) {
     Indices mi = mgrid.out(i);
     int mdim = 0;
@@ -94,6 +95,8 @@ void BoxBuilder::formModelTable()
         ++mdim;
     modelTable[i].dim = mdim;
     modelTable[i].tag = nd[mdim]++;
+//    modelTable[i].tag = nds[mdim]++;
+//    nd[mdim]++;
   }
   for (int i = 0; i < 4; ++i)
     modelCounts[i] = nd[i];
diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index de76d0dd2..918c04607 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -11,6 +11,7 @@
 #include <string.h>
 #include <cassert>
 #include <algorithm>
+//#include <apfBox.h>
 
 /* from https://github.com/SCOREC/core/issues/205
 0=fully interior of the volume
@@ -29,7 +30,7 @@
 #define VERTEX_LAST 38
 
 /* model entity ids */
-#define INTERIOR_REGION 1
+#define INTERIOR_REGION 0
 
 apf::ModelEntity* getMdlRgn(gmi_model* model) {
   apf::ModelEntity* rgn = reinterpret_cast<apf::ModelEntity*>(
@@ -62,11 +63,11 @@ void setVtxClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtxC
   (void)vtxClass;
   apf::MeshIterator* it = mesh->begin(0);
   apf::MeshEntity* v;
-  apf::Vector3 vCoord;
+  //apf::Vector3 vCoord;
   int c;
   //int count=0,cint=0,cface=0,cedge=0,cvtx=0;
   while( (v = mesh->iterate(it)) ) {
-    mesh->getPoint(v, 0, vCoord);
+    //mesh->getPoint(v, 0, vCoord);
     //std::cout<<"Coordinates: "<<vCoord[0]<<" "<<vCoord[1]<<" "<<vCoord[2]<<std::endl;
     mesh->getIntTag(v,vtxClass,&c);
     //std::cout<<"Returned tag is c= "<<c<<std::endl;
@@ -96,15 +97,23 @@ void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh,apf::MeshTag* vtxC
   (void)vtxClass;
   apf::MeshIterator* it = mesh->begin(1);
   apf::MeshEntity* e;
+  //apf::Vector3 vCoord;
   int c;
+  //int count=0;
   apf::Adjacent verts;
   while( (e = mesh->iterate(it)) ) {
+    //std::cout<<"Edge number "<<count++<<" with nodes"<<std::endl;
     mesh->getAdjacent(e, 0, verts);
     int cmin=100;
-    for(int i=0; i<verts.size(); i++) {
+// won't compile now??    for(int i=0; i<verts.size(); i++) {
+    for(int i=0; i<2; i++) {
       mesh->getIntTag(verts[i],vtxClass,&c);
+      //mesh->getPoint(verts[i], 0, vCoord);
+      //std::cout<<vCoord[0]<<" "<<vCoord[1]<<" "<<vCoord[2]<<std::endl;
       cmin=std::min(cmin,c);
     }
+    //std::cout<<"has classification "<<cmin<<std::endl;
+    //std::cout<<" "<<std::endl;
     if (cmin == INTERIORTAG) {
        mesh->setModelEntity(e,getMdlRgn(model));
     } else if (cmin >= FACE && cmin <= FACE_LAST) {
@@ -148,7 +157,8 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
   while( (f = mesh->iterate(it)) ) {
     mesh->getAdjacent(f, 0, verts);
     int cmin=100;
-    for(int i=0; i<verts.size(); i++) {
+// won't comile now    for(int i=0; i<verts.size(); i++) {
+    for(int i=0; i<4; i++) {
       mesh->getIntTag(verts[i],vtxClass,&c);
       cmin=std::min(cmin,c);
     }
@@ -375,10 +385,12 @@ void readMesh(const char* meshfilename,
   readCoords(fc, mesh.numVerts, mesh.localNumVerts, &(mesh.coords));
   fclose(fc);
 
-  FILE* ff = fopen(classfilename, "r");
-  PCU_ALWAYS_ASSERT(ff);
-  readClassification(ff, mesh.numVerts, &(mesh.classification));
-  fclose(ff);
+  if( strcmp(matchfilename, "NULL") ) {
+    FILE* ff = fopen(classfilename, "r");
+    PCU_ALWAYS_ASSERT(ff);
+    readClassification(ff, mesh.numVerts, &(mesh.classification));
+    fclose(ff);
+  }
 
   if( strcmp(matchfilename, "NULL") ) {
     FILE* fm = fopen(matchfilename, "r");
@@ -395,6 +407,7 @@ void readMesh(const char* meshfilename,
   fclose(f);
 }
 
+
 int main(int argc, char** argv)
 {
   MPI_Init(&argc,&argv);
@@ -406,7 +419,7 @@ int main(int argc, char** argv)
           "<ascii vertex coordinates .crd> "
           "<ascii vertex matching flag .match> "
           "<ascii vertex classification flag .class> "
-          "<output model .dmg> <output mesh .smb>\n",
+          "<input model .dmg> <output mesh .smb>\n",
           argv[0]);
     }
     return 0;
@@ -427,13 +440,21 @@ int main(int argc, char** argv)
   if(!PCU_Comm_Self())
     fprintf(stderr, "isMatched %d\n", isMatched);
 
-  gmi_model* model = gmi_load(".null");
+  gmi_model* model = gmi_load("modbox.dmg");
+/*  int nx=3;
+  int ny=3;
+  int nz=3;
+  mgrid(nx ? 3 : 1, ny ? 3 : 1, nz ? 3 : 1);  
+  formModelTable();
+  gmi_model* model = buildModel(); 
+*/
+
   apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, m.dim, isMatched);
   apf::GlobalToVert outMap;
   apf::construct(mesh, m.elements, m.localNumElms, m.elementType, outMap);
   delete [] m.elements;
   apf::alignMdsRemotes(mesh);
-  apf::deriveMdsModel(mesh);
+//  apf::deriveMdsModel(mesh);
   /*for (int i=0; i<81; i++) {
   std::cout<<m.coords[i]<<std::endl;
   }*/

From 95890431e082e2305270fdbaab87ca1c9c459108 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Thu, 14 Feb 2019 12:36:52 -0700
Subject: [PATCH 033/555] merged in Riccardos changes that altered the box code
 to return a model. Confirmed this work on his small case

---
 mds/apfBox.cc                |   4 +-
 mds/apfBox.h                 |   2 +-
 test/matchedNodeElmReader.cc | 215 ++++++++++++++++++++++++++++++-----
 3 files changed, 187 insertions(+), 34 deletions(-)

diff --git a/mds/apfBox.cc b/mds/apfBox.cc
index 8891aca9b..56154c916 100644
--- a/mds/apfBox.cc
+++ b/mds/apfBox.cc
@@ -330,12 +330,12 @@ void BoxBuilder::buildMeshAndModel()
   m->acceptChanges();
 }
 
-Mesh2* makeMdsBox(
+gmi_model* makeMdsBox(
     int nex, int ney, int nez,
     double wx, double wy, double wz, bool is)
 {
   BoxBuilder bb(nex, ney, nez, wx, wy, wz, is);
-  return bb.m;
+  return bb.buildModel();
 }
 
 }
diff --git a/mds/apfBox.h b/mds/apfBox.h
index 5e61ae317..33d6c94b5 100644
--- a/mds/apfBox.h
+++ b/mds/apfBox.h
@@ -70,7 +70,7 @@ struct BoxBuilder
   \param wz z dimension width
   \param is true = simplical mesh, false = quad/hex
   \details set ny,nz=0 for a 1D mesh, set nz=0 for a 2D mesh */
-Mesh2* makeMdsBox(
+gmi_model* makeMdsBox(
     int nx, int ny, int nz, double wx, double wy, double wz, bool is);
 
 }
diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 918c04607..d5d02f039 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -11,7 +11,7 @@
 #include <string.h>
 #include <cassert>
 #include <algorithm>
-//#include <apfBox.h>
+#include <apfBox.h>
 
 /* from https://github.com/SCOREC/core/issues/205
 0=fully interior of the volume
@@ -77,13 +77,65 @@ void setVtxClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtxC
        mesh->setModelEntity(v,getMdlRgn(model));
        //cint++;
     } else if (c >= FACE && c <= FACE_LAST) {
-       mesh->setModelEntity(v,getMdlFace(mesh,c));
+       if (c == 1) { //face tag 1 corresponds to model face 0
+          mesh->setModelEntity(v,getMdlFace(mesh,0));
+       } else if (c == 2) { //face tag 2 corresponds to model face 1
+          mesh->setModelEntity(v,getMdlFace(mesh,1));
+       } else if (c == 3) { //face tag 3 corresponds to model face 3
+          mesh->setModelEntity(v,getMdlFace(mesh,3));
+       } else if (c == 4) { //face tag 4 corresponds to model face 4
+          mesh->setModelEntity(v,getMdlFace(mesh,4));
+       } else if (c == 5) { //face tag 5 corresponds to model face 2
+          mesh->setModelEntity(v,getMdlFace(mesh,2));
+       } else if (c == 6) { //face tag 6 corresponds to model face 5
+          mesh->setModelEntity(v,getMdlFace(mesh,5));
+       }
        //cface++;
     } else if (c >= EDGE && c <= EDGE_LAST) {
-       mesh->setModelEntity(v,getMdlEdge(mesh,c));
+       if (c == 11) { //edge tag 11 corresponds to model edge 0
+          mesh->setModelEntity(v,getMdlEdge(mesh,0));
+       } else if (c == 12) {
+          mesh->setModelEntity(v,getMdlEdge(mesh,2));
+       } else if (c == 13) {
+          mesh->setModelEntity(v,getMdlEdge(mesh,3));
+       } else if (c == 14) {
+          mesh->setModelEntity(v,getMdlEdge(mesh,1));
+       } else if (c == 15) {
+          mesh->setModelEntity(v,getMdlEdge(mesh,4));
+       } else if (c == 16) {
+          mesh->setModelEntity(v,getMdlEdge(mesh,5));
+       } else if (c == 17) {
+          mesh->setModelEntity(v,getMdlEdge(mesh,7));
+       } else if (c == 18) {
+          mesh->setModelEntity(v,getMdlEdge(mesh,6));
+       } else if (c == 19) {
+          mesh->setModelEntity(v,getMdlEdge(mesh,8));
+       } else if (c == 20) {
+          mesh->setModelEntity(v,getMdlEdge(mesh,10));
+       } else if (c == 21) {
+          mesh->setModelEntity(v,getMdlEdge(mesh,11));
+       } else if (c == 22) {
+          mesh->setModelEntity(v,getMdlEdge(mesh,9));
+       }
        //cedge++;
     } else if (c >= VERTEX && c <= VERTEX_LAST) {
-       mesh->setModelEntity(v,getMdlVtx(mesh,c));
+       if (c == 31) { //vertex tag 31 corresponds to model vertex 0
+          mesh->setModelEntity(v,getMdlVtx(mesh,0));
+       } else if (c == 32) { //vertex tag 32 corresponds to model vertex 1
+          mesh->setModelEntity(v,getMdlVtx(mesh,1));
+       } else if (c == 33) { //vertex tag 33 corresponds to model vertex 3
+          mesh->setModelEntity(v,getMdlVtx(mesh,3));
+       } else if (c == 34) { //vertex tag 34 corresponds to model vertex 2
+          mesh->setModelEntity(v,getMdlVtx(mesh,2));
+       } else if (c == 35) { //vertex tag 35 corresponds to model vertex 4
+          mesh->setModelEntity(v,getMdlVtx(mesh,4));
+       } else if (c == 36) { //vertex tag 36 corresponds to model vertex 5
+          mesh->setModelEntity(v,getMdlVtx(mesh,5));
+       } else if (c == 37) { //vertex tag 37 corresponds to model vertex 7
+          mesh->setModelEntity(v,getMdlVtx(mesh,7));
+       } else if (c == 38) { //vertex tag 38 corresponds to model vertex 6
+          mesh->setModelEntity(v,getMdlVtx(mesh,6));
+       }
        //cvtx++;
     }
   }
@@ -105,8 +157,7 @@ void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh,apf::MeshTag* vtxC
     //std::cout<<"Edge number "<<count++<<" with nodes"<<std::endl;
     mesh->getAdjacent(e, 0, verts);
     int cmin=100;
-// won't compile now??    for(int i=0; i<verts.size(); i++) {
-    for(int i=0; i<2; i++) {
+    for(size_t i=0; i<verts.size(); i++) {
       mesh->getIntTag(verts[i],vtxClass,&c);
       //mesh->getPoint(verts[i], 0, vCoord);
       //std::cout<<vCoord[0]<<" "<<vCoord[1]<<" "<<vCoord[2]<<std::endl;
@@ -116,12 +167,68 @@ void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh,apf::MeshTag* vtxC
     //std::cout<<" "<<std::endl;
     if (cmin == INTERIORTAG) {
        mesh->setModelEntity(e,getMdlRgn(model));
+       //cint++;
     } else if (cmin >= FACE && cmin <= FACE_LAST) {
-       mesh->setModelEntity(e,getMdlFace(mesh,cmin));
+       if (cmin == 1) { //face tag 1 corresponds to model face 0
+          mesh->setModelEntity(e,getMdlFace(mesh,0));
+       } else if (cmin == 2) { //face tag 2 corresponds to model face 1
+          mesh->setModelEntity(e,getMdlFace(mesh,1));
+       } else if (cmin == 3) { //face tag 3 corresponds to model face 3
+          mesh->setModelEntity(e,getMdlFace(mesh,3));
+       } else if (cmin == 4) { //face tag 4 corresponds to model face 4
+          mesh->setModelEntity(e,getMdlFace(mesh,4));
+       } else if (cmin == 5) { //face tag 5 corresponds to model face 2
+          mesh->setModelEntity(e,getMdlFace(mesh,2));
+       } else if (cmin == 6) { //face tag 6 corresponds to model face 5
+          mesh->setModelEntity(e,getMdlFace(mesh,5));
+       }
+       //cface++;
     } else if (cmin >= EDGE && cmin <= EDGE_LAST) {
-       mesh->setModelEntity(e,getMdlEdge(mesh,cmin));
+       if (cmin == 11) { //edge tag 11 corresponds to model edge 0
+          mesh->setModelEntity(e,getMdlEdge(mesh,0));
+       } else if (cmin == 12) {
+          mesh->setModelEntity(e,getMdlEdge(mesh,2));
+       } else if (cmin == 13) {
+          mesh->setModelEntity(e,getMdlEdge(mesh,3));
+       } else if (cmin == 14) {
+          mesh->setModelEntity(e,getMdlEdge(mesh,1));
+       } else if (cmin == 15) {
+          mesh->setModelEntity(e,getMdlEdge(mesh,4));
+       } else if (cmin == 16) {
+          mesh->setModelEntity(e,getMdlEdge(mesh,5));
+       } else if (cmin == 17) {
+          mesh->setModelEntity(e,getMdlEdge(mesh,7));
+       } else if (cmin == 18) {
+          mesh->setModelEntity(e,getMdlEdge(mesh,6));
+       } else if (cmin == 19) {
+          mesh->setModelEntity(e,getMdlEdge(mesh,8));
+       } else if (cmin == 20) {
+          mesh->setModelEntity(e,getMdlEdge(mesh,10));
+       } else if (cmin == 21) {
+          mesh->setModelEntity(e,getMdlEdge(mesh,11));
+       } else if (cmin == 22) {
+          mesh->setModelEntity(e,getMdlEdge(mesh,9));
+       }
+       //cedge++;
     } else if (cmin >= VERTEX && cmin <= VERTEX_LAST) {
-       mesh->setModelEntity(e,getMdlVtx(mesh,cmin));
+       if (cmin == 31) { //vertex tag 31 corresponds to model vertex 0
+          mesh->setModelEntity(e,getMdlVtx(mesh,0));
+       } else if (cmin == 32) { //vertex tag 32 corresponds to model vertex 1
+          mesh->setModelEntity(e,getMdlVtx(mesh,1));
+       } else if (cmin == 33) { //vertex tag 33 corresponds to model vertex 3
+          mesh->setModelEntity(e,getMdlVtx(mesh,3));
+       } else if (cmin == 34) { //vertex tag 34 corresponds to model vertex 2
+          mesh->setModelEntity(e,getMdlVtx(mesh,2));
+       } else if (cmin == 35) { //vertex tag 35 corresponds to model vertex 4
+          mesh->setModelEntity(e,getMdlVtx(mesh,4));
+       } else if (cmin == 36) { //vertex tag 36 corresponds to model vertex 5
+          mesh->setModelEntity(e,getMdlVtx(mesh,5));
+       } else if (cmin == 37) { //vertex tag 37 corresponds to model vertex 7
+          mesh->setModelEntity(e,getMdlVtx(mesh,7));
+       } else if (cmin == 38) { //vertex tag 38 corresponds to model vertex 6
+          mesh->setModelEntity(e,getMdlVtx(mesh,6));
+       }
+       //cvtx++;
     }
   }
   mesh->end(it);
@@ -157,19 +264,74 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
   while( (f = mesh->iterate(it)) ) {
     mesh->getAdjacent(f, 0, verts);
     int cmin=100;
-// won't comile now    for(int i=0; i<verts.size(); i++) {
-    for(int i=0; i<4; i++) {
+    for(size_t i=0; i<verts.size(); i++) {
       mesh->getIntTag(verts[i],vtxClass,&c);
       cmin=std::min(cmin,c);
     }
     if (cmin == INTERIORTAG) {
        mesh->setModelEntity(f,getMdlRgn(model));
+       //cint++;
     } else if (cmin >= FACE && cmin <= FACE_LAST) {
-       mesh->setModelEntity(f,getMdlFace(mesh,cmin));
+       if (cmin == 1) { //face tag 1 corresponds to model face 0
+          mesh->setModelEntity(f,getMdlFace(mesh,0));
+       } else if (cmin == 2) { //face tag 2 corresponds to model face 1
+          mesh->setModelEntity(f,getMdlFace(mesh,1));
+       } else if (cmin == 3) { //face tag 3 corresponds to model face 3
+          mesh->setModelEntity(f,getMdlFace(mesh,3));
+       } else if (cmin == 4) { //face tag 4 corresponds to model face 4
+          mesh->setModelEntity(f,getMdlFace(mesh,4));
+       } else if (cmin == 5) { //face tag 5 corresponds to model face 2
+          mesh->setModelEntity(f,getMdlFace(mesh,2));
+       } else if (cmin == 6) { //face tag 6 corresponds to model face 5
+          mesh->setModelEntity(f,getMdlFace(mesh,5));
+       }
+       //cface++;
     } else if (cmin >= EDGE && cmin <= EDGE_LAST) {
-       mesh->setModelEntity(f,getMdlEdge(mesh,cmin));
+       if (cmin == 11) { //edge tag 11 corresponds to model edge 0
+          mesh->setModelEntity(f,getMdlEdge(mesh,0));
+       } else if (cmin == 12) {
+          mesh->setModelEntity(f,getMdlEdge(mesh,2));
+       } else if (cmin == 13) {
+          mesh->setModelEntity(f,getMdlEdge(mesh,3));
+       } else if (cmin == 14) {
+          mesh->setModelEntity(f,getMdlEdge(mesh,1));
+       } else if (cmin == 15) {
+          mesh->setModelEntity(f,getMdlEdge(mesh,4));
+       } else if (cmin == 16) {
+          mesh->setModelEntity(f,getMdlEdge(mesh,5));
+       } else if (cmin == 17) {
+          mesh->setModelEntity(f,getMdlEdge(mesh,7));
+       } else if (cmin == 18) {
+          mesh->setModelEntity(f,getMdlEdge(mesh,6));
+       } else if (cmin == 19) {
+          mesh->setModelEntity(f,getMdlEdge(mesh,8));
+       } else if (cmin == 20) {
+          mesh->setModelEntity(f,getMdlEdge(mesh,10));
+       } else if (cmin == 21) {
+          mesh->setModelEntity(f,getMdlEdge(mesh,11));
+       } else if (cmin == 22) {
+          mesh->setModelEntity(f,getMdlEdge(mesh,9));
+       }
+       //cedge++;
     } else if (cmin >= VERTEX && cmin <= VERTEX_LAST) {
-       mesh->setModelEntity(f,getMdlVtx(mesh,cmin));
+       if (cmin == 31) { //vertex tag 31 corresponds to model vertex 0
+          mesh->setModelEntity(f,getMdlVtx(mesh,0));
+       } else if (cmin == 32) { //vertex tag 32 corresponds to model vertex 1
+          mesh->setModelEntity(f,getMdlVtx(mesh,1));
+       } else if (cmin == 33) { //vertex tag 33 corresponds to model vertex 3
+          mesh->setModelEntity(f,getMdlVtx(mesh,3));
+       } else if (cmin == 34) { //vertex tag 34 corresponds to model vertex 2
+          mesh->setModelEntity(f,getMdlVtx(mesh,2));
+       } else if (cmin == 35) { //vertex tag 35 corresponds to model vertex 4
+          mesh->setModelEntity(f,getMdlVtx(mesh,4));
+       } else if (cmin == 36) { //vertex tag 36 corresponds to model vertex 5
+          mesh->setModelEntity(f,getMdlVtx(mesh,5));
+       } else if (cmin == 37) { //vertex tag 37 corresponds to model vertex 7
+          mesh->setModelEntity(f,getMdlVtx(mesh,7));
+       } else if (cmin == 38) { //vertex tag 38 corresponds to model vertex 6
+          mesh->setModelEntity(f,getMdlVtx(mesh,6));
+       }
+       //cvtx++;
     }
   }
   mesh->end(it);
@@ -385,12 +547,10 @@ void readMesh(const char* meshfilename,
   readCoords(fc, mesh.numVerts, mesh.localNumVerts, &(mesh.coords));
   fclose(fc);
 
-  if( strcmp(matchfilename, "NULL") ) {
-    FILE* ff = fopen(classfilename, "r");
-    PCU_ALWAYS_ASSERT(ff);
-    readClassification(ff, mesh.numVerts, &(mesh.classification));
-    fclose(ff);
-  }
+  FILE* ff = fopen(classfilename, "r");
+  PCU_ALWAYS_ASSERT(ff);
+  readClassification(ff, mesh.numVerts, &(mesh.classification));
+  fclose(ff);
 
   if( strcmp(matchfilename, "NULL") ) {
     FILE* fm = fopen(matchfilename, "r");
@@ -419,7 +579,7 @@ int main(int argc, char** argv)
           "<ascii vertex coordinates .crd> "
           "<ascii vertex matching flag .match> "
           "<ascii vertex classification flag .class> "
-          "<input model .dmg> <output mesh .smb>\n",
+          "<output model .dmg> <output mesh .smb>\n",
           argv[0]);
     }
     return 0;
@@ -440,21 +600,14 @@ int main(int argc, char** argv)
   if(!PCU_Comm_Self())
     fprintf(stderr, "isMatched %d\n", isMatched);
 
-  gmi_model* model = gmi_load("modbox.dmg");
-/*  int nx=3;
-  int ny=3;
-  int nz=3;
-  mgrid(nx ? 3 : 1, ny ? 3 : 1, nz ? 3 : 1);  
-  formModelTable();
-  gmi_model* model = buildModel(); 
-*/
-
+  //gmi_model* model = gmi_load(".null");
+  gmi_model* model = apf::makeMdsBox(2,2,2,1,1,1,0);
   apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, m.dim, isMatched);
   apf::GlobalToVert outMap;
   apf::construct(mesh, m.elements, m.localNumElms, m.elementType, outMap);
   delete [] m.elements;
   apf::alignMdsRemotes(mesh);
-//  apf::deriveMdsModel(mesh);
+  apf::deriveMdsModel(mesh);
   /*for (int i=0; i<81; i++) {
   std::cout<<m.coords[i]<<std::endl;
   }*/

From 55170126eeb7224e9f9ba78ae7ffbe98bb579d1a Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Thu, 14 Feb 2019 13:03:34 -0700
Subject: [PATCH 034/555] first cut at reading fathers2D array and attaching to
 the mesh

---
 test/matchedNodeElmReader.cc | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index d5d02f039..231e2ef9e 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -525,6 +525,7 @@ struct MeshInfo {
   int* elements;
   int* matches;
   int* classification;
+  int* fathers2D;
   unsigned dim;
   unsigned elementType;
   unsigned numVerts;
@@ -538,6 +539,7 @@ void readMesh(const char* meshfilename,
     const char* coordfilename,
     const char* matchfilename,
     const char* classfilename,
+    const char* fathers2Dfilename,
     MeshInfo& mesh) {
   FILE* fc = fopen(coordfilename, "r");
   PCU_ALWAYS_ASSERT(fc);
@@ -552,6 +554,12 @@ void readMesh(const char* meshfilename,
   readClassification(ff, mesh.numVerts, &(mesh.classification));
   fclose(ff);
 
+  //add an argument to readMesh for the fathers2D
+  FILE* fff = fopen(fathers2Dfilename, "r");
+  PCU_ALWAYS_ASSERT(fff);
+  readClassification(fff, mesh.numVerts, &(mesh.fathers2D)); // note we re-use classification reader
+  fclose(fff);
+
   if( strcmp(matchfilename, "NULL") ) {
     FILE* fm = fopen(matchfilename, "r");
     PCU_ALWAYS_ASSERT(fm);
@@ -573,12 +581,13 @@ int main(int argc, char** argv)
   MPI_Init(&argc,&argv);
   PCU_Comm_Init();
   lion_set_verbosity(1);
-  if( argc != 7 ) {
+  if( argc != 8 ) {
     if( !PCU_Comm_Self() ) {
       printf("Usage: %s <ascii mesh connectivity .cnn> "
           "<ascii vertex coordinates .crd> "
           "<ascii vertex matching flag .match> "
           "<ascii vertex classification flag .class> "
+          "<ascii vertex fathers2D flag .fathers2D> "
           "<output model .dmg> <output mesh .smb>\n",
           argv[0]);
     }
@@ -591,7 +600,7 @@ int main(int argc, char** argv)
 
   double t0 = PCU_Time();
   MeshInfo m;
-  readMesh(argv[1],argv[2],argv[3],argv[4],m);
+  readMesh(argv[1],argv[2],argv[3],argv[4],arg[5],m);
 
   bool isMatched = true;
   if( !strcmp(argv[3], "NULL") )
@@ -620,6 +629,8 @@ int main(int argc, char** argv)
   }
   apf::MeshTag* t = setIntTag(mesh, m.classification, 1,
       m.localNumVerts, outMap);
+  apf::MeshTag* tfathers2D = setIntTag(mesh, m.fathers2D, 1,
+      m.localNumVerts, outMap);
   outMap.clear();
   setClassification(model,mesh,t);
   apf::removeTagFromDimension(mesh, t, 0);
@@ -628,10 +639,14 @@ int main(int argc, char** argv)
     fprintf(stderr, "seconds to create mesh %.3f\n", PCU_Time()-t0);
   mesh->verify();
 
-  gmi_write_dmg(model, argv[5]);
-  mesh->writeNative(argv[6]);
+  gmi_write_dmg(model, argv[6]);
+  mesh->writeNative(argv[7]);
   apf::writeVtkFiles("rendered",mesh);
 
+  // clean up the tag for the fathers2D array
+  apf::removeTagFromDimension(mesh, tfathers2D, 0);
+  mesh->destroyTag(tfathers2D);
+
   mesh->destroyNative();
   apf::destroyMesh(mesh);
   PCU_Comm_Free();

From d2a1ebfdae96716627a59c29cb11341cfc1802df Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Thu, 14 Feb 2019 13:34:31 -0700
Subject: [PATCH 035/555] arggggg  needs a v

---
 test/matchedNodeElmReader.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 231e2ef9e..20dae056b 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -600,7 +600,7 @@ int main(int argc, char** argv)
 
   double t0 = PCU_Time();
   MeshInfo m;
-  readMesh(argv[1],argv[2],argv[3],argv[4],arg[5],m);
+  readMesh(argv[1],argv[2],argv[3],argv[4],argv[5],m);
 
   bool isMatched = true;
   if( !strcmp(argv[3], "NULL") )

From b026d68feeb007cc365bc57a84b3bddda1847335 Mon Sep 17 00:00:00 2001
From: rickybalin <riccardo.balin@colorado.edu>
Date: Tue, 19 Feb 2019 09:33:55 -0700
Subject: [PATCH 036/555] Moved around call to setIntTag for fathers2D field

---
 test/matchedNodeElmReader.cc | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 20dae056b..29d0625ab 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -627,14 +627,18 @@ int main(int argc, char** argv)
     mesh->acceptChanges();
     delete [] m.matches;
   }
-  apf::MeshTag* t = setIntTag(mesh, m.classification, 1,
+  //apf::MeshTag* tc = setIntTag(mesh, m.classification, 1,
+  //    m.localNumVerts, outMap);
+  //outMap.clear();
+  //setClassification(model,mesh,tc);
+  //apf::removeTagFromDimension(mesh, tc, 0);
+  //mesh->destroyTag(tc);
+ 
+  apf::MeshTag* tf = setIntTag(mesh, m.fathers2D, 1,
       m.localNumVerts, outMap);
-  apf::MeshTag* tfathers2D = setIntTag(mesh, m.fathers2D, 1,
-      m.localNumVerts, outMap);
-  outMap.clear();
-  setClassification(model,mesh,t);
-  apf::removeTagFromDimension(mesh, t, 0);
-  mesh->destroyTag(t);
+  apf::removeTagFromDimension(mesh, tf, 0);
+  mesh->destroyTag(tf);
+
   if(!PCU_Comm_Self())
     fprintf(stderr, "seconds to create mesh %.3f\n", PCU_Time()-t0);
   mesh->verify();
@@ -643,10 +647,6 @@ int main(int argc, char** argv)
   mesh->writeNative(argv[7]);
   apf::writeVtkFiles("rendered",mesh);
 
-  // clean up the tag for the fathers2D array
-  apf::removeTagFromDimension(mesh, tfathers2D, 0);
-  mesh->destroyTag(tfathers2D);
-
   mesh->destroyNative();
   apf::destroyMesh(mesh);
   PCU_Comm_Free();

From 04f1519585e8d86dd9cb002ce4f51f7399566c0a Mon Sep 17 00:00:00 2001
From: rickybalin <riccardo.balin@colorado.edu>
Date: Tue, 19 Feb 2019 09:44:07 -0700
Subject: [PATCH 037/555] Added Cameron's fix - tag creation needs a string

---
 apf/apfConstruct.cc          |  4 ++--
 apf/apfConvert.h             |  3 ++-
 test/matchedNodeElmReader.cc | 14 +++++++-------
 3 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index daee252b2..228295c5a 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -238,7 +238,7 @@ void setCoords(Mesh2* m, const double* coords, int nverts,
   delete [] c;
 }
 
-apf::MeshTag* setIntTag(Mesh2* m, const int* vals, const int entries,
+apf::MeshTag* setIntTag(Mesh2* m, const char* name, const int* vals, const int entries,
     int nverts, GlobalToVert& globalToVert)
 {
   Gid max = getMax(globalToVert);
@@ -308,7 +308,7 @@ apf::MeshTag* setIntTag(Mesh2* m, const int* vals, const int entries,
     }
   }
   PCU_Comm_Send();
-  apf::MeshTag* t = m->createIntTag("userInts",entries);
+  apf::MeshTag* t = m->createIntTag(name,entries);
   int* v = new int[entries];
   while (PCU_Comm_Receive()) {
     int gid;
diff --git a/apf/apfConvert.h b/apf/apfConvert.h
index 2bb27282b..542693c7c 100644
--- a/apf/apfConvert.h
+++ b/apf/apfConvert.h
@@ -62,6 +62,7 @@ void setCoords(Mesh2* m, const double* coords, int nverts,
 
 /** \brief Assign coordinates to the mesh
   * \param vals (in) array of nverts ints
+  * \param name (in) name to assign the created tag
   * \param entries (in) number of ints per vertex
   * \param nverts (in) number of vertices for this process
   * \param globalToVert (in) map from global mesh vertex ids
@@ -69,7 +70,7 @@ void setCoords(Mesh2* m, const double* coords, int nverts,
   * \details
   * See 'setCoords' for distribution details
   */
-MeshTag* setIntTag(Mesh2* m, const int* vals, const int entries,
+MeshTag* setIntTag(Mesh2* m, const char* name, const int* vals, const int entries,
     int nverts, GlobalToVert& globalToVert);
 
 /** \brief Assign matching to the mesh
diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 29d0625ab..03737ce47 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -627,14 +627,14 @@ int main(int argc, char** argv)
     mesh->acceptChanges();
     delete [] m.matches;
   }
-  //apf::MeshTag* tc = setIntTag(mesh, m.classification, 1,
-  //    m.localNumVerts, outMap);
-  //outMap.clear();
-  //setClassification(model,mesh,tc);
-  //apf::removeTagFromDimension(mesh, tc, 0);
-  //mesh->destroyTag(tc);
+  apf::MeshTag* tc = setIntTag(mesh, "classification", m.classification, 1,
+      m.localNumVerts, outMap);
+  outMap.clear();
+  setClassification(model,mesh,tc);
+  apf::removeTagFromDimension(mesh, tc, 0);
+  mesh->destroyTag(tc);
  
-  apf::MeshTag* tf = setIntTag(mesh, m.fathers2D, 1,
+  apf::MeshTag* tf = setIntTag(mesh, "fathers2D", m.fathers2D, 1,
       m.localNumVerts, outMap);
   apf::removeTagFromDimension(mesh, tf, 0);
   mesh->destroyTag(tf);

From b462a2452ca9017c819eaa05979449efdc9401fb Mon Sep 17 00:00:00 2001
From: rickybalin <riccardo.balin@colorado.edu>
Date: Tue, 19 Feb 2019 11:24:21 -0700
Subject: [PATCH 038/555] Moved one line and now runs without crashing adding
 both classification and fathers2D tags

---
 test/matchedNodeElmReader.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 03737ce47..09cc519fa 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -629,7 +629,6 @@ int main(int argc, char** argv)
   }
   apf::MeshTag* tc = setIntTag(mesh, "classification", m.classification, 1,
       m.localNumVerts, outMap);
-  outMap.clear();
   setClassification(model,mesh,tc);
   apf::removeTagFromDimension(mesh, tc, 0);
   mesh->destroyTag(tc);
@@ -643,6 +642,7 @@ int main(int argc, char** argv)
     fprintf(stderr, "seconds to create mesh %.3f\n", PCU_Time()-t0);
   mesh->verify();
 
+  outMap.clear();
   gmi_write_dmg(model, argv[6]);
   mesh->writeNative(argv[7]);
   apf::writeVtkFiles("rendered",mesh);

From 84198a5c1a70f4c3d831e181ae03e92096d8bdaa Mon Sep 17 00:00:00 2001
From: rickybalin <riccardo.balin@colorado.edu>
Date: Wed, 20 Feb 2019 15:57:51 -0700
Subject: [PATCH 039/555] Added spanwise averaging array creation to
 phOutput.cc but can't seem to find desired tag

---
 phasta/phOutput.cc | 38 +++++++++++++++++++++++++++++++++++++-
 phasta/phOutput.h  | 10 ++++++++++
 2 files changed, 47 insertions(+), 1 deletion(-)

diff --git a/phasta/phOutput.cc b/phasta/phOutput.cc
index e27e95bc2..13eff04b7 100644
--- a/phasta/phOutput.cc
+++ b/phasta/phOutput.cc
@@ -220,7 +220,7 @@ static void checkBoundaryVertex(apf::Mesh* m,
   if (type == TETRAHEDRON || type == WEDGE_QUAD || type == PYRAMID_TRI) // outward
     PCU_ALWAYS_ASSERT((p[3]-p[0]) * apf::cross((p[1]-p[0]), (p[2]-p[0])) < 0);
   else // inward
-    PCU_ALWAYS_ASSERT((p[3]-p[0]) * apf::cross((p[1]-p[0]), (p[2]-p[0])) > 0);
+    PCU_ALWAYS_ASSERT((p[3]-p[0]) * apf::cross((p[1]-p[0]), (p[2]-p[0])) < 0);
 }
 
 static void getBoundary(Output& o, BCs& bcs, apf::Numbering* n)
@@ -941,6 +941,41 @@ static void getEdges(Output& o, apf::Numbering* vn, apf::Numbering* rn, BCs& bcs
   }
 }
 
+static void getSpanwiseAverageArrays(Input& in, Output& o) {
+  apf::Mesh* m = o.mesh;
+  gmi_model* gm = m->getModel();
+  int nnodes = m->count(0); // number of nodes of wole mesh or part??
+  /* this will come from the adapt.inp file and is constant for all geombc
+     it is the total number of father nodes, nx*ny, and each geombc loads this */
+  int nfather = 9;
+  o.arrays.nfather = nfather;
+  /* this will come from the adapt.inp file and is constant for all geombc
+     this is the number of son nodes for each father, nz, and each geombc loads this
+     loading from adapt.inp only works when all fathers have same nz */
+  int nsons = 3;
+  o.arrays.nsons = nsons;
+  o.arrays.nsonsArr = new int[nfather]; //initialize nsonsArr
+  for (int i=0; i<nfather; i++) { // fill nsonsArr
+    /* set each entry in nsonsArr[nfather] to equal nsons */
+    o.arrays.nsonsArr[i] = nsons; // this is point2nsons(nfath) in PHASTA
+  }
+  apf::MeshEntity* v;
+  apf::MeshIterator* it = m->begin(0);
+  o.arrays.ifather = new int[nnodes]; //initialize ifath
+  apf::MeshTag* t = m->findTag("fathers2D");
+  if (t==NULL) {std::cout<<"Didn't find tag"<<std::endl;}
+  int tagNum; 
+  int count = 0;
+  while ((v = m->iterate(it))) { // loop over mesh vertices
+    m->getIntTag(v,t,&tagNum);
+    o.arrays.ifather[count] = tagNum;
+    count++;
+  }
+  m->end(it);
+  m->destroyTag(t);
+  PCU_ALWAYS_ASSERT(count == nnodes);
+}
+
 Output::~Output()
 {
   delete [] arrays.coordinates;
@@ -1048,6 +1083,7 @@ void generateOutput(Input& in, BCs& bcs, apf::Mesh* mesh, Output& o)
   getGCEssentialBCs(o, n);
   getInitialConditions(bcs, o);
   getElementGraph(o, rn, bcs);
+  getSpanwiseAverageArrays(in, o);
   apf::destroyNumbering(n);
   apf::destroyNumbering(rn);
   if (in.initBubbles)
diff --git a/phasta/phOutput.h b/phasta/phOutput.h
index 07ee7c265..ef8c82458 100644
--- a/phasta/phOutput.h
+++ b/phasta/phOutput.h
@@ -126,6 +126,16 @@ idx:   0  1 2  3   4   5  6   7   8   9  10   11  12  13  14  15   16   17   18
   int* rigidBodyMTs;
 /* an integer to indicate rigid body tag of a vertex */
   int* rigidBodyTag;
+/* an integer to indicate how many father nodes there are in the mesh*/
+  int nfather;
+/* an integer to indicate how many sons there are for each father */
+/* NOTE: currently this assumes each father has the same number of sons */
+/* uniform z spacing and constant domain thickness */
+  int nsons;
+/* an array of integers that holds the number of the father node */
+  int* ifather;
+/* an array of integers of size nfather that has nsons in each entry */
+  int* nsonsArr;
 };
 
 

From b12d20943cf82e349ce9abb24beca20b225628b4 Mon Sep 17 00:00:00 2001
From: rickybalin <riccardo.balin@colorado.edu>
Date: Thu, 21 Feb 2019 09:32:34 -0700
Subject: [PATCH 040/555] Changed matchedNodeElmReader to not delete fathers2D
 tag after adding it, and generateOutput with getSpanwiseAverageArrays now can
 read the tag

---
 phasta/phOutput.cc           |  7 ++++++-
 test/matchedNodeElmReader.cc | 17 +++++++++++++++--
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/phasta/phOutput.cc b/phasta/phOutput.cc
index 13eff04b7..c152e5c4b 100644
--- a/phasta/phOutput.cc
+++ b/phasta/phOutput.cc
@@ -963,12 +963,17 @@ static void getSpanwiseAverageArrays(Input& in, Output& o) {
   apf::MeshIterator* it = m->begin(0);
   o.arrays.ifather = new int[nnodes]; //initialize ifath
   apf::MeshTag* t = m->findTag("fathers2D");
-  if (t==NULL) {std::cout<<"Didn't find tag"<<std::endl;}
+  if (t==NULL) {
+      std::cout<<"Did not find tag fathers2D"<<std::endl;
+  } else if (t != NULL) {
+      std::cout<<"Found tag fathers2D"<<std::endl;
+  }
   int tagNum; 
   int count = 0;
   while ((v = m->iterate(it))) { // loop over mesh vertices
     m->getIntTag(v,t,&tagNum);
     o.arrays.ifather[count] = tagNum;
+    std::cout<<"Tag number "<<tagNum<<std::endl;
     count++;
   }
   m->end(it);
diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 09cc519fa..442a4d2c9 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -635,8 +635,21 @@ int main(int argc, char** argv)
  
   apf::MeshTag* tf = setIntTag(mesh, "fathers2D", m.fathers2D, 1,
       m.localNumVerts, outMap);
-  apf::removeTagFromDimension(mesh, tf, 0);
-  mesh->destroyTag(tf);
+  //mesh->destroyTag(tf);
+
+  /* // Print the father2D tags
+  apf::MeshEntity* v;
+  apf::MeshIterator* it = mesh->begin(0);
+  apf::MeshTag* t = mesh->findTag("fathers2D");
+  if (t==NULL) {std::cout<<"Didn't find tag"<<std::endl;}
+  int tagNum; 
+  int count = 0;
+  while ((v = mesh->iterate(it))) { // loop over mesh vertices
+    mesh->getIntTag(v,t,&tagNum);
+    std::cout<<"Tag number "<<tagNum<<std::endl;
+    count++;
+  }
+  mesh->end(it);*/
 
   if(!PCU_Comm_Self())
     fprintf(stderr, "seconds to create mesh %.3f\n", PCU_Time()-t0);

From 462c69ed062b5e73d4bdab255f97b5ac2f1f36a1 Mon Sep 17 00:00:00 2001
From: rickybalin <riccardo.balin@colorado.edu>
Date: Thu, 21 Feb 2019 13:48:54 -0700
Subject: [PATCH 041/555] Chef reads the fathers2D tag, fills desired array,
 and prints them in geombc, and inputs from adapt.inp read properly. Verified
 that PHASTA reads the arrays well, and that the arrays are printed also the
 second partition.

---
 phasta/phGeomBC.cc | 10 +++++++
 phasta/phInput.cc  |  6 ++++
 phasta/phInput.h   |  4 +++
 phasta/phOutput.cc | 72 ++++++++++++++++++++++++----------------------
 4 files changed, 58 insertions(+), 34 deletions(-)

diff --git a/phasta/phGeomBC.cc b/phasta/phGeomBC.cc
index 9ab9ecbc4..3eefef4f2 100644
--- a/phasta/phGeomBC.cc
+++ b/phasta/phGeomBC.cc
@@ -283,6 +283,15 @@ static void writeGrowthCurves(Output& o, FILE* f)
   }
 }
 
+static void writeSpanwiseAvgArrays(Output& o, FILE* f)
+{
+  if (o.arrays.nfather > 0) {
+    writeInt(f, "number of father-nodes", o.arrays.nfather);
+    writeInts(f, "number of son-nodes for each father", o.arrays.nsonsArr,o.arrays.nfather);
+    writeInts(f, "keyword ifath", o.arrays.ifather,o.mesh->count(0));
+  }
+}
+
 void writeGeomBC(Output& o, std::string path, int timestep)
 {
   double t0 = PCU_Time();
@@ -363,6 +372,7 @@ void writeGeomBC(Output& o, std::string path, int timestep)
   writeElementGraph(o, f);
   writeEdges(o, f);
   writeGrowthCurves(o, f);
+  writeSpanwiseAvgArrays(o, f);
   PHASTAIO_CLOSETIME(fclose(f);)
   double t1 = PCU_Time();
   if (!PCU_Comm_Self())
diff --git a/phasta/phInput.cc b/phasta/phInput.cc
index 661965dbe..faa5607cb 100644
--- a/phasta/phInput.cc
+++ b/phasta/phInput.cc
@@ -83,6 +83,9 @@ static void setDefaults(Input& in)
   in.nRigidBody = 0;
   in.nRBParam = 12;
   in.gradingFactor = 1.25;
+  in.spanAvg = 0; // prepare and write spanwise average arrays
+  in.nfathers = 0; // number of father nodes (nx*ny for z averaging)
+  in.nsons = 0; // number of sons for each father (constant nz for each father)
 }
 
 Input::Input()
@@ -164,6 +167,9 @@ static void formMaps(Input& in, StringMap& stringMap, IntMap& intMap, DblMap& db
   intMap["nRigidBody"] = &in.nRigidBody;
   intMap["nRBParam"] = &in.nRBParam;
   dblMap["gradingFactor"] = &in.gradingFactor;
+  intMap["spanAverage"] = &in.spanAvg;
+  intMap["nfathers"] = &in.nfathers;
+  intMap["nsons"] = &in.nsons;
 }
 
 template <class T>
diff --git a/phasta/phInput.h b/phasta/phInput.h
index bccb66562..108133085 100644
--- a/phasta/phInput.h
+++ b/phasta/phInput.h
@@ -189,6 +189,10 @@ class Input
     std::vector<double> rbParamData;
     /** \brief factor \beta used for mesh smooth/gradation */
     double gradingFactor;
+    /* Stuff for spanwise averaging */
+    int spanAvg;
+    int nfathers;
+    int nsons;
 };
 
 int countNaturalBCs(Input& in);
diff --git a/phasta/phOutput.cc b/phasta/phOutput.cc
index c152e5c4b..38183142e 100644
--- a/phasta/phOutput.cc
+++ b/phasta/phOutput.cc
@@ -942,43 +942,47 @@ static void getEdges(Output& o, apf::Numbering* vn, apf::Numbering* rn, BCs& bcs
 }
 
 static void getSpanwiseAverageArrays(Input& in, Output& o) {
-  apf::Mesh* m = o.mesh;
-  gmi_model* gm = m->getModel();
-  int nnodes = m->count(0); // number of nodes of wole mesh or part??
-  /* this will come from the adapt.inp file and is constant for all geombc
+  if (in.spanAvg == 1) {
+    apf::Mesh* m = o.mesh;
+    gmi_model* gm = m->getModel();
+    int nnodes = m->count(0); // number of nodes of wole mesh or part??
+    /* this will come from the adapt.inp file and is constant for all geombc
      it is the total number of father nodes, nx*ny, and each geombc loads this */
-  int nfather = 9;
-  o.arrays.nfather = nfather;
-  /* this will come from the adapt.inp file and is constant for all geombc
+    int nfather = in.nfathers;
+    o.arrays.nfather = nfather;
+    /* this will come from the adapt.inp file and is constant for all geombc
      this is the number of son nodes for each father, nz, and each geombc loads this
      loading from adapt.inp only works when all fathers have same nz */
-  int nsons = 3;
-  o.arrays.nsons = nsons;
-  o.arrays.nsonsArr = new int[nfather]; //initialize nsonsArr
-  for (int i=0; i<nfather; i++) { // fill nsonsArr
-    /* set each entry in nsonsArr[nfather] to equal nsons */
-    o.arrays.nsonsArr[i] = nsons; // this is point2nsons(nfath) in PHASTA
-  }
-  apf::MeshEntity* v;
-  apf::MeshIterator* it = m->begin(0);
-  o.arrays.ifather = new int[nnodes]; //initialize ifath
-  apf::MeshTag* t = m->findTag("fathers2D");
-  if (t==NULL) {
-      std::cout<<"Did not find tag fathers2D"<<std::endl;
-  } else if (t != NULL) {
-      std::cout<<"Found tag fathers2D"<<std::endl;
-  }
-  int tagNum; 
-  int count = 0;
-  while ((v = m->iterate(it))) { // loop over mesh vertices
-    m->getIntTag(v,t,&tagNum);
-    o.arrays.ifather[count] = tagNum;
-    std::cout<<"Tag number "<<tagNum<<std::endl;
-    count++;
-  }
-  m->end(it);
-  m->destroyTag(t);
-  PCU_ALWAYS_ASSERT(count == nnodes);
+    int nsons = in.nsons;
+    o.arrays.nsons = nsons;
+    o.arrays.nsonsArr = new int[nfather]; //initialize nsonsArr
+    for (int i=0; i<nfather; i++) { // fill nsonsArr
+      /* set each entry in nsonsArr[nfather] to equal nsons */
+      o.arrays.nsonsArr[i] = nsons; // this is point2nsons(nfath) in PHASTA
+    }
+    apf::MeshEntity* v;
+    apf::MeshIterator* it = m->begin(0);
+    o.arrays.ifather = new int[nnodes]; //initialize ifath
+    apf::MeshTag* t = m->findTag("fathers2D");
+    if (t==NULL) {
+        std::cout<<"Did not find tag fathers2D"<<std::endl;
+    } else if (t != NULL) {
+        std::cout<<"Found tag fathers2D"<<std::endl;
+    }
+    int tagNum; 
+    int count = 0;
+    while ((v = m->iterate(it))) { // loop over mesh vertices
+      m->getIntTag(v,t,&tagNum);
+      o.arrays.ifather[count] = tagNum;
+      //std::cout<<"Tag number "<<tagNum<<std::endl;
+      count++;
+    }
+    m->end(it);
+    PCU_ALWAYS_ASSERT(count == nnodes);
+  } else {
+    o.arrays.nfather = 0;
+    o.arrays.nsons = 0;
+  } 
 }
 
 Output::~Output()

From 411b97fbe3a31baf24a32afe843bc3f7e84c2be9 Mon Sep 17 00:00:00 2001
From: rickybalin <riccardo.balin@colorado.edu>
Date: Wed, 27 Feb 2019 09:48:38 -0700
Subject: [PATCH 042/555] Reverted sign of cross product assert for hexes in
 phOutput because changed Matlab to print positive volume classification

---
 phasta/phOutput.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/phasta/phOutput.cc b/phasta/phOutput.cc
index 38183142e..0e527be8d 100644
--- a/phasta/phOutput.cc
+++ b/phasta/phOutput.cc
@@ -220,7 +220,7 @@ static void checkBoundaryVertex(apf::Mesh* m,
   if (type == TETRAHEDRON || type == WEDGE_QUAD || type == PYRAMID_TRI) // outward
     PCU_ALWAYS_ASSERT((p[3]-p[0]) * apf::cross((p[1]-p[0]), (p[2]-p[0])) < 0);
   else // inward
-    PCU_ALWAYS_ASSERT((p[3]-p[0]) * apf::cross((p[1]-p[0]), (p[2]-p[0])) < 0);
+    PCU_ALWAYS_ASSERT((p[3]-p[0]) * apf::cross((p[1]-p[0]), (p[2]-p[0])) > 0);
 }
 
 static void getBoundary(Output& o, BCs& bcs, apf::Numbering* n)

From ab190fd58d1b0871a1951e4a01a3d88bf503ea7e Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Tue, 2 Jul 2019 13:28:34 -0600
Subject: [PATCH 043/555] usual version dance

---
 cmake/FindSimModSuite.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/FindSimModSuite.cmake b/cmake/FindSimModSuite.cmake
index fffc8221b..a2d7cd050 100644
--- a/cmake/FindSimModSuite.cmake
+++ b/cmake/FindSimModSuite.cmake
@@ -84,7 +84,7 @@ string(REGEX REPLACE
   "${SIM_VERSION}")
 
 set(MIN_VALID_SIM_VERSION 12.0.190225)
-set(MAX_VALID_SIM_VERSION 14.0.190604)
+set(MAX_VALID_SIM_VERSION 14.0.190617)
 if( ${SKIP_SIMMETRIX_VERSION_CHECK} )
   message(STATUS "Skipping Simmetrix SimModSuite version check."
     " This may result in undefined behavior")

From 5b362d5d30b90b75aa03959f45d085a7b2ef84d6 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Sun, 29 Dec 2019 13:49:13 -0700
Subject: [PATCH 044/555] almost works but trying a different route using model
 adjacency next

---
 phasta/phInput.cc            |   4 +-
 phasta/phOutput.cc           |  10 ++-
 test/matchedNodeElmReader.cc | 168 +++++++++++++++++++----------------
 3 files changed, 101 insertions(+), 81 deletions(-)

diff --git a/phasta/phInput.cc b/phasta/phInput.cc
index faa5607cb..c7d23524e 100644
--- a/phasta/phInput.cc
+++ b/phasta/phInput.cc
@@ -84,8 +84,8 @@ static void setDefaults(Input& in)
   in.nRBParam = 12;
   in.gradingFactor = 1.25;
   in.spanAvg = 0; // prepare and write spanwise average arrays
-  in.nfathers = 0; // number of father nodes (nx*ny for z averaging)
-  in.nsons = 0; // number of sons for each father (constant nz for each father)
+  in.nfathers = 0; // number of father nodes (# of pts in xy (2D) plane for z averaging)
+  in.nsons = 0; // number of sons for each father (constant nz for each father ONLY for ijk grids)
 }
 
 Input::Input()
diff --git a/phasta/phOutput.cc b/phasta/phOutput.cc
index 0e527be8d..f6fd39cd8 100644
--- a/phasta/phOutput.cc
+++ b/phasta/phOutput.cc
@@ -944,8 +944,8 @@ static void getEdges(Output& o, apf::Numbering* vn, apf::Numbering* rn, BCs& bcs
 static void getSpanwiseAverageArrays(Input& in, Output& o) {
   if (in.spanAvg == 1) {
     apf::Mesh* m = o.mesh;
-    gmi_model* gm = m->getModel();
-    int nnodes = m->count(0); // number of nodes of wole mesh or part??
+// not used    gmi_model* gm = m->getModel();
+    int nnodes = m->count(0); // number of nodes of whole mesh or part??
     /* this will come from the adapt.inp file and is constant for all geombc
      it is the total number of father nodes, nx*ny, and each geombc loads this */
     int nfather = in.nfathers;
@@ -958,7 +958,8 @@ static void getSpanwiseAverageArrays(Input& in, Output& o) {
     o.arrays.nsonsArr = new int[nfather]; //initialize nsonsArr
     for (int i=0; i<nfather; i++) { // fill nsonsArr
       /* set each entry in nsonsArr[nfather] to equal nsons */
-      o.arrays.nsonsArr[i] = nsons; // this is point2nsons(nfath) in PHASTA
+      o.arrays.nsonsArr[i] = 0; // not alwasy structured nsons; // this is point2nsons(nfath) in PHASTA
+                           // -1 if father is not counted in sons .... I think it is so leaving it at 0
     }
     apf::MeshEntity* v;
     apf::MeshIterator* it = m->begin(0);
@@ -974,6 +975,9 @@ static void getSpanwiseAverageArrays(Input& in, Output& o) {
     while ((v = m->iterate(it))) { // loop over mesh vertices
       m->getIntTag(v,t,&tagNum);
       o.arrays.ifather[count] = tagNum;
+//      o.arrays.nsonsArr[tagNum] =o.arrays.nsonArr[tagNum]+1;  // increment the nsons counter 
+//      o.arrays.nsonsArr[tagNum]+=1;  // increment the nsons counter 
+      ++o.arrays.nsonsArr[tagNum];  // increment the nsons counter 
       //std::cout<<"Tag number "<<tagNum<<std::endl;
       count++;
     }
diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 442a4d2c9..5ba098b80 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -242,96 +242,111 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
   (void)model;
   (void)mesh;
   (void)vtxClass;
-
   apf::MeshIterator* it = mesh->begin(2);
   apf::MeshEntity* f;
   int c;
-/* What was here before was commented when we went to simple min rule
   apf::Adjacent verts;
+  double xd1=0.914478; double yd1=0.0145401; double zd1=0.04;
+  double xd2=-0.548533; double yd2=7.46251e-06; double zd2=0.0399287;
+  double dx1,dy1,dz1,distFromDebug1;
+  double dx2,dy2,dz2,distFromDebug2;
+  int edgesOfFaces[6][4];
+  int vertsOfEdges[12][2];
+  edgesOfFaces[0][0]=0;
   while( (f = mesh->iterate(it)) ) {
-    m->getAdjacent(f, 0, verts) = 0;
-    bool hasRgClass = false;
-    for(int i=0; i<verts.size(); i++) {
-      m->getIntTag(i,vtxClass,&c);
-      if( c == INTERIORTAG )
-        mesh->setModelEntity(f,mdlRgn);
-      }
-      
+    apf::Vector3 Centroid=apf::getLinearCentroid(mesh,f);
+    dx1=xd1-Centroid[0];
+    dy1=yd1-Centroid[1];
+    dz1=zd1-Centroid[2];
+    dx2=xd2-Centroid[0];
+    dy2=yd2-Centroid[1];
+    dz2=zd2-Centroid[2];
+    distFromDebug1=dx1*dx1+dy1*dy1+dz1*dz1;
+    distFromDebug2=dx2*dx2+dy2*dy2+dz2*dz2;
+    if(std::min(distFromDebug1,distFromDebug2) < 1e-12) {
+         apf::MeshEntity*  stophere=f;  fprintf(stderr, "%d \n", stophere);
+         (void) stophere;
     }
-  }
-*/
-  apf::Adjacent verts;
-  while( (f = mesh->iterate(it)) ) {
     mesh->getAdjacent(f, 0, verts);
     int cmin=100;
+    int cmax=-100;
+    int cmid=-100;
+    int ctri[4];  // up to 4 points on a face
     for(size_t i=0; i<verts.size(); i++) {
       mesh->getIntTag(verts[i],vtxClass,&c);
       cmin=std::min(cmin,c);
+      cmax=std::max(cmax,c);
+      ctri[i]=c;
     }
+       for(size_t i=0; i<verts.size(); i++) {
+         if((ctri[i] != cmin) && (ctri[i] != cmax)) cmid=std::max(cmid,ctri[i]);  // max is to catch lowest dim/highest code for quads....actually not necessary since either of the other two will follow switches as noted below.
+       }
     if (cmin == INTERIORTAG) {
        mesh->setModelEntity(f,getMdlRgn(model));
        //cint++;
-    } else if (cmin >= FACE && cmin <= FACE_LAST) {
-       if (cmin == 1) { //face tag 1 corresponds to model face 0
-          mesh->setModelEntity(f,getMdlFace(mesh,0));
-       } else if (cmin == 2) { //face tag 2 corresponds to model face 1
-          mesh->setModelEntity(f,getMdlFace(mesh,1));
-       } else if (cmin == 3) { //face tag 3 corresponds to model face 3
-          mesh->setModelEntity(f,getMdlFace(mesh,3));
-       } else if (cmin == 4) { //face tag 4 corresponds to model face 4
-          mesh->setModelEntity(f,getMdlFace(mesh,4));
-       } else if (cmin == 5) { //face tag 5 corresponds to model face 2
-          mesh->setModelEntity(f,getMdlFace(mesh,2));
-       } else if (cmin == 6) { //face tag 6 corresponds to model face 5
-          mesh->setModelEntity(f,getMdlFace(mesh,5));
-       }
-       //cface++;
-    } else if (cmin >= EDGE && cmin <= EDGE_LAST) {
-       if (cmin == 11) { //edge tag 11 corresponds to model edge 0
-          mesh->setModelEntity(f,getMdlEdge(mesh,0));
-       } else if (cmin == 12) {
-          mesh->setModelEntity(f,getMdlEdge(mesh,2));
-       } else if (cmin == 13) {
-          mesh->setModelEntity(f,getMdlEdge(mesh,3));
-       } else if (cmin == 14) {
-          mesh->setModelEntity(f,getMdlEdge(mesh,1));
-       } else if (cmin == 15) {
-          mesh->setModelEntity(f,getMdlEdge(mesh,4));
-       } else if (cmin == 16) {
-          mesh->setModelEntity(f,getMdlEdge(mesh,5));
-       } else if (cmin == 17) {
-          mesh->setModelEntity(f,getMdlEdge(mesh,7));
-       } else if (cmin == 18) {
-          mesh->setModelEntity(f,getMdlEdge(mesh,6));
-       } else if (cmin == 19) {
-          mesh->setModelEntity(f,getMdlEdge(mesh,8));
-       } else if (cmin == 20) {
-          mesh->setModelEntity(f,getMdlEdge(mesh,10));
-       } else if (cmin == 21) {
-          mesh->setModelEntity(f,getMdlEdge(mesh,11));
-       } else if (cmin == 22) {
-          mesh->setModelEntity(f,getMdlEdge(mesh,9));
-       }
-       //cedge++;
-    } else if (cmin >= VERTEX && cmin <= VERTEX_LAST) {
-       if (cmin == 31) { //vertex tag 31 corresponds to model vertex 0
-          mesh->setModelEntity(f,getMdlVtx(mesh,0));
-       } else if (cmin == 32) { //vertex tag 32 corresponds to model vertex 1
-          mesh->setModelEntity(f,getMdlVtx(mesh,1));
-       } else if (cmin == 33) { //vertex tag 33 corresponds to model vertex 3
-          mesh->setModelEntity(f,getMdlVtx(mesh,3));
-       } else if (cmin == 34) { //vertex tag 34 corresponds to model vertex 2
-          mesh->setModelEntity(f,getMdlVtx(mesh,2));
-       } else if (cmin == 35) { //vertex tag 35 corresponds to model vertex 4
-          mesh->setModelEntity(f,getMdlVtx(mesh,4));
-       } else if (cmin == 36) { //vertex tag 36 corresponds to model vertex 5
-          mesh->setModelEntity(f,getMdlVtx(mesh,5));
-       } else if (cmin == 37) { //vertex tag 37 corresponds to model vertex 7
-          mesh->setModelEntity(f,getMdlVtx(mesh,7));
-       } else if (cmin == 38) { //vertex tag 38 corresponds to model vertex 6
-          mesh->setModelEntity(f,getMdlVtx(mesh,6));
-       }
-       //cvtx++;
+    } else  if(cmax <= FACE_LAST && cmin >= FACE) { // all nodes on  model face(s?)
+         if(cmax != cmin) { // all on faces but not all on same so classified on interior
+             mesh->setModelEntity(f,getMdlRgn(model));
+         } else { // all on same face so classify on that one
+          mesh->setModelEntity(f,getMdlFace(mesh,(cmin-1)));
+         }
+    } else if (cmin <= FACE_LAST) {  // one or two but not more on model faces 
+         if( cmid > FACE_LAST || cmin==cmid) {   // for tris, the first happens if 2 other on e and or v;  second happens if 2 on SAME face and max NOT on face due to previous else...this also catches quads of both types that can be on a face
+             mesh->setModelEntity(f,getMdlFace(mesh,(cmin-1))); // classified on cmin face either way
+         } else { // getting here only if cmid is a different face so must be interior
+             mesh->setModelEntity(f,getMdlRgn(model));
+         }
+    } else if (cmax <= EDGE_LAST){  // since none on face and none on verts must be on 
+        mesh->setModelEntity(f,getMdlRgn(model));
+    } else { // getting here only if none of the face verts are interior or on model face so must be 2 e and 1 v but
+// none of these nodal classifications know directly the face to classify this face on.  HARD CODE for now to our case
+         int iface=0;
+         if(cmax==31) {
+            if( cmin== 11) {
+              if( cmid==14) iface=1; else iface=2;
+           }
+           else iface=5;
+         } else if(cmax==32) {
+            if( cmin== 11) {
+              if( cmid==12) iface=1; else iface=2;
+            }
+            else iface=3;
+         } else if(cmax==33) {
+            if( cmin== 12) {
+              if( cmid==13) iface=1; else iface=3;
+            }
+            else iface=4;
+         } else if(cmax==34) {
+            if( cmin== 13) {
+              if( cmid==14) iface=1; else iface=4;
+            }
+            else iface=5;
+         } else if(cmax==35) {
+            if( cmin== 15) {
+              if( cmid==19) iface=2; else iface=5;
+            }
+            else iface=6;
+         } else if(cmax==36) {
+            if( cmin== 16) {
+              if( cmid==19) iface=2; else iface=3;
+            }
+            else iface=6;
+         } else if(cmax==37) {
+            if( cmin== 17) {
+              if( cmid==20) iface=3; else iface=4;
+            }
+            else iface=6;
+         } else if(cmax==38) {
+            if( cmin== 18) {
+              if( cmid==21) iface=4; else iface=5;
+            }
+            else iface=6;
+         }
+         if(iface==0){
+            fprintf(stderr, "%d %d  %d \n", cmin, cmid, cmax);
+         }
+         PCU_ALWAYS_ASSERT(iface != 0);
+         mesh->setModelEntity(f,getMdlFace(mesh,(iface-1))); // classified on chosen face 
     }
   }
   mesh->end(it);
@@ -634,7 +649,8 @@ int main(int argc, char** argv)
   mesh->destroyTag(tc);
  
   apf::MeshTag* tf = setIntTag(mesh, "fathers2D", m.fathers2D, 1,
-      m.localNumVerts, outMap);
+       m.localNumVerts, outMap);
+  (void) tf;
   //mesh->destroyTag(tf);
 
   /* // Print the father2D tags

From 1d402787735cf05f5b8b5d139bfc7d10c5c40c6d Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Sun, 29 Dec 2019 18:43:25 -0700
Subject: [PATCH 045/555] stil not working

---
 test/matchedNodeElmReader.cc | 166 +++++++++++++++++++++++++++++------
 1 file changed, 140 insertions(+), 26 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 5ba098b80..d4d77b861 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -246,42 +246,118 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
   apf::MeshEntity* f;
   int c;
   apf::Adjacent verts;
-  double xd1=0.914478; double yd1=0.0145401; double zd1=0.04;
-  double xd2=-0.548533; double yd2=7.46251e-06; double zd2=0.0399287;
+  double xd1[3]={-0.54864, 7.44015e-06, 0.0397148 };
+  double xd2[3]={0.914478, 0.0145401, 0.04 };
+//  double xd1=0.914478; double yd1=0.0145401; double zd1=0.04;
+//  double xd2=-0.548533; double yd2=7.46251e-06; double zd2=0.0399287;
   double dx1,dy1,dz1,distFromDebug1;
   double dx2,dy2,dz2,distFromDebug2;
-  int edgesOfFaces[6][4];
-  int vertsOfEdges[12][2];
-  edgesOfFaces[0][0]=0;
+  // int EofF[6][4];
+  // int VofE[12][2];
+  int FofE[12][2];
+  FofE[0][0]=0;
+  FofE[0][1]=1;
+  FofE[1][0]=0;
+  FofE[1][1]=2;
+  FofE[2][0]=0;
+  FofE[2][1]=3;
+  FofE[3][0]=0;
+  FofE[3][1]=4;
+  FofE[4][0]=1;
+  FofE[4][1]=4;
+  FofE[5][0]=1;
+  FofE[5][1]=2;
+  FofE[6][0]=2;
+  FofE[6][1]=3;
+  FofE[7][0]=3;
+  FofE[7][1]=4;
+  FofE[8][0]=1;
+  FofE[8][1]=5;
+  FofE[9][0]=2;
+  FofE[9][1]=5;
+  FofE[10][0]=3;
+  FofE[10][1]=5;
+  FofE[11][0]=4;
+  FofE[11][1]=5;
+/*
+  VofE[0][0]=0;
+  VofE[0][1]=1;
+  VofE[1][0]=1;
+  VofE[1][1]=2;
+  VofE[2][0]=2;
+  VofE[2][1]=3;
+  VofE[3][0]=3;
+  VofE[3][1]=0;
+  VofE[4][0]=0;
+  VofE[4][1]=4;
+  VofE[5][0]=1;
+  VofE[5][1]=5;
+  VofE[6][0]=2;
+  VofE[6][1]=6;
+  VofE[7][0]=3;
+  VofE[7][1]=7;
+  VofE[8][0]=4;
+  VofE[8][1]=5;
+  VofE[9][0]=5;
+  VofE[9][1]=6;
+  VofE[10][0]=6;
+  VofE[10][1]=7;
+  VofE[11][0]=7;
+  VofE[11][1]=4;
+  EofF[0][0]=0;
+  EofF[0][1]=1;
+  EofF[0][2]=2;
+  EofF[0][3]=3;
+  EofF[1][0]=4;
+  EofF[1][1]=8;
+  EofF[1][2]=5;
+  EofF[1][3]=0;
+  EofF[2][0]=1;
+  EofF[2][1]=6;
+  EofF[2][2]=9;
+  EofF[2][3]=5;
+  EofF[3][0]=2;
+  EofF[3][1]=6;
+  EofF[3][2]=10;
+  EofF[3][3]=7;
+  EofF[4][0]=3;
+  EofF[4][1]=7;
+  EofF[4][2]=11;
+  EofF[4][3]=4;
+  EofF[5][0]=8;
+  EofF[5][1]=11;
+  EofF[5][2]=10;
+  EofF[5][3]=9;
+*/
   while( (f = mesh->iterate(it)) ) {
     apf::Vector3 Centroid=apf::getLinearCentroid(mesh,f);
-    dx1=xd1-Centroid[0];
-    dy1=yd1-Centroid[1];
-    dz1=zd1-Centroid[2];
-    dx2=xd2-Centroid[0];
-    dy2=yd2-Centroid[1];
-    dz2=zd2-Centroid[2];
+    dx1=xd1[0]-Centroid[0];
+    dy1=xd1[1]-Centroid[1];
+    dz1=xd1[2]-Centroid[2];
+    dx2=xd2[0]-Centroid[0];
+    dy2=xd2[1]-Centroid[1];
+    dz2=xd2[2]-Centroid[2];
     distFromDebug1=dx1*dx1+dy1*dy1+dz1*dz1;
     distFromDebug2=dx2*dx2+dy2*dy2+dz2*dz2;
-    if(std::min(distFromDebug1,distFromDebug2) < 1e-12) {
-         apf::MeshEntity*  stophere=f;  fprintf(stderr, "%d \n", stophere);
-         (void) stophere;
-    }
-    mesh->getAdjacent(f, 0, verts);
     int cmin=100;
     int cmax=-100;
     int cmid=-100;
     int ctri[4];  // up to 4 points on a face
+    int f1, f2, f1x, f2x, f1d, f2d;
     for(size_t i=0; i<verts.size(); i++) {
       mesh->getIntTag(verts[i],vtxClass,&c);
       cmin=std::min(cmin,c);
       cmax=std::max(cmax,c);
       ctri[i]=c;
     }
-       for(size_t i=0; i<verts.size(); i++) {
-         if((ctri[i] != cmin) && (ctri[i] != cmax)) cmid=std::max(cmid,ctri[i]);  // max is to catch lowest dim/highest code for quads....actually not necessary since either of the other two will follow switches as noted below.
-       }
-    if (cmin == INTERIORTAG) {
+    for(size_t i=0; i<verts.size(); i++) {
+      if((ctri[i] != cmin) && (ctri[i] != cmax)) cmid=std::max(cmid,ctri[i]);  // max is to catch lowest dim/highest code for quads....actually not necessary since either of the other two will follow switches as noted below.
+    }
+    if(std::min(distFromDebug1,distFromDebug2) < 1e-12) {
+         fprintf(stderr, "%d %d %d %.15e %.15E %.15E \n", cmin, cmid, cmax, Centroid[0], Centroid[1], Centroid[2]);
+    }
+    mesh->getAdjacent(f, 0, verts);
+    if (cmin == INTERIORTAG) {  // no brainer since a point on the interior always classifies interior
        mesh->setModelEntity(f,getMdlRgn(model));
        //cint++;
     } else  if(cmax <= FACE_LAST && cmin >= FACE) { // all nodes on  model face(s?)
@@ -290,16 +366,53 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
          } else { // all on same face so classify on that one
           mesh->setModelEntity(f,getMdlFace(mesh,(cmin-1)));
          }
-    } else if (cmin <= FACE_LAST) {  // one or two but not more on model faces 
-         if( cmid > FACE_LAST || cmin==cmid) {   // for tris, the first happens if 2 other on e and or v;  second happens if 2 on SAME face and max NOT on face due to previous else...this also catches quads of both types that can be on a face
-             mesh->setModelEntity(f,getMdlFace(mesh,(cmin-1))); // classified on cmin face either way
-         } else { // getting here only if cmid is a different face so must be interior
-             mesh->setModelEntity(f,getMdlRgn(model));
+    } else if (cmid <= FACE_LAST )   {  // two points on face(s?)
+         if( cmin==cmid) {   // happens if 2 on SAME face
+             if( cmax > EDGE_LAST) {   // happens if third is a model vert which is ALWAYS in the same plane
+               mesh->setModelEntity(f,getMdlFace(mesh,(cmin-1))); // classified on cmin face either way
+             } else {  // third on a model edge but might not be in closure of the face
+               f1=1+FofE[(cmax-11)][0];
+               f2=1+FofE[(cmax-11)][1];
+               if(f1==cmin || f2==cmin) { // if either face usage of this edge is cmin (or cmid) then class on face
+                   mesh->setModelEntity(f,getMdlFace(mesh,(cmin-1)));  
+               } else mesh->setModelEntity(f,getMdlRgn(model));  // edge not in face closure so class on region
+             }
+          } else mesh->setModelEntity(f,getMdlRgn(model)); // cmin != cmid means on different faces so clas on region
+    } else if (cmin <=FACE_LAST) { // since 3, and 2 face class handled already this is only 1 pt class on face
+         if (cmax > EDGE_LAST ){ // never 2 on verts so  1face, 1edge and 1 vert  check to see if edge in closure of face
+               f1=1+FofE[(cmid-11)][0];  // get two uses of the edge which is cmid
+               f2=1+FofE[(cmid-11)][1];
+               if(f1==cmin || f2==cmin) { // one of the usages is on cmin face and vertex is always so class on face
+                   mesh->setModelEntity(f,getMdlFace(mesh,(cmin-1)));  
+               } else mesh->setModelEntity(f,getMdlRgn(model));  // edge not in closor so classify on region
+         } else { // getting here only if 1face and 2 edges...most complicated case only face if  both edges in closure
+               f1x=1+FofE[(cmax-11)][0]; // first two find 2 face usages of max edge
+               f2x=1+FofE[(cmax-11)][1];
+               f1d=1+FofE[(cmid-11)][0]; // next two find 2 face usages of mid edge
+               f2d=1+FofE[(cmid-11)][1];
+               if(   (f1x==cmin || f2x==cmin)   // cmax edge is in cmin closure
+                  && (f1d==cmin || f2d==cmin) ) { // cmid edge is in cmin closure -- must have both to class on face
+                     mesh->setModelEntity(f,getMdlFace(mesh,(cmin-1)));  
+                  } else mesh->setModelEntity(f,getMdlRgn(model)); // at least one of the edges not in face closure so clssify on region
          }
-    } else if (cmax <= EDGE_LAST){  // since none on face and none on verts must be on 
+    } else if (cmax <= EDGE_LAST){  // since none on face (3, 2, and 1 done above) and none on verts, 3 edges-> interior 
         mesh->setModelEntity(f,getMdlRgn(model));
     } else { // getting here only if none of the face verts are interior or on model face so must be 2 e and 1 v but
+        f1x=1+FofE[(cmin-11)][0]; // first two find 2 face usages of min edge
+        f2x=1+FofE[(cmin-11)][1];
+        f1d=1+FofE[(cmid-11)][0]; // next two find 2 face usages of mid edge
+        f2d=1+FofE[(cmid-11)][1];
+        if (f1x==f1d || f2x==f1d) {  // these two check if ether face using cmin match first face using cmid
+              mesh->setModelEntity(f,getMdlFace(mesh,(f1d-1)));  
+        } else if ( f1x==f2d || f2x==f2d) {  // these two check if ether face using cmin match second face using cmid
+              mesh->setModelEntity(f,getMdlFace(mesh,(f2d-1)));  
+        } else {
+            fprintf(stderr, "face classification of these vert classification failed %d %d  %d \n", cmin, cmid, cmax);
+        }
+        if (0) {   
+// above is cleaner but below was first idea
 // none of these nodal classifications know directly the face to classify this face on.  HARD CODE for now to our case
+         
          int iface=0;
          if(cmax==31) {
             if( cmin== 11) {
@@ -347,6 +460,7 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
          }
          PCU_ALWAYS_ASSERT(iface != 0);
          mesh->setModelEntity(f,getMdlFace(mesh,(iface-1))); // classified on chosen face 
+        }
     }
   }
   mesh->end(it);

From 767bb3b5426defa2699245ef9eba4c9714fc1bab Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Sun, 29 Dec 2019 18:46:49 -0700
Subject: [PATCH 046/555] More Debug

---
 test/matchedNodeElmReader.cc | 41 +++++++++++++++++++++---------------
 1 file changed, 24 insertions(+), 17 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index d4d77b861..d69529de8 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -245,13 +245,6 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
   apf::MeshIterator* it = mesh->begin(2);
   apf::MeshEntity* f;
   int c;
-  apf::Adjacent verts;
-  double xd1[3]={-0.54864, 7.44015e-06, 0.0397148 };
-  double xd2[3]={0.914478, 0.0145401, 0.04 };
-//  double xd1=0.914478; double yd1=0.0145401; double zd1=0.04;
-//  double xd2=-0.548533; double yd2=7.46251e-06; double zd2=0.0399287;
-  double dx1,dy1,dz1,distFromDebug1;
-  double dx2,dy2,dz2,distFromDebug2;
   // int EofF[6][4];
   // int VofE[12][2];
   int FofE[12][2];
@@ -329,16 +322,27 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
   EofF[5][2]=10;
   EofF[5][3]=9;
 */
+  apf::Adjacent verts;
+//  double xd1[3]={-0.54864, 7.44015e-06, 0.0397148 };
+//  double xd2[3]={0.914478, 0.0145401, 0.04 };
+//  double dx1,dy1,dz1,distFromDebug1;
+//  double dx2,dy2,dz2,distFromDebug2;
+  apf::Vector3 xd1(-0.54864, 7.44015e-06, 0.0397148 );
+  apf::Vector3 xd2(0.914478, 0.0145401, 0.04 );
+  apf::Vector3 tmp;
+  apf::Vector3 Centroid;
   while( (f = mesh->iterate(it)) ) {
-    apf::Vector3 Centroid=apf::getLinearCentroid(mesh,f);
-    dx1=xd1[0]-Centroid[0];
-    dy1=xd1[1]-Centroid[1];
-    dz1=xd1[2]-Centroid[2];
-    dx2=xd2[0]-Centroid[0];
-    dy2=xd2[1]-Centroid[1];
-    dz2=xd2[2]-Centroid[2];
-    distFromDebug1=dx1*dx1+dy1*dy1+dz1*dz1;
-    distFromDebug2=dx2*dx2+dy2*dy2+dz2*dz2;
+    mesh->getAdjacent(f, 0, verts);
+    int nverts = verts.size();
+    Centroid=apf::getLinearCentroid(mesh,f);
+    dx1=xd1-Centroid;
+    dx2=xd2-Centroid;
+    distFromDebug1=dx1[0]*dx1[0]
+                  +dx1[1]*dx1[1]
+                  +dx1[2]*dx1[2];
+    distFromDebug2=dx2[0]*dx2[0]
+                  +dx2[1]*dx2[1]
+                  +dx2[2]*dx2[2];
     int cmin=100;
     int cmax=-100;
     int cmid=-100;
@@ -355,8 +359,11 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
     }
     if(std::min(distFromDebug1,distFromDebug2) < 1e-12) {
          fprintf(stderr, "%d %d %d %.15e %.15E %.15E \n", cmin, cmid, cmax, Centroid[0], Centroid[1], Centroid[2]);
+         for (size_t i=0; i < nverts; i++) {
+            mesh->getPoint(verts[i],0,tmp);
+            fprintf(stderr, "%d %.15e %.15E %.15E \n", i , tmp[0], tmp[1], tmp[2]);
+         }
     }
-    mesh->getAdjacent(f, 0, verts);
     if (cmin == INTERIORTAG) {  // no brainer since a point on the interior always classifies interior
        mesh->setModelEntity(f,getMdlRgn(model));
        //cint++;

From 0695a51a31e74ee08805a4703fca5d386777b972 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Sun, 29 Dec 2019 20:43:53 -0700
Subject: [PATCH 047/555] gets past verify so might be ok but ran out of
 memory...move to Cooley

---
 test/matchedNodeElmReader.cc | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index d69529de8..ae96ce698 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -327,8 +327,11 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
 //  double xd2[3]={0.914478, 0.0145401, 0.04 };
 //  double dx1,dy1,dz1,distFromDebug1;
 //  double dx2,dy2,dz2,distFromDebug2;
+  double distFromDebug1, distFromDebug2;
   apf::Vector3 xd1(-0.54864, 7.44015e-06, 0.0397148 );
   apf::Vector3 xd2(0.914478, 0.0145401, 0.04 );
+  apf::Vector3 dx1;
+  apf::Vector3 dx2;
   apf::Vector3 tmp;
   apf::Vector3 Centroid;
   while( (f = mesh->iterate(it)) ) {
@@ -354,8 +357,22 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
       cmax=std::max(cmax,c);
       ctri[i]=c;
     }
-    for(size_t i=0; i<verts.size(); i++) {
-      if((ctri[i] != cmin) && (ctri[i] != cmax)) cmid=std::max(cmid,ctri[i]);  // max is to catch lowest dim/highest code for quads....actually not necessary since either of the other two will follow switches as noted below.
+    int imax=0;  
+    int imin=0;  
+    for(size_t i=0; i<nverts; i++) {
+      if(ctri[i]==cmax) imax++;
+      if(ctri[i]==cmin) imin++;
+    }
+    if(cmax==cmin) { 
+       cmid= cmin;  // all nverts on same face
+    } else if (imax>=2) { // >=2 but not all with cmax
+       cmid=cmax;
+    } else if (imin>=2 ) { // >=2 but not all with min
+       cmid=cmin;
+    } else { // not 2 of either so mid is distinct
+      for(size_t i=0; i<nverts; i++) { // find it
+        if((ctri[i] != cmin) && (ctri[i] != cmax)) cmid=std::max(cmid,ctri[i]);  // max is to catch lowest dim/highest code for quads....actually not necessary since either of the other two will follow switches as noted below.
+      }
     }
     if(std::min(distFromDebug1,distFromDebug2) < 1e-12) {
          fprintf(stderr, "%d %d %d %.15e %.15E %.15E \n", cmin, cmid, cmax, Centroid[0], Centroid[1], Centroid[2]);

From 7afeab7305e6ce47b50c8bab37db8e517dd5be13 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Thu, 2 Jan 2020 13:41:35 -0700
Subject: [PATCH 048/555] face numbering map

---
 test/matchedNodeElmReader.cc | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index ae96ce698..1e32ca2b6 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -245,6 +245,7 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
   apf::MeshIterator* it = mesh->begin(2);
   apf::MeshEntity* f;
   int c;
+  int TofM[6] = {0, 1, 3, 4, 2, 5}; // tags of discrete model are NOT same as model faces
   // int EofF[6][4];
   // int VofE[12][2];
   int FofE[12][2];
@@ -322,6 +323,7 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
   EofF[5][2]=10;
   EofF[5][3]=9;
 */
+
   apf::Adjacent verts;
 //  double xd1[3]={-0.54864, 7.44015e-06, 0.0397148 };
 //  double xd2[3]={0.914478, 0.0145401, 0.04 };
@@ -388,17 +390,17 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
          if(cmax != cmin) { // all on faces but not all on same so classified on interior
              mesh->setModelEntity(f,getMdlRgn(model));
          } else { // all on same face so classify on that one
-          mesh->setModelEntity(f,getMdlFace(mesh,(cmin-1)));
+          mesh->setModelEntity(f,getMdlFace(mesh,TofM[cmin]));
          }
     } else if (cmid <= FACE_LAST )   {  // two points on face(s?)
          if( cmin==cmid) {   // happens if 2 on SAME face
              if( cmax > EDGE_LAST) {   // happens if third is a model vert which is ALWAYS in the same plane
-               mesh->setModelEntity(f,getMdlFace(mesh,(cmin-1))); // classified on cmin face either way
+               mesh->setModelEntity(f,getMdlFace(mesh,TofM[cmin])); // classified on cmin face either way
              } else {  // third on a model edge but might not be in closure of the face
                f1=1+FofE[(cmax-11)][0];
                f2=1+FofE[(cmax-11)][1];
                if(f1==cmin || f2==cmin) { // if either face usage of this edge is cmin (or cmid) then class on face
-                   mesh->setModelEntity(f,getMdlFace(mesh,(cmin-1)));  
+                   mesh->setModelEntity(f,getMdlFace(mesh,TofM[cmin]));  
                } else mesh->setModelEntity(f,getMdlRgn(model));  // edge not in face closure so class on region
              }
           } else mesh->setModelEntity(f,getMdlRgn(model)); // cmin != cmid means on different faces so clas on region
@@ -407,7 +409,7 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
                f1=1+FofE[(cmid-11)][0];  // get two uses of the edge which is cmid
                f2=1+FofE[(cmid-11)][1];
                if(f1==cmin || f2==cmin) { // one of the usages is on cmin face and vertex is always so class on face
-                   mesh->setModelEntity(f,getMdlFace(mesh,(cmin-1)));  
+                   mesh->setModelEntity(f,getMdlFace(mesh,TofM[cmin]));  
                } else mesh->setModelEntity(f,getMdlRgn(model));  // edge not in closor so classify on region
          } else { // getting here only if 1face and 2 edges...most complicated case only face if  both edges in closure
                f1x=1+FofE[(cmax-11)][0]; // first two find 2 face usages of max edge
@@ -416,7 +418,7 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
                f2d=1+FofE[(cmid-11)][1];
                if(   (f1x==cmin || f2x==cmin)   // cmax edge is in cmin closure
                   && (f1d==cmin || f2d==cmin) ) { // cmid edge is in cmin closure -- must have both to class on face
-                     mesh->setModelEntity(f,getMdlFace(mesh,(cmin-1)));  
+                     mesh->setModelEntity(f,getMdlFace(mesh,TofM[cmin]));  
                   } else mesh->setModelEntity(f,getMdlRgn(model)); // at least one of the edges not in face closure so clssify on region
          }
     } else if (cmax <= EDGE_LAST){  // since none on face (3, 2, and 1 done above) and none on verts, 3 edges-> interior 
@@ -427,9 +429,9 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
         f1d=1+FofE[(cmid-11)][0]; // next two find 2 face usages of mid edge
         f2d=1+FofE[(cmid-11)][1];
         if (f1x==f1d || f2x==f1d) {  // these two check if ether face using cmin match first face using cmid
-              mesh->setModelEntity(f,getMdlFace(mesh,(f1d-1)));  
+              mesh->setModelEntity(f,getMdlFace(mesh,TofM[f1d]));  
         } else if ( f1x==f2d || f2x==f2d) {  // these two check if ether face using cmin match second face using cmid
-              mesh->setModelEntity(f,getMdlFace(mesh,(f2d-1)));  
+              mesh->setModelEntity(f,getMdlFace(mesh,TofM[f2d]));  
         } else {
             fprintf(stderr, "face classification of these vert classification failed %d %d  %d \n", cmin, cmid, cmax);
         }
@@ -483,7 +485,7 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
             fprintf(stderr, "%d %d  %d \n", cmin, cmid, cmax);
          }
          PCU_ALWAYS_ASSERT(iface != 0);
-         mesh->setModelEntity(f,getMdlFace(mesh,(iface-1))); // classified on chosen face 
+         mesh->setModelEntity(f,getMdlFace(mesh,TofM[iface-1])); // classified on chosen face 
         }
     }
   }

From 5a2ea09a41f37583babdc8bd4682606e9d60dd9b Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Tue, 7 Jan 2020 16:04:25 +0000
Subject: [PATCH 049/555]  commenting out gmi call that won't compile on Cray
 also disabled verify and fixed some bugs for triangle classificaiton in
 matchedNodeElmReader

---
 gmi/gmi_analytic.c           |  2 +-
 phasta/phCook.cc             |  2 +-
 test/matchedNodeElmReader.cc | 24 +++++++++++++-----------
 3 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/gmi/gmi_analytic.c b/gmi/gmi_analytic.c
index 04b34ce57..c7771c54f 100644
--- a/gmi/gmi_analytic.c
+++ b/gmi/gmi_analytic.c
@@ -124,7 +124,7 @@ static void reparam_path(struct gmi_analytic* m, struct agm_use* path,
     return;
   }
   reparam_across(m, *path, from_p, tmp);
-  reparam_path(m, path + 1, pathlen - 1, tmp, to_p);
+//  reparam_path(m, (path + 1), (pathlen - 1), tmp, to_p);
 }
 
 static void reparam(struct gmi_model* m, struct gmi_ent* from,
diff --git a/phasta/phCook.cc b/phasta/phCook.cc
index 35abd2714..205360600 100644
--- a/phasta/phCook.cc
+++ b/phasta/phCook.cc
@@ -78,7 +78,7 @@ void originalMain(apf::Mesh2*& m, ph::Input& in,
     m = loadMesh(g, in);
   else
     apf::printStats(m);
-  m->verify();
+// Need to set a flag to enable avoiding this when short on time  m->verify();
   if (in.solutionMigration && !in.useAttachedFields)
     ph::readAndAttachFields(in, m);
   else
diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 1e32ca2b6..81f7434d2 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -325,10 +325,7 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
 */
 
   apf::Adjacent verts;
-//  double xd1[3]={-0.54864, 7.44015e-06, 0.0397148 };
-//  double xd2[3]={0.914478, 0.0145401, 0.04 };
-//  double dx1,dy1,dz1,distFromDebug1;
-//  double dx2,dy2,dz2,distFromDebug2;
+/*
   double distFromDebug1, distFromDebug2;
   apf::Vector3 xd1(-0.54864, 7.44015e-06, 0.0397148 );
   apf::Vector3 xd2(0.914478, 0.0145401, 0.04 );
@@ -336,9 +333,11 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
   apf::Vector3 dx2;
   apf::Vector3 tmp;
   apf::Vector3 Centroid;
+  */
   while( (f = mesh->iterate(it)) ) {
     mesh->getAdjacent(f, 0, verts);
     int nverts = verts.size();
+/* 
     Centroid=apf::getLinearCentroid(mesh,f);
     dx1=xd1-Centroid;
     dx2=xd2-Centroid;
@@ -348,6 +347,7 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
     distFromDebug2=dx2[0]*dx2[0]
                   +dx2[1]*dx2[1]
                   +dx2[2]*dx2[2];
+  */
     int cmin=100;
     int cmax=-100;
     int cmid=-100;
@@ -376,6 +376,7 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
         if((ctri[i] != cmin) && (ctri[i] != cmax)) cmid=std::max(cmid,ctri[i]);  // max is to catch lowest dim/highest code for quads....actually not necessary since either of the other two will follow switches as noted below.
       }
     }
+/* 
     if(std::min(distFromDebug1,distFromDebug2) < 1e-12) {
          fprintf(stderr, "%d %d %d %.15e %.15E %.15E \n", cmin, cmid, cmax, Centroid[0], Centroid[1], Centroid[2]);
          for (size_t i=0; i < nverts; i++) {
@@ -383,6 +384,7 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
             fprintf(stderr, "%d %.15e %.15E %.15E \n", i , tmp[0], tmp[1], tmp[2]);
          }
     }
+     */
     if (cmin == INTERIORTAG) {  // no brainer since a point on the interior always classifies interior
        mesh->setModelEntity(f,getMdlRgn(model));
        //cint++;
@@ -390,17 +392,17 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
          if(cmax != cmin) { // all on faces but not all on same so classified on interior
              mesh->setModelEntity(f,getMdlRgn(model));
          } else { // all on same face so classify on that one
-          mesh->setModelEntity(f,getMdlFace(mesh,TofM[cmin]));
+          mesh->setModelEntity(f,getMdlFace(mesh,TofM[cmin-1]));
          }
     } else if (cmid <= FACE_LAST )   {  // two points on face(s?)
          if( cmin==cmid) {   // happens if 2 on SAME face
              if( cmax > EDGE_LAST) {   // happens if third is a model vert which is ALWAYS in the same plane
-               mesh->setModelEntity(f,getMdlFace(mesh,TofM[cmin])); // classified on cmin face either way
+               mesh->setModelEntity(f,getMdlFace(mesh,TofM[cmin-1])); // classified on cmin face either way
              } else {  // third on a model edge but might not be in closure of the face
                f1=1+FofE[(cmax-11)][0];
                f2=1+FofE[(cmax-11)][1];
                if(f1==cmin || f2==cmin) { // if either face usage of this edge is cmin (or cmid) then class on face
-                   mesh->setModelEntity(f,getMdlFace(mesh,TofM[cmin]));  
+                   mesh->setModelEntity(f,getMdlFace(mesh,TofM[cmin-1]));  
                } else mesh->setModelEntity(f,getMdlRgn(model));  // edge not in face closure so class on region
              }
           } else mesh->setModelEntity(f,getMdlRgn(model)); // cmin != cmid means on different faces so clas on region
@@ -409,7 +411,7 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
                f1=1+FofE[(cmid-11)][0];  // get two uses of the edge which is cmid
                f2=1+FofE[(cmid-11)][1];
                if(f1==cmin || f2==cmin) { // one of the usages is on cmin face and vertex is always so class on face
-                   mesh->setModelEntity(f,getMdlFace(mesh,TofM[cmin]));  
+                   mesh->setModelEntity(f,getMdlFace(mesh,TofM[cmin-1]));  
                } else mesh->setModelEntity(f,getMdlRgn(model));  // edge not in closor so classify on region
          } else { // getting here only if 1face and 2 edges...most complicated case only face if  both edges in closure
                f1x=1+FofE[(cmax-11)][0]; // first two find 2 face usages of max edge
@@ -418,7 +420,7 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
                f2d=1+FofE[(cmid-11)][1];
                if(   (f1x==cmin || f2x==cmin)   // cmax edge is in cmin closure
                   && (f1d==cmin || f2d==cmin) ) { // cmid edge is in cmin closure -- must have both to class on face
-                     mesh->setModelEntity(f,getMdlFace(mesh,TofM[cmin]));  
+                     mesh->setModelEntity(f,getMdlFace(mesh,TofM[cmin-1]));  
                   } else mesh->setModelEntity(f,getMdlRgn(model)); // at least one of the edges not in face closure so clssify on region
          }
     } else if (cmax <= EDGE_LAST){  // since none on face (3, 2, and 1 done above) and none on verts, 3 edges-> interior 
@@ -429,9 +431,9 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
         f1d=1+FofE[(cmid-11)][0]; // next two find 2 face usages of mid edge
         f2d=1+FofE[(cmid-11)][1];
         if (f1x==f1d || f2x==f1d) {  // these two check if ether face using cmin match first face using cmid
-              mesh->setModelEntity(f,getMdlFace(mesh,TofM[f1d]));  
+              mesh->setModelEntity(f,getMdlFace(mesh,TofM[f1d-1]));  
         } else if ( f1x==f2d || f2x==f2d) {  // these two check if ether face using cmin match second face using cmid
-              mesh->setModelEntity(f,getMdlFace(mesh,TofM[f2d]));  
+              mesh->setModelEntity(f,getMdlFace(mesh,TofM[f2d-1]));  
         } else {
             fprintf(stderr, "face classification of these vert classification failed %d %d  %d \n", cmin, cmid, cmax);
         }

From d9391058313785ab1ce88ee362d11bdd8a876d77 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Thu, 6 Feb 2020 00:29:41 +0000
Subject: [PATCH 050/555] adding fields for transfer

---
 phasta/phRestart.cc | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/phasta/phRestart.cc b/phasta/phRestart.cc
index b8c8a2af6..ac069cf4b 100644
--- a/phasta/phRestart.cc
+++ b/phasta/phRestart.cc
@@ -283,13 +283,16 @@ static bool isNodalField(const char* fieldname, int nnodes, apf::Mesh* m)
     "wssbar",
     "pressure projection vectors",
     "projection vectors",
-    "vorticity"
+    "vorticity",
+    "BCs",
+    "cdelsq"
   };
   static char const* const known_cell_fields[] = {
     "VOF solution",
     "meshQ",
     "meshCFL",
-    "material_type"
+    "material_type",
+    "lesnut"
   };
   static char const* const known_rand_fields[] = {
     "rbParams"
@@ -507,6 +510,10 @@ void detachAndWriteSolution(Input& in, Output& out, apf::Mesh* m, std::string pa
     detachAndWriteField(in, m, f, "dc_lag");
   if (m->findField("pressure projection vectors"))
     detachAndWriteField(in, m, f, "pressure projection vectors");
+  if (m->findField("BCs"))
+    detachAndWriteField(in, m, f, "BCs");
+  if (m->findField("cdelsq"))
+    detachAndWriteField(in, m, f, "cdelsq");
   if (in.displacementMigration)
     detachAndWriteField(in, m, f, "displacement");
   if (in.dwalMigration)

From f567fc13be01f560192fdad97a4e55432d3bec88 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Thu, 6 Feb 2020 09:04:09 -0700
Subject: [PATCH 051/555] most of the way to reading a solution into stream

---
 test/matchedNodeElmReader.cc | 29 ++++++++++++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 81f7434d2..f36d3b8bb 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -619,6 +619,24 @@ void readCoords(FILE* f, unsigned numvtx, unsigned& localnumvtx, double** coordi
   }
 }
 
+void readSolution(FILE* f, unsigned numvtx, unsigned& localnumvtx, double** solution) {
+  long firstVtx, lastVtx;
+  getLocalRange(numvtx, localnumvtx,firstVtx,lastVtx);
+  *solution = new double[localnumvtx*4];
+  rewind(f);
+  int vidx = 0;
+  for(unsigned i=0; i<numvtx; i++) {
+    int id;
+    double pos[4];
+    gmi_fscanf(f, 5, "%d %lf %lf %lf %lf", &id, pos+0, pos+1, pos+2, pos+3);
+    if( i >= firstVtx && i < lastVtx ) {
+      for(unsigned j=0; j<4; j++)
+        (*solution)[vidx*4+j] = pos[j];
+      vidx++;
+    }
+  }
+}
+
 void readMatches(FILE* f, unsigned numvtx, int** matches) {
   long firstVtx, lastVtx;
   unsigned localnumvtx;
@@ -679,6 +697,7 @@ void readElements(FILE* f, unsigned &dim, unsigned& numElms,
 
 struct MeshInfo {
   double* coords;
+  double* solution;
   int* elements;
   int* matches;
   int* classification;
@@ -697,6 +716,7 @@ void readMesh(const char* meshfilename,
     const char* matchfilename,
     const char* classfilename,
     const char* fathers2Dfilename,
+    const char* solutionfilename,
     MeshInfo& mesh) {
   FILE* fc = fopen(coordfilename, "r");
   PCU_ALWAYS_ASSERT(fc);
@@ -705,6 +725,13 @@ void readMesh(const char* meshfilename,
     fprintf(stderr, "numVerts %u\n", mesh.numVerts);
   readCoords(fc, mesh.numVerts, mesh.localNumVerts, &(mesh.coords));
   fclose(fc);
+ 
+  if(0==1) {
+  FILE* fs = fopen(solutionfilename, "r");
+  PCU_ALWAYS_ASSERT(fs);
+  readSolution(fs, mesh.numVerts, mesh.localNumVerts, &(mesh.solution));
+  fclose(fs);
+  }
 
   FILE* ff = fopen(classfilename, "r");
   PCU_ALWAYS_ASSERT(ff);
@@ -757,7 +784,7 @@ int main(int argc, char** argv)
 
   double t0 = PCU_Time();
   MeshInfo m;
-  readMesh(argv[1],argv[2],argv[3],argv[4],argv[5],m);
+  readMesh(argv[1],argv[2],argv[3],argv[4],argv[5],argv[6],m);
 
   bool isMatched = true;
   if( !strcmp(argv[3], "NULL") )

From 807b9fe2cf59d2bb96c34a74896ce49307e6622f Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 7 Feb 2020 08:44:08 -0500
Subject: [PATCH 052/555] templated function to set mapped tags

---
 apf/apfConstruct.cc          |  83 --------------------
 apf/apfConvert.h             |  11 ---
 apf/apfConvertTags.h         | 144 +++++++++++++++++++++++++++++++++++
 test/matchedNodeElmReader.cc |   9 ++-
 4 files changed, 149 insertions(+), 98 deletions(-)
 create mode 100644 apf/apfConvertTags.h

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index daee252b2..1d24e3b55 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -238,89 +238,6 @@ void setCoords(Mesh2* m, const double* coords, int nverts,
   delete [] c;
 }
 
-apf::MeshTag* setIntTag(Mesh2* m, const int* vals, const int entries,
-    int nverts, GlobalToVert& globalToVert)
-{
-  Gid max = getMax(globalToVert);
-  Gid total = max + 1;
-  int peers = PCU_Comm_Peers();
-  int quotient = total / peers;
-  int remainder = total % peers;
-  int mySize = quotient;
-  int self = PCU_Comm_Self();
-  if (self == (peers - 1))
-    mySize += remainder;
-  int myOffset = self * quotient;
-
-  /* Force each peer to have exactly mySize verts.
-     This means we might need to send and recv some coords */
-  int* c = new int[mySize*entries];
-
-  int start = PCU_Exscan_Int(nverts);
-
-  PCU_Comm_Begin();
-  int to = std::min(peers - 1, start / quotient);
-  int n = std::min((to+1)*quotient-start, nverts);
-  while (nverts > 0) {
-    PCU_COMM_PACK(to, start);
-    PCU_COMM_PACK(to, n);
-    PCU_Comm_Pack(to, vals, n*entries*sizeof(int));
-
-    nverts -= n;
-    start += n;
-    vals += n*entries;
-    to = std::min(peers - 1, to + 1);
-    n = std::min(quotient, nverts);
-  }
-  PCU_Comm_Send();
-  while (PCU_Comm_Receive()) {
-    PCU_COMM_UNPACK(start);
-    PCU_COMM_UNPACK(n);
-    PCU_Comm_Unpack(&c[(start - myOffset) * entries], n*entries*sizeof(int));
-  }
-
-  /* Tell all the owners of the data what we need */
-  typedef std::vector< std::vector<int> > TmpParts;
-  TmpParts tmpParts(mySize);
-  PCU_Comm_Begin();
-  APF_CONST_ITERATE(GlobalToVert, globalToVert, it) {
-    int gid = it->first;
-    int to = std::min(peers - 1, gid / quotient);
-    PCU_COMM_PACK(to, gid);
-  }
-  PCU_Comm_Send();
-  while (PCU_Comm_Receive()) {
-    int gid;
-    PCU_COMM_UNPACK(gid);
-    int from = PCU_Comm_Sender();
-    tmpParts.at(gid - myOffset).push_back(from);
-  }
-
-  /* Send the data to everybody who want them */
-  PCU_Comm_Begin();
-  for (int i = 0; i < mySize; ++i) {
-    std::vector<int>& parts = tmpParts[i];
-    for (size_t j = 0; j < parts.size(); ++j) {
-      int to = parts[j];
-      int gid = i + myOffset;
-      PCU_COMM_PACK(to, gid);
-      PCU_Comm_Pack(to, &c[i*entries], entries*sizeof(int));
-    }
-  }
-  PCU_Comm_Send();
-  apf::MeshTag* t = m->createIntTag("userInts",entries);
-  int* v = new int[entries];
-  while (PCU_Comm_Receive()) {
-    int gid;
-    PCU_COMM_UNPACK(gid);
-    PCU_Comm_Unpack(v, entries*sizeof(int));
-    m->setIntTag(globalToVert[gid],t,v);
-  }
-  delete [] v;
-  delete [] c;
-  return t;
-}
-
 void setMatches(Mesh2* m, const int* matches, int nverts,
     GlobalToVert& globalToVert)
 {
diff --git a/apf/apfConvert.h b/apf/apfConvert.h
index 2bb27282b..ac271f44a 100644
--- a/apf/apfConvert.h
+++ b/apf/apfConvert.h
@@ -60,17 +60,6 @@ void construct(Mesh2* m, const int* conn, int nelem, int etype,
 void setCoords(Mesh2* m, const double* coords, int nverts,
     GlobalToVert& globalToVert);
 
-/** \brief Assign coordinates to the mesh
-  * \param vals (in) array of nverts ints
-  * \param entries (in) number of ints per vertex
-  * \param nverts (in) number of vertices for this process
-  * \param globalToVert (in) map from global mesh vertex ids
-  *                          to local vertex * pointers
-  * \details
-  * See 'setCoords' for distribution details
-  */
-MeshTag* setIntTag(Mesh2* m, const int* vals, const int entries,
-    int nverts, GlobalToVert& globalToVert);
 
 /** \brief Assign matching to the mesh
   * \details
diff --git a/apf/apfConvertTags.h b/apf/apfConvertTags.h
new file mode 100644
index 000000000..ab2673f85
--- /dev/null
+++ b/apf/apfConvertTags.h
@@ -0,0 +1,144 @@
+#ifndef APF_CONVERT_TAGS_H
+#define APF_CONVERT_TAGS_H
+
+#include <PCU.h>
+#include "apf.h"
+#include "apfConvert.h"
+
+namespace {
+  typedef int Gid;
+  static Gid getMax(const apf::GlobalToVert& globalToVert)
+  {
+    Gid max = -1;
+    APF_CONST_ITERATE(apf::GlobalToVert, globalToVert, it)
+      max = std::max(max, it->first);
+    return PCU_Max_Int(max); // this is type-dependent
+  }
+
+  template <class T> inline
+  apf::MeshTag* createTag(apf::Mesh*,
+      const char*, const int) {
+    exit(EXIT_FAILURE);
+  }
+  
+  template <> inline
+  apf::MeshTag* createTag<int>(apf::Mesh* m,
+      const char* name, const int entries) {
+    return m->createIntTag(name,entries);
+  }
+  
+  template <> inline
+  apf::MeshTag* createTag<double>(apf::Mesh* m,
+      const char* name, const int entries) {
+    return m->createDoubleTag(name,entries);
+  }
+  
+  inline void setEntTag(apf::Mesh* m, apf::MeshTag* t,
+      apf::MeshEntity* e, int* vals) {
+    return m->setIntTag(e,t,vals);
+  }
+  
+  inline void setEntTag(apf::Mesh* m, apf::MeshTag* t,
+      apf::MeshEntity* e, double* vals) {
+    return m->setDoubleTag(e,t,vals);
+  }
+}
+
+namespace apf {
+/** \brief Assign tag values to the mesh
+  * \param m (in) mesh
+  * \param vals (in) T array of length nverts 
+  * \param entries (in) number of values per vertex
+  * \param nverts (in) number of vertices for this process
+  * \param globalToVert (in) map from global mesh vertex ids
+  *                          to local vertex * pointers
+  * \param tagName (in) name of returned tag
+  * \details
+  * See 'setCoords' for distribution details
+  */
+template<class T> 
+apf::MeshTag* setMappedTag(Mesh2* m, const T* vals, const int entries,
+    int nverts, GlobalToVert& globalToVert, const char* tagName)
+{
+  Gid max = getMax(globalToVert);
+  Gid total = max + 1;
+  int peers = PCU_Comm_Peers();
+  int quotient = total / peers;
+  int remainder = total % peers;
+  int mySize = quotient;
+  int self = PCU_Comm_Self();
+  if (self == (peers - 1))
+    mySize += remainder;
+  int myOffset = self * quotient;
+
+  /* Force each peer to have exactly mySize verts.
+     This means we might need to send and recv some coords */
+  T* c = new T[mySize*entries];
+
+  int start = PCU_Exscan_Int(nverts);
+
+  PCU_Comm_Begin();
+  int to = std::min(peers - 1, start / quotient);
+  int n = std::min((to+1)*quotient-start, nverts);
+  while (nverts > 0) {
+    PCU_COMM_PACK(to, start);
+    PCU_COMM_PACK(to, n);
+    PCU_Comm_Pack(to, vals, n*entries*sizeof(T));
+
+    nverts -= n;
+    start += n;
+    vals += n*entries;
+    to = std::min(peers - 1, to + 1);
+    n = std::min(quotient, nverts);
+  }
+  PCU_Comm_Send();
+  while (PCU_Comm_Receive()) {
+    PCU_COMM_UNPACK(start);
+    PCU_COMM_UNPACK(n);
+    PCU_Comm_Unpack(&c[(start - myOffset) * entries], n*entries*sizeof(T));
+  }
+
+  /* Tell all the owners of the data what we need */
+  typedef std::vector< std::vector<int> > TmpParts;
+  TmpParts tmpParts(mySize);
+  PCU_Comm_Begin();
+  APF_CONST_ITERATE(GlobalToVert, globalToVert, it) {
+    int gid = it->first;
+    int to = std::min(peers - 1, gid / quotient);
+    PCU_COMM_PACK(to, gid);
+  }
+  PCU_Comm_Send();
+  while (PCU_Comm_Receive()) {
+    int gid;
+    PCU_COMM_UNPACK(gid);
+    int from = PCU_Comm_Sender();
+    tmpParts.at(gid - myOffset).push_back(from);
+  }
+
+  /* Send the data to everybody who want them */
+  PCU_Comm_Begin();
+  for (int i = 0; i < mySize; ++i) {
+    std::vector<int>& parts = tmpParts[i];
+    for (size_t j = 0; j < parts.size(); ++j) {
+      int to = parts[j];
+      int gid = i + myOffset;
+      PCU_COMM_PACK(to, gid);
+      PCU_Comm_Pack(to, &c[i*entries], entries*sizeof(T));
+    }
+  }
+  PCU_Comm_Send();
+  apf::MeshTag* t = createTag<T>(m,tagName,entries);
+  T* v = new T[entries];
+  while (PCU_Comm_Receive()) {
+    int gid;
+    PCU_COMM_UNPACK(gid);
+    PCU_Comm_Unpack(v, entries*sizeof(T));
+    setEntTag(m,t,globalToVert[gid],v);
+  }
+  delete [] v;
+  delete [] c;
+  return t;
+}
+
+}
+#endif
diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 20dae056b..2f7355a17 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -3,6 +3,7 @@
 #include <apfMDS.h>
 #include <apfMesh2.h>
 #include <apfConvert.h>
+#include <apfConvertTags.h>
 #include <apf.h>
 #include <PCU.h>
 #include <lionPrint.h>
@@ -627,10 +628,10 @@ int main(int argc, char** argv)
     mesh->acceptChanges();
     delete [] m.matches;
   }
-  apf::MeshTag* t = setIntTag(mesh, m.classification, 1,
-      m.localNumVerts, outMap);
-  apf::MeshTag* tfathers2D = setIntTag(mesh, m.fathers2D, 1,
-      m.localNumVerts, outMap);
+  apf::MeshTag* t = setMappedTag(mesh, m.classification, 1,
+      m.localNumVerts, outMap, "mner_classification");
+  apf::MeshTag* tfathers2D = setMappedTag(mesh, m.fathers2D, 1,
+      m.localNumVerts, outMap, "mner_fathers2D");
   outMap.clear();
   setClassification(model,mesh,t);
   apf::removeTagFromDimension(mesh, t, 0);

From 4b87b725947d4d7c9a671845efaff68cdca545f3 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 7 Feb 2020 08:51:33 -0500
Subject: [PATCH 053/555] fix arg order

---
 apf/apfConvertTags.h | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/apf/apfConvertTags.h b/apf/apfConvertTags.h
index ab2673f85..6629dcc36 100644
--- a/apf/apfConvertTags.h
+++ b/apf/apfConvertTags.h
@@ -47,18 +47,19 @@ namespace {
 namespace apf {
 /** \brief Assign tag values to the mesh
   * \param m (in) mesh
+  * \param tagName (in) name of returned tag
   * \param vals (in) T array of length nverts 
   * \param entries (in) number of values per vertex
   * \param nverts (in) number of vertices for this process
   * \param globalToVert (in) map from global mesh vertex ids
   *                          to local vertex * pointers
-  * \param tagName (in) name of returned tag
   * \details
   * See 'setCoords' for distribution details
   */
 template<class T> 
-apf::MeshTag* setMappedTag(Mesh2* m, const T* vals, const int entries,
-    int nverts, GlobalToVert& globalToVert, const char* tagName)
+apf::MeshTag* setMappedTag(Mesh2* m, const char* tagName,
+    const T* vals, const int entries,
+    int nverts, GlobalToVert& globalToVert)
 {
   Gid max = getMax(globalToVert);
   Gid total = max + 1;

From e9c439ccb18dea073852d8ceaaa7b4a244ad6d99 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 7 Feb 2020 08:52:37 -0500
Subject: [PATCH 054/555] fix sign comparision warning

---
 test/matchedNodeElmReader.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index d4afbc5ef..9551803cf 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -362,7 +362,7 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
     }
     int imax=0;  
     int imin=0;  
-    for(size_t i=0; i<nverts; i++) {
+    for(int i=0; i<nverts; i++) {
       if(ctri[i]==cmax) imax++;
       if(ctri[i]==cmin) imin++;
     }
@@ -373,7 +373,7 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
     } else if (imin>=2 ) { // >=2 but not all with min
        cmid=cmin;
     } else { // not 2 of either so mid is distinct
-      for(size_t i=0; i<nverts; i++) { // find it
+      for(int i=0; i<nverts; i++) { // find it
         if((ctri[i] != cmin) && (ctri[i] != cmax)) cmid=std::max(cmid,ctri[i]);  // max is to catch lowest dim/highest code for quads....actually not necessary since either of the other two will follow switches as noted below.
       }
     }

From 54a0d16a9fa3b315adf6d5a483a4096f7462b884 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Fri, 7 Feb 2020 09:21:17 -0700
Subject: [PATCH 055/555] partial merge but not seeing setMappedTag calls

---
 phasta/phCook.cc             |  4 +++-
 test/matchedNodeElmReader.cc | 30 ++++++++++++++++--------------
 2 files changed, 19 insertions(+), 15 deletions(-)

diff --git a/phasta/phCook.cc b/phasta/phCook.cc
index 205360600..a085e4272 100644
--- a/phasta/phCook.cc
+++ b/phasta/phCook.cc
@@ -79,7 +79,9 @@ void originalMain(apf::Mesh2*& m, ph::Input& in,
   else
     apf::printStats(m);
 // Need to set a flag to enable avoiding this when short on time  m->verify();
-  if (in.solutionMigration && !in.useAttachedFields)
+  if (in.useAttachedFields) 
+     lion_eprint(1,"because useAttachedFields set restart not read\n");
+  else if (in.solutionMigration && !in.useAttachedFields)
     ph::readAndAttachFields(in, m);
   else
     ph::attachZeroSolution(in, m);
diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 9551803cf..4a00968be 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -337,7 +337,7 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
   */
   while( (f = mesh->iterate(it)) ) {
     mesh->getAdjacent(f, 0, verts);
-    int nverts = verts.size();
+    size_t nverts = verts.size();
 /* 
     Centroid=apf::getLinearCentroid(mesh,f);
     dx1=xd1-Centroid;
@@ -362,7 +362,7 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
     }
     int imax=0;  
     int imin=0;  
-    for(int i=0; i<nverts; i++) {
+    for(size_t i=0; i<nverts; i++) {
       if(ctri[i]==cmax) imax++;
       if(ctri[i]==cmin) imin++;
     }
@@ -373,7 +373,7 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
     } else if (imin>=2 ) { // >=2 but not all with min
        cmid=cmin;
     } else { // not 2 of either so mid is distinct
-      for(int i=0; i<nverts; i++) { // find it
+      for(size_t i=0; i<nverts; i++) { // find it
         if((ctri[i] != cmin) && (ctri[i] != cmax)) cmid=std::max(cmid,ctri[i]);  // max is to catch lowest dim/highest code for quads....actually not necessary since either of the other two will follow switches as noted below.
       }
     }
@@ -623,16 +623,16 @@ void readCoords(FILE* f, unsigned numvtx, unsigned& localnumvtx, double** coordi
 void readSolution(FILE* f, unsigned numvtx, unsigned& localnumvtx, double** solution) {
   long firstVtx, lastVtx;
   getLocalRange(numvtx, localnumvtx,firstVtx,lastVtx);
-  *solution = new double[localnumvtx*4];
+  *solution = new double[localnumvtx*5];
   rewind(f);
   int vidx = 0;
   for(unsigned i=0; i<numvtx; i++) {
-    int id;
-    double pos[4];
-    gmi_fscanf(f, 5, "%d %lf %lf %lf %lf", &id, pos+0, pos+1, pos+2, pos+3);
+    double pos[5];
+    pos[4]=0; //temperature
+    gmi_fscanf(f, 4, "%lf %lf %lf %lf", pos+0, pos+1, pos+2, pos+3);
     if( i >= firstVtx && i < lastVtx ) {
-      for(unsigned j=0; j<4; j++)
-        (*solution)[vidx*4+j] = pos[j];
+      for(unsigned j=0; j<5; j++)
+        (*solution)[vidx*5+j] = pos[j];
       vidx++;
     }
   }
@@ -727,7 +727,7 @@ void readMesh(const char* meshfilename,
   readCoords(fc, mesh.numVerts, mesh.localNumVerts, &(mesh.coords));
   fclose(fc);
  
-  if(0==1) {
+  if(1==1) {
   FILE* fs = fopen(solutionfilename, "r");
   PCU_ALWAYS_ASSERT(fs);
   readSolution(fs, mesh.numVerts, mesh.localNumVerts, &(mesh.solution));
@@ -766,13 +766,14 @@ int main(int argc, char** argv)
   MPI_Init(&argc,&argv);
   PCU_Comm_Init();
   lion_set_verbosity(1);
-  if( argc != 8 ) {
+  if( argc != 9 ) {
     if( !PCU_Comm_Self() ) {
       printf("Usage: %s <ascii mesh connectivity .cnn> "
           "<ascii vertex coordinates .crd> "
           "<ascii vertex matching flag .match> "
           "<ascii vertex classification flag .class> "
           "<ascii vertex fathers2D flag .fathers2D> "
+          "<ascii solution flag .soln> "
           "<output model .dmg> <output mesh .smb>\n",
           argv[0]);
     }
@@ -812,17 +813,18 @@ int main(int argc, char** argv)
     mesh->acceptChanges();
     delete [] m.matches;
   }
-  apf::MeshTag* tc = setMappedTag(mesh, "classification", m.classification, 1,
+  apf::MeshTag* tc = setIntTag(mesh, "classification", m.classification, 1,
       m.localNumVerts, outMap);
   setClassification(model,mesh,tc);
   apf::removeTagFromDimension(mesh, tc, 0);
   mesh->destroyTag(tc);
  
-  apf::MeshTag* tf = setMappedTag(mesh, "fathers2D", m.fathers2D, 1,
-      m.localNumVerts, outMap);
+  apf::MeshTag* tf = setIntTag(mesh, "fathers2D", m.fathers2D, 1,
+       m.localNumVerts, outMap);
   (void) tf;
   //mesh->destroyTag(tf);
 
+
   /* // Print the father2D tags
   apf::MeshEntity* v;
   apf::MeshIterator* it = mesh->begin(0);

From 897fdb40ac3e56d4dd67adbbda453ea532545d7e Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Fri, 7 Feb 2020 10:17:14 -0700
Subject: [PATCH 056/555] reading solution and attaching v1.

---
 test/matchedNodeElmReader.cc | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 4a00968be..6870e846a 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -813,17 +813,22 @@ int main(int argc, char** argv)
     mesh->acceptChanges();
     delete [] m.matches;
   }
-  apf::MeshTag* tc = setIntTag(mesh, "classification", m.classification, 1,
+  apf::MeshTag* tc = setMappedTag(mesh, "classification", m.classification, 1,
       m.localNumVerts, outMap);
   setClassification(model,mesh,tc);
   apf::removeTagFromDimension(mesh, tc, 0);
   mesh->destroyTag(tc);
  
-  apf::MeshTag* tf = setIntTag(mesh, "fathers2D", m.fathers2D, 1,
-       m.localNumVerts, outMap);
+  apf::MeshTag* tf = setMappedTag(mesh, "fathers2D", m.fathers2D, 1,
+      m.localNumVerts, outMap);
   (void) tf;
   //mesh->destroyTag(tf);
 
+  apf::MeshTag* ts = setMappedTag(mesh, "solution", m.solution, 1,
+      m.localNumVerts, outMap);
+  (void) ts;
+  //mesh->destroyTag(tf);
+
 
   /* // Print the father2D tags
   apf::MeshEntity* v;
@@ -844,8 +849,8 @@ int main(int argc, char** argv)
   mesh->verify();
 
   outMap.clear();
-  gmi_write_dmg(model, argv[6]);
-  mesh->writeNative(argv[7]);
+  gmi_write_dmg(model, argv[7]);
+  mesh->writeNative(argv[8]);
   apf::writeVtkFiles("rendered",mesh);
 
   mesh->destroyNative();

From 0b5366fb3eff2407fa1fe767c33007420d025a4d Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Fri, 7 Feb 2020 15:02:18 -0700
Subject: [PATCH 057/555] I think this is close to working but needs the final
 step of converting a tag to a field

---
 test/matchedNodeElmReader.cc | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 6870e846a..e4694829d 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -727,7 +727,7 @@ void readMesh(const char* meshfilename,
   readCoords(fc, mesh.numVerts, mesh.localNumVerts, &(mesh.coords));
   fclose(fc);
  
-  if(1==1) {
+  if(0==1) {
   FILE* fs = fopen(solutionfilename, "r");
   PCU_ALWAYS_ASSERT(fs);
   readSolution(fs, mesh.numVerts, mesh.localNumVerts, &(mesh.solution));
@@ -824,10 +824,11 @@ int main(int argc, char** argv)
   (void) tf;
   //mesh->destroyTag(tf);
 
-  apf::MeshTag* ts = setMappedTag(mesh, "solution", m.solution, 1,
+  if(0==1) {
+  apf::MeshTag* ts = setMappedTag(mesh, "solution", m.solution, 5,
       m.localNumVerts, outMap);
   (void) ts;
-  //mesh->destroyTag(tf);
+  }
 
 
   /* // Print the father2D tags

From 3951cccaf3cafc1a79820b4033eecc19800fb2f6 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Fri, 7 Feb 2020 16:23:45 -0700
Subject: [PATCH 058/555] adapt.inp value of nsons should be nz when fixed or
 zero if variable in which case on-part values are computed. This still needs
 to be fixed in PHASTA or here for variable nsons

---
 phasta/phOutput.cc | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/phasta/phOutput.cc b/phasta/phOutput.cc
index f6fd39cd8..50702e6b1 100644
--- a/phasta/phOutput.cc
+++ b/phasta/phOutput.cc
@@ -958,8 +958,8 @@ static void getSpanwiseAverageArrays(Input& in, Output& o) {
     o.arrays.nsonsArr = new int[nfather]; //initialize nsonsArr
     for (int i=0; i<nfather; i++) { // fill nsonsArr
       /* set each entry in nsonsArr[nfather] to equal nsons */
-      o.arrays.nsonsArr[i] = 0; // not alwasy structured nsons; // this is point2nsons(nfath) in PHASTA
-                           // -1 if father is not counted in sons .... I think it is so leaving it at 0
+      o.arrays.nsonsArr[i] = nsons; // if structured set nsons in adapt.inp and use,
+                                    // if unstructured input a zero and compute below
     }
     apf::MeshEntity* v;
     apf::MeshIterator* it = m->begin(0);
@@ -975,9 +975,7 @@ static void getSpanwiseAverageArrays(Input& in, Output& o) {
     while ((v = m->iterate(it))) { // loop over mesh vertices
       m->getIntTag(v,t,&tagNum);
       o.arrays.ifather[count] = tagNum;
-//      o.arrays.nsonsArr[tagNum] =o.arrays.nsonArr[tagNum]+1;  // increment the nsons counter 
-//      o.arrays.nsonsArr[tagNum]+=1;  // increment the nsons counter 
-      ++o.arrays.nsonsArr[tagNum];  // increment the nsons counter 
+      if(nsons==0) ++o.arrays.nsonsArr[tagNum];  // increment the nsons counter 
       //std::cout<<"Tag number "<<tagNum<<std::endl;
       count++;
     }

From b3f51ff69ee747dc69138ad5fdea1bda23fbcb22 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Fri, 7 Feb 2020 16:26:29 -0700
Subject: [PATCH 059/555] slight improvement to comment

---
 phasta/phOutput.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/phasta/phOutput.cc b/phasta/phOutput.cc
index 50702e6b1..40654a4ca 100644
--- a/phasta/phOutput.cc
+++ b/phasta/phOutput.cc
@@ -958,8 +958,8 @@ static void getSpanwiseAverageArrays(Input& in, Output& o) {
     o.arrays.nsonsArr = new int[nfather]; //initialize nsonsArr
     for (int i=0; i<nfather; i++) { // fill nsonsArr
       /* set each entry in nsonsArr[nfather] to equal nsons */
-      o.arrays.nsonsArr[i] = nsons; // if structured set nsons in adapt.inp and use,
-                                    // if unstructured input a zero and compute below
+      o.arrays.nsonsArr[i] = nsons; // if fixed nz set nsons in adapt.inp and use,
+                                    // if variable nz input a zero and compute below
     }
     apf::MeshEntity* v;
     apf::MeshIterator* it = m->begin(0);

From 11807b1e4e90074cf60f331c0dde50b6d8789f76 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Sun, 23 Feb 2020 16:19:45 +0000
Subject: [PATCH 060/555] rough hack in that writes coords of each part to a
 flat text file that matches our current interpolate initial condition code. 
 Drops them right in the top directory which is not so great.  Longer term it
 would be good to either bring the interpolation code into Chef or at least
 have it read in posix so that existing chef output can be used

---
 apf/apfConvertTags.h |  2 +-
 phasta/phGeomBC.cc   | 29 ++++++++++++++++++++++++++++-
 phasta/phOutput.cc   |  6 ++++--
 3 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/apf/apfConvertTags.h b/apf/apfConvertTags.h
index 6629dcc36..d36bd1d23 100644
--- a/apf/apfConvertTags.h
+++ b/apf/apfConvertTags.h
@@ -18,7 +18,7 @@ namespace {
   template <class T> inline
   apf::MeshTag* createTag(apf::Mesh*,
       const char*, const int) {
-    exit(EXIT_FAILURE);
+  //  exit(EXIT_FAILURE);
   }
   
   template <> inline
diff --git a/phasta/phGeomBC.cc b/phasta/phGeomBC.cc
index 3eefef4f2..eec37316f 100644
--- a/phasta/phGeomBC.cc
+++ b/phasta/phGeomBC.cc
@@ -17,6 +17,16 @@ static std::string buildGeomBCFileName(std::string timestep_or_dat)
   return ss.str();
 }
 
+/* abandoned 
+static std::string buildCoordsFileName()
+{
+  std::stringstream ss;
+  int rank = PCU_Comm_Self() + 1;
+  ss << "coords."  << "." << rank;
+  return ss.str();
+}
+*/
+
 enum {
   MAX_PARAMS = 12
 };
@@ -350,11 +360,28 @@ void writeGeomBC(Output& o, std::string path, int timestep)
     ph_write_doubles(f, "m2g parametric coordinate", o.arrays.m2gParCoord,
       params[0] * params[1], 2, params);
   }
-
   params[0] = m->count(0);
   params[1] = 3;
   ph_write_doubles(f, "co-ordinates", o.arrays.coordinates,
       params[0] * params[1], 2, params);
+// start effort to write coords to a flat ascii file for each part
+  int npts=params[0];
+  char coordfilename[64];
+  bzero((void*)coordfilename,64);
+  int rank = PCU_Comm_Self() + 1;
+  sprintf(coordfilename, "coords.%d",rank);
+  FILE* fc = fopen(coordfilename, "w");
+  fprintf ( fc, "%d \n", npts );
+  double x,y,z;
+      
+  for  (int j = 0; j < npts; j++) {
+     x=o.arrays.coordinates[j];
+     y=o.arrays.coordinates[j+npts];
+     z=o.arrays.coordinates[j+2*npts];
+    fprintf ( fc, "%.15E,%.15E,%.15E,\n", x,y,z);
+  }
+  fclose(fc);
+
   writeInt(f, "number of processors", PCU_Comm_Peers());
   writeInt(f, "size of ilwork array", o.nlwork);
   if (o.nlwork)
diff --git a/phasta/phOutput.cc b/phasta/phOutput.cc
index 40654a4ca..f3b238183 100644
--- a/phasta/phOutput.cc
+++ b/phasta/phOutput.cc
@@ -966,9 +966,11 @@ static void getSpanwiseAverageArrays(Input& in, Output& o) {
     o.arrays.ifather = new int[nnodes]; //initialize ifath
     apf::MeshTag* t = m->findTag("fathers2D");
     if (t==NULL) {
-        std::cout<<"Did not find tag fathers2D"<<std::endl;
+      if (!PCU_Comm_Self())
+       lion_oprint(1,"Did not find tag fathers2D\n");
     } else if (t != NULL) {
-        std::cout<<"Found tag fathers2D"<<std::endl;
+      if (!PCU_Comm_Self())
+       lion_oprint(1,"Found tag fathers2D\n");
     }
     int tagNum; 
     int count = 0;

From c386a9a89103d5eca63f8a5171c92390c9affe20 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Mon, 23 Mar 2020 20:26:51 -0400
Subject: [PATCH 061/555] fix compile errors under gcc 9.2

---
 apf/apfConvertTags.h | 3 ++-
 ma/maSnap.cc         | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/apf/apfConvertTags.h b/apf/apfConvertTags.h
index d36bd1d23..3d5a0e1ea 100644
--- a/apf/apfConvertTags.h
+++ b/apf/apfConvertTags.h
@@ -18,7 +18,8 @@ namespace {
   template <class T> inline
   apf::MeshTag* createTag(apf::Mesh*,
       const char*, const int) {
-  //  exit(EXIT_FAILURE);
+    exit(EXIT_FAILURE);
+    return 0;
   }
   
   template <> inline
diff --git a/ma/maSnap.cc b/ma/maSnap.cc
index a5a8252ff..2e62caad2 100644
--- a/ma/maSnap.cc
+++ b/ma/maSnap.cc
@@ -56,7 +56,7 @@ static size_t isSurfUnderlyingFaceDegenerate(
     (range[periodicAxes][0] + range[periodicAxes][1]) / 2.0;
   for (int i = 0; i < 2; i++) {
     double candidateDegenParam = range[degenAxes][i];
-    double param[2];
+    double param[3] = {0,0,0};
     Vector uTan;
     Vector vTan;
     param[periodicAxes] = candidatePeriodicParam;

From b296e6263041d1622b57b821ef80b2b5c1cec124 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Mon, 23 Mar 2020 20:28:06 -0400
Subject: [PATCH 062/555] makeMdsBox was hijacked for building tensor meshes

---
 test/CMakeLists.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 46a359723..3f9d94eaf 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -32,7 +32,6 @@ util_exe_func(mktopomodel mktopomodel.cc)
 util_exe_func(modelInfo modelInfo.cc)
 
 # Mesh generation utilities
-util_exe_func(box box.cc)
 util_exe_func(uniform uniform.cc)
 util_exe_func(refine2x refine2x.cc)
 util_exe_func(serialize serialize.cc)

From 54bba5cada7d70324e724fc7cbb350312f4863fa Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Tue, 31 Mar 2020 08:20:10 -0600
Subject: [PATCH 063/555] yet another corner case----literally

---
 test/matchedNodeElmReader.cc | 36 ++++++++++++++++++++++++++----------
 1 file changed, 26 insertions(+), 10 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index e4694829d..0061873cc 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -326,19 +326,20 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
 */
 
   apf::Adjacent verts;
-/*
+
   double distFromDebug1, distFromDebug2;
-  apf::Vector3 xd1(-0.54864, 7.44015e-06, 0.0397148 );
+//  apf::Vector3 xd1(-0.54864, 7.44015e-06, 0.0397148 );
+  apf::Vector3 xd1(-0.306845, 0.443585, 0.0291);
   apf::Vector3 xd2(0.914478, 0.0145401, 0.04 );
   apf::Vector3 dx1;
   apf::Vector3 dx2;
   apf::Vector3 tmp;
   apf::Vector3 Centroid;
-  */
+
   while( (f = mesh->iterate(it)) ) {
     mesh->getAdjacent(f, 0, verts);
     size_t nverts = verts.size();
-/* 
+ 
     Centroid=apf::getLinearCentroid(mesh,f);
     dx1=xd1-Centroid;
     dx2=xd2-Centroid;
@@ -348,12 +349,13 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
     distFromDebug2=dx2[0]*dx2[0]
                   +dx2[1]*dx2[1]
                   +dx2[2]*dx2[2];
-  */
+
     int cmin=100;
     int cmax=-100;
     int cmid=-100;
     int ctri[4];  // up to 4 points on a face
     int f1, f2, f1x, f2x, f1d, f2d;
+    int emax,emin,F0max,F0min,F1max,F1min;
     for(size_t i=0; i<verts.size(); i++) {
       mesh->getIntTag(verts[i],vtxClass,&c);
       cmin=std::min(cmin,c);
@@ -377,15 +379,15 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
         if((ctri[i] != cmin) && (ctri[i] != cmax)) cmid=std::max(cmid,ctri[i]);  // max is to catch lowest dim/highest code for quads....actually not necessary since either of the other two will follow switches as noted below.
       }
     }
-/* 
+
     if(std::min(distFromDebug1,distFromDebug2) < 1e-12) {
          fprintf(stderr, "%d %d %d %.15e %.15E %.15E \n", cmin, cmid, cmax, Centroid[0], Centroid[1], Centroid[2]);
          for (size_t i=0; i < nverts; i++) {
             mesh->getPoint(verts[i],0,tmp);
-            fprintf(stderr, "%d %.15e %.15E %.15E \n", i , tmp[0], tmp[1], tmp[2]);
+//            fprintf(stderr, "%d %.15e %.15E %.15E \n", i , tmp[0], tmp[1], tmp[2]);
          }
     }
-     */
+     
     if (cmin == INTERIORTAG) {  // no brainer since a point on the interior always classifies interior
        mesh->setModelEntity(f,getMdlRgn(model));
        //cint++;
@@ -424,8 +426,22 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
                      mesh->setModelEntity(f,getMdlFace(mesh,TofM[cmin-1]));  
                   } else mesh->setModelEntity(f,getMdlRgn(model)); // at least one of the edges not in face closure so clssify on region
          }
-    } else if (cmax <= EDGE_LAST){  // since none on face (3, 2, and 1 done above) and none on verts, 3 edges-> interior 
-        mesh->setModelEntity(f,getMdlRgn(model));
+    } else if (cmax <= EDGE_LAST){ 
+         if( cmax==cmid || cmin== cmid) { // two on same edge one on other so this is on a face--  find which one
+                emax=cmax-11;
+                emin=cmin-11;
+   		F0max=FofE[emax][0];
+ 		F0min=FofE[emin][0];
+  		F1max=FofE[emax][1];
+  		F1min=FofE[emin][1];
+                if(F0max==F0min) mesh->setModelEntity(f,getMdlFace(mesh,F0max)); // don't need map TofM[f1d-1]));
+                if(F0max==F1min) mesh->setModelEntity(f,getMdlFace(mesh,F0max)); 
+                if(F1max==F0min) mesh->setModelEntity(f,getMdlFace(mesh,F1max)); 
+                if(F1max==F1min) mesh->setModelEntity(f,getMdlFace(mesh,F1max)); 
+         } else  {// catch all now correct?? was not before
+ // since none on face (3, 2, and 1 done above) and none on verts, 3 edges-> interior 
+               mesh->setModelEntity(f,getMdlRgn(model));
+         }
     } else { // getting here only if none of the face verts are interior or on model face so must be 2 e and 1 v but
         f1x=1+FofE[(cmin-11)][0]; // first two find 2 face usages of min edge
         f2x=1+FofE[(cmin-11)][1];

From 51ca64f52bfa7934d0498bba558725487d819d82 Mon Sep 17 00:00:00 2001
From: Gerrett Diamond <diamog@rpi.edu>
Date: Tue, 28 Apr 2020 17:13:21 -0400
Subject: [PATCH 064/555] Add utility to print a graph partition for pumipic

---
 test/CMakeLists.txt             |  5 +-
 test/print_pumipic_partition.cc | 85 +++++++++++++++++++++++++++++++++
 test/testing.cmake              | 10 +++-
 3 files changed, 98 insertions(+), 2 deletions(-)
 create mode 100644 test/print_pumipic_partition.cc

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index a19e4fd56..a16d5382c 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -109,7 +109,7 @@ util_exe_func(reorder reorder.cc)
 util_exe_func(fixshape fixshape.cc)
 util_exe_func(fixlayer fixlayer.cc)
 util_exe_func(fixDisconnected fixDisconnected.cc)
-util_exe_func(scale scale.cc) 
+util_exe_func(scale scale.cc)
 
 # Phasta utilities
 util_exe_func(chef ../phasta/chef.cc)
@@ -129,6 +129,9 @@ if(SIMMODSUITE_SimAdvMeshing_FOUND)
   target_compile_definitions(cut_interface PRIVATE -DHAVE_SIMADVMESHING)
 endif()
 
+#PUMIPic print partition utility
+util_exe_func(print_pumipic_partition print_pumipic_partition.cc)
+
 # Unit tests / functionality regression tests
 test_exe_func(qr qr.cc)
 test_exe_func(eigen_test eigen_test.cc)
diff --git a/test/print_pumipic_partition.cc b/test/print_pumipic_partition.cc
new file mode 100644
index 000000000..3a6be4bc8
--- /dev/null
+++ b/test/print_pumipic_partition.cc
@@ -0,0 +1,85 @@
+#include <gmi_mesh.h>
+#include <apf.h>
+#include <apfMesh2.h>
+#include <apfMDS.h>
+#include <PCU.h>
+#include <lionPrint.h>
+#include <parma.h>
+
+#ifdef HAVE_SIMMETRIX
+#include <gmi_sim.h>
+#include <SimUtil.h>
+#include <MeshSim.h>
+#include <SimModel.h>
+#endif
+#include <apfZoltan.h>
+#include <pcu_util.h>
+#include <cstdlib>
+#include <fstream>
+
+int main(int argc, char** argv)
+{
+  MPI_Init(&argc,&argv);
+  PCU_Comm_Init();
+  lion_set_verbosity(1);
+  if ( argc != 5 && argc != 6) {
+    if ( !PCU_Comm_Self() )
+      printf("Usage: %s <model> <mesh> <number of output parts> <partition file prefix>\n", argv[0]);
+    MPI_Finalize();
+    exit(EXIT_FAILURE);
+  }
+  if (PCU_Comm_Peers() > 1) {
+    if ( !PCU_Comm_Self() )
+      printf("This tool must be run in serial.\n");
+    MPI_Finalize();
+    exit(EXIT_FAILURE);
+  }
+#ifdef HAVE_SIMMETRIX
+  MS_init();
+  SimModel_start();
+  Sim_readLicenseFile(NULL);
+  gmi_sim_start();
+  gmi_register_sim();
+#endif
+  gmi_register_mesh();
+
+
+  apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]);
+
+  int num_ranks = atoi(argv[3]);
+  //Partition the mesh (Taken from zsplit.cc)
+  apf::Splitter* splitter = apf::makeZoltanSplitter(
+      m, apf::GRAPH, apf::PARTITION, false);
+  apf::MeshTag* weights = Parma_WeighByMemory(m);
+  apf::Migration* plan = splitter->split(weights, 1.05, num_ranks);
+  apf::removeTagFromDimension(m, weights, m->getDimension());
+  m->destroyTag(weights);
+  delete splitter;
+
+  //Write the partition out
+  char filename[256];
+  sprintf(filename , "%s_%d.ptn", argv[4], num_ranks);
+  printf("Writing partition to %s\n", filename);
+  std::ofstream out(filename);
+  apf::MeshIterator* mitr = m->begin(m->getDimension());
+  while (apf::MeshEntity* ent = m->iterate(mitr)) {
+    if (plan->has(ent))
+      out << plan->sending(ent) << '\n';
+    else
+      out << 0 << '\n';
+  }
+  m->end(mitr);
+
+  delete plan;
+
+  m->destroyNative();
+  apf::destroyMesh(m);
+#ifdef HAVE_SIMMETRIX
+  gmi_sim_stop();
+  Sim_unregisterAllKeys();
+  SimModel_stop();
+  MS_exit();
+#endif
+  PCU_Comm_Free();
+  MPI_Finalize();
+}
diff --git a/test/testing.cmake b/test/testing.cmake
index e0d368fee..56d4f9770 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -28,6 +28,14 @@ mpi_test(bezierSubdivision 1 ./bezierSubdivision)
 mpi_test(bezierValidity 1 ./bezierValidity)
 mpi_test(ma_analytic 1 ./ma_test_analytic_model)
 
+mpi_test(print_pumipic_partion 1
+         ./print_pumipic_partition
+         ${MESHES}/cube/cube.dmg
+         ${MESHES}/cube/pumi11/cube.smb
+         4
+         pumipic_cube
+         )
+
 mpi_test(align 1 ./align)
 mpi_test(eigen_test 1 ./eigen_test)
 mpi_test(integrate 1 ./integrate)
@@ -486,7 +494,7 @@ mpi_test(mixedNumbering 4
   out)
 set(MDIR ${MESHES}/square)
 mpi_test(hierarchic_2p_2D 1
-  ./hierarchic 
+  ./hierarchic
   "${MDIR}/square.dmg"
   "${MDIR}/square.smb"
   2)

From 6367f8dad6d1648fc008532f4c233e51dbaf0206 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Thu, 13 Feb 2020 14:20:26 -0500
Subject: [PATCH 065/555] Initial commit for Nedelec Shapes

---
 apf/CMakeLists.txt |  1 +
 apf/apf.cc         | 12 +++++++
 apf/apf.h          | 13 ++++++++
 apf/apfElement.cc  |  8 +++++
 apf/apfElement.h   |  2 ++
 apf/apfNedelec.cc  | 81 ++++++++++++++++++++++++++++++++++++++++++++++
 apf/apfShape.h     |  5 +++
 7 files changed, 122 insertions(+)
 create mode 100644 apf/apfNedelec.cc

diff --git a/apf/CMakeLists.txt b/apf/CMakeLists.txt
index 881a9fe65..2d0fa0276 100644
--- a/apf/CMakeLists.txt
+++ b/apf/CMakeLists.txt
@@ -25,6 +25,7 @@ set(SOURCES
   apfShape.cc
   apfIPShape.cc
   apfHierarchic.cc
+  apfNedelec.cc
   apfVector.cc
   apfVectorElement.cc
   apfVectorField.cc
diff --git a/apf/apf.cc b/apf/apf.cc
index ceefdf53f..c6d2476a1 100644
--- a/apf/apf.cc
+++ b/apf/apf.cc
@@ -410,6 +410,18 @@ void getShapeGrads(Element* e, Vector3 const& local,
   e->getGlobalGradients(local,grads);
 }
 
+void getVectorShapeValues(Element* e, Vector3 const& local,
+    NewArray<Vector3>& values)
+{
+  e->getShape()->getVectorValues(e->getMesh(), e->getEntity(), local,values);
+}
+
+void getVectorShapeGrads(Element* e, Vector3 const& local,
+    NewArray<Matrix3x3>& grads)
+{
+  e->getGlobalVectorGradients(local,grads);
+}
+
 FieldShape* getShape(Field* f)
 {
   return f->getShape();
diff --git a/apf/apf.h b/apf/apf.h
index 1ab3ee30f..410d1bf2e 100644
--- a/apf/apf.h
+++ b/apf/apf.h
@@ -603,6 +603,19 @@ void getShapeValues(Element* e, Vector3 const& local,
 void getShapeGrads(Element* e, Vector3 const& local,
     NewArray<Vector3>& grads);
 
+/** \brief Returns the vector shape function values at a point
+ *  \details used only for Nedelec shapes
+  */
+void getVectorShapeValues(Element* e, Vector3 const& local,
+    NewArray<Vector3>& values);
+
+/** \brief Returns the vector shape function gradients at a point
+  *
+  * \details used only for Nedelec shapes
+  */
+void getVectorShapeGrads(Element* e, Vector3 const& local,
+    NewArray<Matrix3x3>& grads);
+
 
 /** \brief Retrieve the apf::FieldShape used by a field
   */
diff --git a/apf/apfElement.cc b/apf/apfElement.cc
index 82c89c602..731ace661 100644
--- a/apf/apfElement.cc
+++ b/apf/apfElement.cc
@@ -85,6 +85,14 @@ void Element::getGlobalGradients(Vector3 const& local,
     globalGradients[i] = jinv * localGradients[i];
 }
 
+void Element::getGlobalVectorGradients(Vector3 const& local,
+                                 NewArray<Matrix3x3>& globalGradients)
+{
+  // TODO
+  // the same as above but for vector shape functions
+  // using getLocalVectorGradients in apfNedelec.cc
+}
+
 void Element::getComponents(Vector3 const& xi, double* c)
 {
   NewArray<double> shapeValues;
diff --git a/apf/apfElement.h b/apf/apfElement.h
index f100d42e8..953fc08f8 100644
--- a/apf/apfElement.h
+++ b/apf/apfElement.h
@@ -25,6 +25,8 @@ class Element
     virtual ~Element();
     void getGlobalGradients(Vector3 const& local,
                             NewArray<Vector3>& globalGradients);
+    void getGlobalVectorGradients(Vector3 const& local,
+                            NewArray<Matrix3x3>& globalGradients);
     int getType() {return mesh->getType(entity);}
     int getDimension() {return Mesh::typeDimension[getType()];}
     int getOrder() {return field->getShape()->getOrder();}
diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
new file mode 100644
index 000000000..9c9072320
--- /dev/null
+++ b/apf/apfNedelec.cc
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2011 Scientific Computation Research Center
+ *
+ * This work is open source software, licensed under the terms of the
+ * BSD license as described in the LICENSE file in the top-level directory.
+ */
+
+#include "apfShape.h"
+#include "apfMesh.h"
+#include "apfFieldOf.h"
+#include "apfElement.h"
+#include "apfVectorElement.h"
+#include <pcu_util.h>
+
+#include <iostream>
+
+namespace apf {
+
+class Nedelec: public FieldShape {
+  public:
+    const char* getName() const { return "Nedelec"; }
+    Nedelec(int order) : P(order) {
+      // TODO
+      ;
+    }
+    class Vertex : public apf::EntityShape
+    {
+    public:
+      void getValues(apf::Mesh*, apf::MeshEntity*,
+	  apf::Vector3 const&, apf::NewArray<double>& values) const
+      {
+	(void)values;
+	// TODO inform the user that this is not implemented and abort()
+      }
+      void getLocalGradients(apf::Mesh*, apf::MeshEntity*,
+	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
+      {
+      }
+      int countNodes() const {return 0;}
+      void alignSharedNodes(apf::Mesh*,
+	  apf::MeshEntity*, apf::MeshEntity*, int order[])
+      {
+	(void)order;
+      }
+    };
+    class Edge : public apf::EntityShape
+    {
+    public:
+      void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const&, apf::NewArray<double>&) const
+      {
+	// TODO inform the user that this is not implemented and abort()
+      }
+      void getLocalGradients(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
+      {
+	// TODO inform the user that this is not implemented and abort()
+      }
+      int countNodes() const {return 0; // TODO update this}
+      void alignSharedNodes(apf::Mesh*,
+	  apf::MeshEntity*, apf::MeshEntity*, int order[])
+      {
+	(void)order;
+      }
+      void getVectorValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
+      {
+      	// TODO: to be completed
+      }
+      void getLocalVectorGradients(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const&, apf::NewArray<apf::Matrix3x3>&) const
+      {
+      	// TODO: to be completed
+      }
+    };
+    int getOrder() {return P;}
+  private:
+    int P;
+};
+
+}
diff --git a/apf/apfShape.h b/apf/apfShape.h
index 60308454d..da907a6d7 100644
--- a/apf/apfShape.h
+++ b/apf/apfShape.h
@@ -136,6 +136,11 @@ FieldShape* getIPFitShape(int dimension, int order);
  */
 FieldShape* getHierarchic(int order);
 
+/** \brief Get the Nedelec shape function of a given order
+  \details TODO: complete later
+ */
+FieldShape* getNedelec(int order);
+
 /** \brief Project a hierarchic field */
 void projectHierarchicField(Field* to, Field* from);
 

From 642e0762f1480665cb6221834d250ba41fd0563f Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Fri, 20 Mar 2020 12:04:08 -0400
Subject: [PATCH 066/555] Adds TODOs to apfNedelec.cc to be completed

---
 apf/apfNedelec.cc | 103 +++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 101 insertions(+), 2 deletions(-)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index 9c9072320..3fa4b9139 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -10,18 +10,33 @@
 #include "apfFieldOf.h"
 #include "apfElement.h"
 #include "apfVectorElement.h"
+#include "mth.h"
 #include <pcu_util.h>
 
 #include <iostream>
 
 namespace apf {
 
+enum {
+  GAUSS_LOBATTO;
+  GAUSS_LEGANDRE;
+};
+
+static const double* getOpenPoints(int order, int type = GAUSS_LOBATTO){
+  // TODO implement open points
+  ;
+}
+
+static void getChebychevT(int order, double xi, double* u)
+{
+  // TODO implement Chebyshev
+  ;
+}
+
 class Nedelec: public FieldShape {
   public:
     const char* getName() const { return "Nedelec"; }
     Nedelec(int order) : P(order) {
-      // TODO
-      ;
     }
     class Vertex : public apf::EntityShape
     {
@@ -73,6 +88,90 @@ class Nedelec: public FieldShape {
       	// TODO: to be completed
       }
     };
+    class Triangle : public apf::EntityShape
+    {
+    public:
+      void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const&, apf::NewArray<double>&) const
+      {
+	// TODO inform the user that this is not implemented and abort()
+      }
+      void getLocalGradients(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
+      {
+	// TODO inform the user that this is not implemented and abort()
+      }
+      int countNodes() const {return 0; // TODO update this}
+      void alignSharedNodes(apf::Mesh*,
+	  apf::MeshEntity*, apf::MeshEntity*, int order[])
+      {
+	(void)order;
+      }
+      void getVectorValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
+      {
+      	// TODO: to be completed
+      }
+      void getLocalVectorGradients(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const&, apf::NewArray<apf::Matrix3x3>&) const
+      {
+      	// TODO: to be completed
+      }
+    };
+    class Tetrahedron : public apf::EntityShape
+    {
+    public:
+      void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const&, apf::NewArray<double>&) const
+      {
+	// TODO inform the user that this is not implemented and abort()
+      }
+      void getLocalGradients(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
+      {
+	// TODO inform the user that this is not implemented and abort()
+      }
+      int countNodes() const {return 0; // TODO update this}
+      void alignSharedNodes(apf::Mesh*,
+	  apf::MeshEntity*, apf::MeshEntity*, int order[])
+      {
+	(void)order;
+      }
+      void getVectorValues(apf::Mesh* m, apf::MeshEntity* e,
+	  apf::Vector3 const& xi, apf::NewArray<apf::Vector3>& u) const
+      {
+      	computeTi();
+      	// TODO: compute shapes
+      }
+      void getLocalVectorGradients(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const&, apf::NewArray<apf::Matrix3x3>&) const
+      {
+      	// TODO: to be completed
+      }
+    private:
+      mth::Matrix<double> Ti;
+      const double tk[18] = // TODO: edge directions
+      {};
+      void computeTi()
+      {
+	int non = ;
+	apf::DynamicArray<apf::Vector3> nodes;
+	apf::DynamicArray<int> dof2tk;
+	nodes.setSize(non);
+	dof2tk.setSize(non);
+
+	// Edge loops to get nodes and dof2tk for edges
+	// Face loops to get nodes and dof2tk for faces
+	// Retion loops to get nodes and dof2tk for regions
+
+	mth::Matrix<double> T(non,non); // T(i,j)
+	// Populate T
+
+
+      }
+    };
+
+
     int getOrder() {return P;}
   private:
     int P;

From c80c22fde7a5babecc3c00be6b00657ba4c7bd12 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Mon, 23 Mar 2020 13:51:46 -0400
Subject: [PATCH 067/555] Implementation of Nedelec shapes for Tet and Tri
 (excluding curlshape).

---
 apf/apfNedelec.cc | 441 ++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 409 insertions(+), 32 deletions(-)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index 3fa4b9139..e97e4c4e8 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -12,25 +12,120 @@
 #include "apfVectorElement.h"
 #include "mth.h"
 #include <pcu_util.h>
+#include <mthQR.h>
 
 #include <iostream>
 
 namespace apf {
 
 enum {
-  GAUSS_LOBATTO;
-  GAUSS_LEGANDRE;
+  GAUSS_LEGENDRE,
+  GAUSS_LOBATTO
 };
 
-static const double* getOpenPoints(int order, int type = GAUSS_LOBATTO){
-  // TODO implement open points
-  ;
+static void getGaussLegendrePoints(int np, double* pts)
+{
+  switch (np)
+  {
+    case 1:
+      pts[0] = 0.5;
+      return;
+    case 2:
+      pts[0] = 0.21132486540518711775;
+      pts[1] = 0.78867513459481288225;
+      return;
+    case 3:
+      pts[0] = 0.11270166537925831148;
+      pts[1] = 0.5;
+      pts[2] = 0.88729833462074168852;
+      return;
+  }
+
+  const int n = np;
+  const int m = (n+1)/2;
+
+  for (int i = 1; i <= m; i++)
+  {
+     double z = cos(M_PI * (i - 0.25) / (n + 0.5));
+     double pp, p1, dz, xi = 0.;
+     bool done = false;
+     while (1)
+     {
+        double p2 = 1;
+        p1 = z;
+        for (int j = 2; j <= n; j++)
+        {
+           double p3 = p2;
+           p2 = p1;
+           p1 = ((2 * j - 1) * z * p2 - (j - 1) * p3) / j;
+        }
+        // p1 is Legendre polynomial
+        pp = n * (z*p1-p2) / (z*z - 1);
+        if (done) { break; }
+        dz = p1/pp;
+        if (fabs(dz) < 1e-16)
+        {
+           done = true;
+           // map the new point (z-dz) to (0,1):
+           xi = ((1 - z) + dz)/2; // (1 - (z - dz))/2 has bad round-off
+           // continue the computation: get pp at the new point, then exit
+        }
+        // update: z = z - dz
+        z -= dz;
+     }
+     pts[i-1] = xi;
+     pts[n-i] = 1 - xi;
+  }
+}
+
+static void getGaussLobattoPoints(int /*np*/, double* /*pts*/)
+{ /* TODO implement Gauss Lobatto points. Later when needed. */
+};
+
+static const double* getPoints(int order, const int type)
+{
+  int np = order + 1;
+  double* points = new double[np];
+  switch (type)
+  {
+    case GAUSS_LEGENDRE:
+    {
+      getGaussLegendrePoints(np, points);
+      break;
+    }
+    case GAUSS_LOBATTO:
+    {
+        getGaussLobattoPoints(np, points);
+      break;
+    }
+  }
+  return points;
 }
 
-static void getChebychevT(int order, double xi, double* u)
+static const double* getOpenPoints(int order, const int type = GAUSS_LEGENDRE)
+{
+  return getPoints(order, type);
+}
+
+static const double* getClosedPoints(int order, const int type = GAUSS_LOBATTO)
+{
+    return getPoints(order, type);
+}
+
+static void getChebyshevT(int order, double xi, double* u)
 {
   // TODO implement Chebyshev
-  ;
+  // recursive definition, z in [-1,1]
+  // T_0(z) = 1,  T_1(z) = z
+  // T_{n+1}(z) = 2*z*T_n(z) - T_{n-1}(z)
+  double z;
+  u[0] = 1.;
+  if (order == 0) { return; }
+  u[1] = z = 2.*xi - 1.;
+  for (int n = 1; n < order; n++)
+  {
+     u[n+1] = 2*z*u[n] - u[n-1];
+  }
 }
 
 class Nedelec: public FieldShape {
@@ -71,7 +166,7 @@ class Nedelec: public FieldShape {
       {
 	// TODO inform the user that this is not implemented and abort()
       }
-      int countNodes() const {return 0; // TODO update this}
+      int countNodes() const {return 0; /* TODO update this*/}
       void alignSharedNodes(apf::Mesh*,
 	  apf::MeshEntity*, apf::MeshEntity*, int order[])
       {
@@ -101,22 +196,136 @@ class Nedelec: public FieldShape {
       {
 	// TODO inform the user that this is not implemented and abort()
       }
-      int countNodes() const {return 0; // TODO update this}
+      int countNodes() const {return P*(P+2);}
       void alignSharedNodes(apf::Mesh*,
 	  apf::MeshEntity*, apf::MeshEntity*, int order[])
       {
 	(void)order;
       }
       void getVectorValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
-	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
+	  apf::Vector3 const& xi, apf::NewArray<apf::Vector3>&) const
       {
-      	// TODO: to be completed
+      	const int pm1 = P - 1;
+        const int p = P;
+
+        double* shape_x = new double[p];
+        double* shape_y = new double[p];
+        double* shape_l = new double[p];
+
+        int dof = countNodes();
+        mth::Matrix<double> u(dof, dim);
+
+        double x = xi[0]; double y = xi[1];
+
+        getChebyshevT(pm1, x, shape_x);
+        getChebyshevT(pm1, y, shape_y);
+        getChebyshevT(pm1, 1. - x - y, shape_l);
+
+        int n = 0;
+        for (int j = 0; j <= pm1; j++)
+          for (int i = 0; i + j <= pm1; i++)
+          {
+            double s = shape_x[i]*shape_y[j]*shape_l[pm1-i-j];
+            u(n,0) = s;  u(n,1) = 0.;  n++;
+            u(n,0) = 0.;  u(n,1) = s;  n++;
+          }
+        for (int j = 0; j <= pm1; j++)
+        {
+          double s = shape_x[pm1-j]*shape_y[j];
+          u(n,0) = s*(y - c);  u(n,1) = -s*(x - c); n++;
+        }
+
+      	computeTi(); // TODO multiply u and QR in a for loop to compute vector shapes
       }
       void getLocalVectorGradients(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<apf::Matrix3x3>&) const
       {
       	// TODO: to be completed
       }
+    private:
+      //mth::Matrix<double> Ti;
+      mth::Matrix<double> Q;
+      mth::Matrix<double> R;
+      const double tk[8] = {1.,0.,  -1.,1.,  0.,-1.,  0.,1.}; // TODO
+      const double c = 1./3.;
+      const int dim = 2; // reference space dim
+      void computeTi()
+      {
+     	  int non = countNodes();
+
+        const double *eop = getOpenPoints(P - 1);
+        const double *iop = (P > 1) ? getOpenPoints(P - 2) : NULL;
+
+        const int pm1 = P - 1, pm2 = P - 2;
+
+        double* shape_x = new double[P];
+        double* shape_y = new double[P];
+        double* shape_l = new double[P];
+
+       	apf::DynamicArray<apf::Vector<2> > nodes (non); // TODO syntax question
+      	apf::DynamicArray<int> dof2tk (non);
+
+        int o = 0;
+      	// Edge loops to get nodes and dof2tk for edges
+        for (int i = 0; i < P; i++)  // (0,1)
+        {
+          nodes[o][0] = eop[i];  nodes[o][1] = 0.;
+          dof2tk[o++] = 0;
+        }
+        for (int i = 0; i < P; i++)  // (1,2)
+        {
+          nodes[o][0] = eop[pm1-i];  nodes[o][1] = eop[i];
+          dof2tk[o++] = 1;
+        }
+        for (int i = 0; i < P; i++)  // (2,0)
+        {
+          nodes[o][0] = 0.;  nodes[o][1] = eop[pm1-i];
+          dof2tk[o++] = 2;
+        }
+
+      	// Region loops to get nodes and dof2tk for regions
+        for (int j = 0; j <= pm2; j++)
+          for (int i = 0; i + j <= pm2; i++)
+              {
+                double w = iop[i] + iop[j] + iop[pm2-i-j];
+                nodes[o][0] = iop[i]/w;  nodes[o][1] = iop[j]/w;
+                dof2tk[o++] = 0;
+                nodes[o][0] = iop[i]/w;  nodes[o][1] = iop[j]/w;
+                dof2tk[o++] = 3;
+              }
+
+      	// Populate T
+      	mth::Matrix<double> T(non,non); // T(i,j)
+        for (int m = 0; m < non; m++)
+        {
+          const double *tm = tk + 2*dof2tk[m];
+          o = 0;
+
+          double x = nodes[o][0]; double y = nodes[o][1];
+
+          getChebyshevT(pm1, x, shape_x);
+          getChebyshevT(pm1, y, shape_y);
+          getChebyshevT(pm1, 1. - x - y, shape_l);
+
+          for (int j = 0; j <= pm1; j++)
+            for (int i = 0; i + j <= pm1; i++)
+            {
+              double s = shape_x[i]*shape_y[j]*shape_l[pm1-i-j];
+              T(o++, m) = s * tm[0];
+              T(o++, m) = s * tm[1];
+            }
+          for (int j = 0; j <= pm1; j++)
+          {
+            T(o++, m) =
+              shape_x[pm1-j]*shape_y[j]*((y - c)*tm[0] - (x - c)*tm[1]);
+          }
+        }
+
+        unsigned rank = mth::decomposeQR(T, Q, R);
+      }
+      int getOrder() {return P;}
+    private:
+      int P;
     };
     class Tetrahedron : public apf::EntityShape
     {
@@ -131,7 +340,7 @@ class Nedelec: public FieldShape {
       {
 	// TODO inform the user that this is not implemented and abort()
       }
-      int countNodes() const {return 0; // TODO update this}
+      int countNodes() const {return (P+3)*(P+2)*P/2;}
       void alignSharedNodes(apf::Mesh*,
 	  apf::MeshEntity*, apf::MeshEntity*, int order[])
       {
@@ -140,8 +349,48 @@ class Nedelec: public FieldShape {
       void getVectorValues(apf::Mesh* m, apf::MeshEntity* e,
 	  apf::Vector3 const& xi, apf::NewArray<apf::Vector3>& u) const
       {
-      	computeTi();
-      	// TODO: compute shapes
+        const int pm1 = P - 1;
+        const int p = P;
+
+        double* shape_x = new double[p];
+        double* shape_y = new double[p];
+        double* shape_z = new double[p];
+        double* shape_l = new double[p];
+
+        int dof = countNodes();
+        mth::Matrix<double> u(dof, dim);
+
+        double x = xi[0]; double y = xi[1]; double z = xi[2];
+
+        getChebyshevT(pm1, x, shape_x);
+        getChebyshevT(pm1, y, shape_y);
+        getChebyshevT(pm1, z, shape_z);
+        getChebyshevT(pm1, 1. - x - y - z, shape_l);
+
+        int n = 0;
+        for (int k = 0; k <= pm1; k++)
+          for (int j = 0; j + k <= pm1; j++)
+            for (int i = 0; i + j + k <= pm1; i++)
+            {
+              double s = shape_x[i]*shape_y[j]*shape_z[k]*shape_l[pm1-i-j-k];
+                    u(n,0) =  s;  u(n,1) = 0.;  u(n,2) = 0.;  n++;
+                    u(n,0) = 0.;  u(n,1) =  s;  u(n,2) = 0.;  n++;
+                    u(n,0) = 0.;  u(n,1) = 0.;  u(n,2) =  s;  n++;
+                 }
+        for (int k = 0; k <= pm1; k++)
+          for (int j = 0; j + k <= pm1; j++)
+          {
+            double s = shape_x[pm1-j-k]*shape_y[j]*shape_z[k];
+            u(n,0) = s*(y - c);  u(n,1) = -s*(x - c);  u(n,2) =  0.;  n++;
+            u(n,0) = s*(z - c);  u(n,1) =  0.;  u(n,2) = -s*(x - c);  n++;
+          }
+        for (int k = 0; k <= pm1; k++)
+        {
+          double s = shape_y[pm1-k]*shape_z[k];
+          u(n,0) = 0.;  u(n,1) = s*(z - c);  u(n,2) = -s*(y - c);  n++;
+        }
+
+      	computeTi(); // TODO multiply u and QR in a for loop to compute vector shapes
       }
       void getLocalVectorGradients(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<apf::Matrix3x3>&) const
@@ -149,32 +398,160 @@ class Nedelec: public FieldShape {
       	// TODO: to be completed
       }
     private:
-      mth::Matrix<double> Ti;
-      const double tk[18] = // TODO: edge directions
-      {};
+      //mth::Matrix<double> Ti;
+      mth::Matrix<double> Q;
+      mth::Matrix<double> R;
+      const double tk[18] = {1.,0.,0.,  -1.,1.,0.,  0.,-1.,0.,  0.,0.,1.,  -1.,0.,1.,  0.,-1.,1.}; // edge directions
+      const double c = 1./4.;
+      const int dim = 3;
       void computeTi()
       {
-	int non = ;
-	apf::DynamicArray<apf::Vector3> nodes;
-	apf::DynamicArray<int> dof2tk;
-	nodes.setSize(non);
-	dof2tk.setSize(non);
+     	  int non = countNodes();
 
-	// Edge loops to get nodes and dof2tk for edges
-	// Face loops to get nodes and dof2tk for faces
-	// Retion loops to get nodes and dof2tk for regions
+        const double *eop = getOpenPoints(P - 1);
+        const double *fop = (P > 1) ? getOpenPoints(P - 2) : NULL;
+        const double *iop = (P > 2) ? getOpenPoints(P - 3) : NULL;
 
-	mth::Matrix<double> T(non,non); // T(i,j)
-	// Populate T
+        const int pm1 = P - 1, pm2 = P - 2, pm3 = P - 3;
 
+        double* shape_x = new double[P];
+        double* shape_y = new double[P];
+        double* shape_z = new double[P];
+        double* shape_l = new double[P];
 
-      }
-    };
+       	apf::DynamicArray<apf::Vector3> nodes (non);
+      	apf::DynamicArray<int> dof2tk (non);
+	      nodes.setSize(non);
+      	dof2tk.setSize(non);
+
+        int o = 0;
+      	// Edge loops to get nodes and dof2tk for edges
+        for (int i = 0; i < P; i++)  // (0,1)
+        {
+          nodes[o][0] = eop[i];  nodes[o][1] = 0.;  nodes[o][2] = 0.;
+          dof2tk[o++] = 0;
+        }
+        for (int i = 0; i < P; i++)  // (1,2)
+        {
+          nodes[o][0] = eop[pm1-i];  nodes[o][1] = eop[i];  nodes[o][2] = 0.;
+          dof2tk[o++] = 1;
+        }
+        for (int i = 0; i < P; i++)  // (2,0)
+        {
+          nodes[o][0] = 0.;  nodes[o][1] = eop[pm1-i];  nodes[o][2] = 0.;
+          dof2tk[o++] = 2;
+        }
+        for (int i = 0; i < P; i++)  // (0,3)
+        {
+          nodes[o][0] = 0.;  nodes[o][1] = 0.;  nodes[o][2] = eop[i];
+          dof2tk[o++] = 3;
+        }
+        for (int i = 0; i < P; i++)  // (1,3)
+        {
+          nodes[o][0] = eop[pm1-i];  nodes[o][1] = 0.;  nodes[o][2] = eop[i];
+          dof2tk[o++] = 4;
+        }
+        for (int i = 0; i < P; i++)  // (2,3)
+        {
+          nodes[o][0] = 0.;  nodes[o][1] = eop[pm1-i];  nodes[o][2] = eop[i];
+          dof2tk[o++] = 5;
+        }
+
+      	// Face loops to get nodes and dof2tk for faces
+        for (int j = 0; j <= pm2; j++)  // (0,1,2)
+          for (int i = 0; i + j <= pm2; i++)
+          {
+            double w = fop[i] + fop[j] + fop[pm2-i-j];
+            nodes[o][0] = fop[i]/w;  nodes[o][1] = fop[j]/w;  nodes[o][2] = 0.;
+            dof2tk[o++] = 0;
+            nodes[o][0] = fop[i]/w;  nodes[o][1] = fop[j]/w;  nodes[o][2] = 0.;
+            dof2tk[o++] = 2;
+          }
+        for (int j = 0; j <= pm2; j++)  // (0,1,3)
+          for (int i = 0; i + j <= pm2; i++)
+          {
+            double w = fop[i] + fop[j] + fop[pm2-i-j];
+            nodes[o][0] = fop[i]/w;  nodes[o][1] = 0.;  nodes[o][2] = fop[j]/w;
+            dof2tk[o++] = 0;
+            nodes[o][0] = fop[i]/w;  nodes[o][1] = 0.;  nodes[o][2] = fop[j]/w;
+            dof2tk[o++] = 3;
+          }
+        for (int j = 0; j <= pm2; j++)  // (1,2,3)
+          for (int i = 0; i + j <= pm2; i++)
+          {
+            double w = fop[i] + fop[j] + fop[pm2-i-j];
+            nodes[o][0] = fop[pm2-i-j]/w;  nodes[o][1] = fop[i]/w;  nodes[o][2] = fop[j]/w;
+            dof2tk[o++] = 1;
+            nodes[o][0] = fop[pm2-i-j]/w;  nodes[o][1] = fop[i]/w;  nodes[o][2] = fop[j]/w;
+            dof2tk[o++] = 4;
+          }
+        for (int j = 0; j <= pm2; j++)  // (0,2,3)
+          for (int i = 0; i + j <= pm2; i++)
+          {
+            double w = fop[i] + fop[j] + fop[pm2-i-j];
+            nodes[o][0] = 0.;  nodes[o][1] = fop[i]/w;  nodes[o][2] = fop[j]/w;
+            dof2tk[o++] = 2;
+            nodes[o][0] = 0.;  nodes[o][1] = fop[i]/w;  nodes[o][2] = fop[j]/w;
+            dof2tk[o++] = 3;
+          }
+
+      	// Region loops to get nodes and dof2tk for regions
+        for (int k = 0; k <= pm3; k++)
+          for (int j = 0; j + k <= pm3; j++)
+            for (int i = 0; i + j + k <= pm3; i++)
+              {
+                double w = iop[i] + iop[j] + iop[k] + iop[pm3-i-j-k];
+                nodes[o][0] = iop[i]/w;  nodes[o][1] = iop[j]/w;  nodes[o][2] = iop[k]/w;
+                dof2tk[o++] = 0;
+                nodes[o][0] = iop[i]/w;  nodes[o][1] = iop[j]/w;  nodes[o][2] = iop[k]/w;
+                dof2tk[o++] = 2;
+                nodes[o][0] = iop[i]/w;  nodes[o][1] = iop[j]/w;  nodes[o][2] = iop[k]/w;
+                dof2tk[o++] = 3;
+              }
+
+      	// Populate T
+      	mth::Matrix<double> T(non,non); // T(i,j)
+        for (int m = 0; m < non; m++)
+        {
+          const double *tm = tk + 3*dof2tk[m];
+          o = 0;
 
+          double x = nodes[o][0]; double y = nodes[o][1]; double z = nodes[o][2];
 
+          getChebyshevT(pm1, x, shape_x);
+          getChebyshevT(pm1, y, shape_y);
+          getChebyshevT(pm1, z, shape_z);
+          getChebyshevT(pm1, 1. - x - y - z, shape_l);
+
+          for (int k = 0; k <= pm1; k++)
+            for (int j = 0; j + k <= pm1; j++)
+              for (int i = 0; i + j + k <= pm1; i++)
+              {
+                double s = shape_x[i]*shape_y[j]*shape_z[k]*shape_l[pm1-i-j-k];
+                T(o++, m) = s * tm[0];
+                T(o++, m) = s * tm[1];
+                T(o++, m) = s * tm[2];
+              }
+          for (int k = 0; k <= pm1; k++)
+            for (int j = 0; j + k <= pm1; j++)
+              {
+                double s = shape_x[pm1-j-k]*shape_y[j]*shape_z[k];
+                T(o++, m) = s*((y - c)*tm[0] - (x - c)*tm[1]);
+                T(o++, m) = s*((z - c)*tm[0] - (x - c)*tm[2]);
+              }
+          for (int k = 0; k <= pm1; k++)
+          {
+            T(o++, m) =
+              shape_y[pm1-k]*shape_z[k]*((z - c)*tm[1] - (y - c)*tm[2]);
+          }
+        }
+
+        unsigned rank = mth::decomposeQR(T, Q, R);
+      }
+    };
     int getOrder() {return P;}
-  private:
-    int P;
+    private:
+      int P;
 };
 
-}
+};

From 136106033aa202156b4dc579fffe4d9e6dddad72 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Mon, 23 Mar 2020 15:19:35 -0400
Subject: [PATCH 068/555] Adds more TODOs to apfNedelec.cc

---
 apf/apfNedelec.cc | 25 +++++++++++++++++--------
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index e97e4c4e8..9dc5ce5e2 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -159,6 +159,7 @@ class Nedelec: public FieldShape {
       void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<double>&) const
       {
+      	PCU_ALWAYS_ASSERT_VERBOSE(0, "getValues not implemented for Nedelec Edges. Aborting()!"
 	// TODO inform the user that this is not implemented and abort()
       }
       void getLocalGradients(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
@@ -177,8 +178,8 @@ class Nedelec: public FieldShape {
       {
       	// TODO: to be completed
       }
-      void getLocalVectorGradients(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
-	  apf::Vector3 const&, apf::NewArray<apf::Matrix3x3>&) const
+      void getLocalVectorCurls(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
       {
       	// TODO: to be completed
       }
@@ -203,11 +204,12 @@ class Nedelec: public FieldShape {
 	(void)order;
       }
       void getVectorValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
-	  apf::Vector3 const& xi, apf::NewArray<apf::Vector3>&) const
+	  apf::Vector3 const& xi, apf::NewArray<apf::Vector3>& y) const
       {
       	const int pm1 = P - 1;
         const int p = P;
 
+	apf::NewArray<double> shape_x(p);
         double* shape_x = new double[p];
         double* shape_y = new double[p];
         double* shape_l = new double[p];
@@ -217,7 +219,7 @@ class Nedelec: public FieldShape {
 
         double x = xi[0]; double y = xi[1];
 
-        getChebyshevT(pm1, x, shape_x);
+        getChebyshevT(pm1, x, &shape_x[0]);
         getChebyshevT(pm1, y, shape_y);
         getChebyshevT(pm1, 1. - x - y, shape_l);
 
@@ -235,10 +237,17 @@ class Nedelec: public FieldShape {
           u(n,0) = s*(y - c);  u(n,1) = -s*(x - c); n++;
         }
 
+
       	computeTi(); // TODO multiply u and QR in a for loop to compute vector shapes
+
+	// let s = Ti x u
+        y.allocate(dof);
+        for (int i = 0; i < dof; i++) {
+	  y(i) = apf::Vector3(s(i,0),s(i,1),0.0);
+        }
       }
-      void getLocalVectorGradients(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
-	  apf::Vector3 const&, apf::NewArray<apf::Matrix3x3>&) const
+      void getLocalVectorCurls(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
       {
       	// TODO: to be completed
       }
@@ -392,8 +401,8 @@ class Nedelec: public FieldShape {
 
       	computeTi(); // TODO multiply u and QR in a for loop to compute vector shapes
       }
-      void getLocalVectorGradients(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
-	  apf::Vector3 const&, apf::NewArray<apf::Matrix3x3>&) const
+      void getLocalVectorCurls(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
       {
       	// TODO: to be completed
       }

From 1e57f856ff2bdb2a8d1ea35925b07b046c74904a Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Tue, 24 Mar 2020 15:44:51 -0400
Subject: [PATCH 069/555] Implementation of Nedelec Shapes for Tet and Tri
 (including curlshape).

---
 apf/apfNedelec.cc | 333 ++++++++++++++++++++++++++++++++++++----------
 1 file changed, 263 insertions(+), 70 deletions(-)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index 9dc5ce5e2..1448f6265 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -13,6 +13,7 @@
 #include "mth.h"
 #include <pcu_util.h>
 #include <mthQR.h>
+#include <PCU.h>
 
 #include <iostream>
 
@@ -128,6 +129,56 @@ static void getChebyshevT(int order, double xi, double* u)
   }
 }
 
+static void getChebyshevT(int order, double xi, double* u, double* d)
+{
+  // recursive definition, z in [-1,1]
+  // T_0(z) = 1,  T_1(z) = z
+  // T_{n+1}(z) = 2*z*T_n(z) - T_{n-1}(z)
+  // T'_n(z) = n*U_{n-1}(z)
+  // U_0(z) = 1  U_1(z) = 2*z
+  // U_{n+1}(z) = 2*z*U_n(z) - U_{n-1}(z)
+  // U_n(z) = z*U_{n-1}(z) + T_n(z) = z*T'_n(z)/n + T_n(z)
+  // T'_{n+1}(z) = (n + 1)*(z*T'_n(z)/n + T_n(z))
+  double z;
+  u[0] = 1.;
+  d[0] = 0.;
+  if (order == 0) { return; }
+  u[1] = z = 2.*xi - 1.;
+  d[1] = 2.;
+  for (int n = 1; n < order; n++)
+  {
+     u[n+1] = 2*z*u[n] - u[n-1];
+     d[n+1] = (n + 1)*(z*d[n]/n + 2*u[n]);
+  }
+}
+
+static void getChebyshevT(int order, double xi, double* u, double* d, double* dd)
+{
+  // recursive definition, z in [-1,1]
+  // T_0(z) = 1,  T_1(z) = z
+  // T_{n+1}(z) = 2*z*T_n(z) - T_{n-1}(z)
+  // T'_n(z) = n*U_{n-1}(z)
+  // U_0(z) = 1  U_1(z) = 2*z
+  // U_{n+1}(z) = 2*z*U_n(z) - U_{n-1}(z)
+  // U_n(z) = z*U_{n-1}(z) + T_n(z) = z*T'_n(z)/n + T_n(z)
+  // T'_{n+1}(z) = (n + 1)*(z*T'_n(z)/n + T_n(z))
+  // T''_{n+1}(z) = (n + 1)*(2*(n + 1)*T'_n(z) + z*T''_n(z)) / n
+  double z;
+  u[0] = 1.;
+  d[0] = 0.;
+  dd[0]= 0.;
+  if (order == 0) { return; }
+  u[1] = z = 2.*xi - 1.;
+  d[1] = 2.;
+  dd[1] = 0;
+  for (int n = 1; n < order; n++)
+  {
+     u[n+1] = 2*z*u[n] - u[n-1];
+     d[n+1] = (n + 1)*(z*d[n]/n + 2*u[n]);
+     dd[n+1] = (n + 1)*(2.*(n + 1)*d[n] + z*dd[n])/n;
+  }
+}
+
 class Nedelec: public FieldShape {
   public:
     const char* getName() const { return "Nedelec"; }
@@ -159,13 +210,12 @@ class Nedelec: public FieldShape {
       void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<double>&) const
       {
-      	PCU_ALWAYS_ASSERT_VERBOSE(0, "getValues not implemented for Nedelec Edges. Aborting()!"
-	// TODO inform the user that this is not implemented and abort()
+      	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getValues not implemented for Nedelec Edges. Aborting()!");
       }
       void getLocalGradients(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
       {
-	// TODO inform the user that this is not implemented and abort()
+      	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getLocalGradients not implemented for Nedelec Edges. Aborting()!");
       }
       int countNodes() const {return 0; /* TODO update this*/}
       void alignSharedNodes(apf::Mesh*,
@@ -187,15 +237,16 @@ class Nedelec: public FieldShape {
     class Triangle : public apf::EntityShape
     {
     public:
+      int getOrder() {return P;}
       void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<double>&) const
       {
-	// TODO inform the user that this is not implemented and abort()
+      	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getValues not implemented for Nedelec Triangle. Try getVectorValues. Aborting()!");
       }
       void getLocalGradients(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
       {
-	// TODO inform the user that this is not implemented and abort()
+      	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getLocalGradients not implemented for Nedelec Triangle. Aborting()!");
       }
       int countNodes() const {return P*(P+2);}
       void alignSharedNodes(apf::Mesh*,
@@ -204,15 +255,14 @@ class Nedelec: public FieldShape {
 	(void)order;
       }
       void getVectorValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
-	  apf::Vector3 const& xi, apf::NewArray<apf::Vector3>& y) const
+	  apf::Vector3 const& xi, apf::NewArray<apf::Vector3>& shapes) const
       {
       	const int pm1 = P - 1;
         const int p = P;
 
-	apf::NewArray<double> shape_x(p);
-        double* shape_x = new double[p];
-        double* shape_y = new double[p];
-        double* shape_l = new double[p];
+      	apf::NewArray<double> shape_x(p);
+      	apf::NewArray<double> shape_y(p);
+      	apf::NewArray<double> shape_l(p);
 
         int dof = countNodes();
         mth::Matrix<double> u(dof, dim);
@@ -220,8 +270,8 @@ class Nedelec: public FieldShape {
         double x = xi[0]; double y = xi[1];
 
         getChebyshevT(pm1, x, &shape_x[0]);
-        getChebyshevT(pm1, y, shape_y);
-        getChebyshevT(pm1, 1. - x - y, shape_l);
+        getChebyshevT(pm1, y, &shape_y[0]);
+        getChebyshevT(pm1, 1. - x - y, &shape_l[0]);
 
         int n = 0;
         for (int j = 0; j <= pm1; j++)
@@ -238,26 +288,77 @@ class Nedelec: public FieldShape {
         }
 
 
-      	computeTi(); // TODO multiply u and QR in a for loop to compute vector shapes
+      	computeTi();
+        mth::Matrix<double> S(dof, dim);
+      	for(int i = 0; i < dim; i++) // S = Ti * u
+        {
+          apf::Vector<dof> B;
+          apf::Vector<double> X(dof);
+          for (int j = 0; j < dof; j++) B[j] = u(i,j); // populate b
+          mth::solveFromQR(Q, R, B, X);
+          for (int j = 0; j < dof; j++)  S(i,j) = X[j]; // populate S with x
+        }
 
-	// let s = Ti x u
-        y.allocate(dof);
-        for (int i = 0; i < dof; i++) {
-	  y(i) = apf::Vector3(s(i,0),s(i,1),0.0);
+        shapes.allocate(dof);
+        for (int i = 0; i < dof; i++) // populate y
+        {
+      	  shapes(i) = apf::Vector3( S(i,0), S(i,1), 0.0 );
         }
       }
       void getLocalVectorCurls(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
-	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
+	  apf::Vector3 const& xi, apf::NewArray<apf::Vector3>&) const
       {
-      	// TODO: to be completed
+      	const int pm1 = P - 1;
+        const int p = P;
+
+      	apf::NewArray<double> shape_x(p);
+      	apf::NewArray<double> shape_y(p);
+      	apf::NewArray<double> shape_l(p);
+      	apf::NewArray<double> dshape_x(p);
+      	apf::NewArray<double> dshape_y(p);
+      	apf::NewArray<double> dshape_l(p);
+
+      	int dof = countNodes();
+        apf::Vector<double> curlu(dof);
+
+        double x = xi[0]; double y = xi[1];
+
+        getChebyshevT(pm1, x, &shape_x[0], &dshape_x[0]);
+        getChebyshevT(pm1, y, &shape_y[0], &dshape_y[0]);
+        getChebyshevT(pm1, 1. - x - y, &shape_l[0], &dshape_l[0]);
+
+        int n = 0;
+        for (int j = 0; j <= pm1; j++)
+          for (int i = 0; i + j <= pm1; i++)
+          {
+            int l = pm1-i-j;
+            const double dx = (dshape_x[i]*shape_l[l] -
+                          shape_x(i)*dshape_l(l)) * shape_y(j);
+            const double dy = (dshape_y[j]*shape_l[l] -
+                          shape_y[j]*dshape_l[l]) * shape_x[i];
+
+            curlu(n++) = -dy;
+            curlu(n++) =  dx;
+          }
+        for (int j = 0; j <= pm1; j++)
+        {
+          int i = pm1 - j;
+          // curl of shape_x(i)*shape_y(j) * (ip.y - c, -(ip.x - c), 0):
+          curlu(n++) = -((dshape_x[i]*(x - c) + shape_x[i]) * shape_y[j] +
+                     (dshape_y[j]*(y - c) + shape_y[j]) * shape_x[i]);
+        }
+
+        computeTi();
+        apf::Vector<double> X(dof);
+        mth::solveFromQR(Q, R, curlu, X);
       }
     private:
-      //mth::Matrix<double> Ti;
+      int P; // polyonmial order
+      int dim = 2; // reference element dim
+      double c = 1./3.; // center of tri
+      const double tk[8] = {1.,0.,  -1.,1.,  0.,-1.,  0.,1.};
       mth::Matrix<double> Q;
       mth::Matrix<double> R;
-      const double tk[8] = {1.,0.,  -1.,1.,  0.,-1.,  0.,1.}; // TODO
-      const double c = 1./3.;
-      const int dim = 2; // reference space dim
       void computeTi()
       {
      	  int non = countNodes();
@@ -265,30 +366,30 @@ class Nedelec: public FieldShape {
         const double *eop = getOpenPoints(P - 1);
         const double *iop = (P > 1) ? getOpenPoints(P - 2) : NULL;
 
-        const int pm1 = P - 1, pm2 = P - 2;
+        const int p = P, pm1 = P - 1, pm2 = P - 2;
 
-        double* shape_x = new double[P];
-        double* shape_y = new double[P];
-        double* shape_l = new double[P];
+      	apf::NewArray<double> shape_x(p);
+      	apf::NewArray<double> shape_y(p);
+      	apf::NewArray<double> shape_l(p);
 
-       	apf::DynamicArray<apf::Vector<2> > nodes (non); // TODO syntax question
+       	apf::DynamicArray<apf::Vector3> nodes (non);
       	apf::DynamicArray<int> dof2tk (non);
 
         int o = 0;
       	// Edge loops to get nodes and dof2tk for edges
         for (int i = 0; i < P; i++)  // (0,1)
         {
-          nodes[o][0] = eop[i];  nodes[o][1] = 0.;
+          nodes[o][0] = eop[i];  nodes[o][1] = 0.;  nodes[o][2] = 0.;
           dof2tk[o++] = 0;
         }
         for (int i = 0; i < P; i++)  // (1,2)
         {
-          nodes[o][0] = eop[pm1-i];  nodes[o][1] = eop[i];
+          nodes[o][0] = eop[pm1-i];  nodes[o][1] = eop[i];  nodes[o][2] = 0.;
           dof2tk[o++] = 1;
         }
         for (int i = 0; i < P; i++)  // (2,0)
         {
-          nodes[o][0] = 0.;  nodes[o][1] = eop[pm1-i];
+          nodes[o][0] = 0.;  nodes[o][1] = eop[pm1-i];  nodes[o][2] = 0.;
           dof2tk[o++] = 2;
         }
 
@@ -297,9 +398,9 @@ class Nedelec: public FieldShape {
           for (int i = 0; i + j <= pm2; i++)
               {
                 double w = iop[i] + iop[j] + iop[pm2-i-j];
-                nodes[o][0] = iop[i]/w;  nodes[o][1] = iop[j]/w;
+                nodes[o][0] = iop[i]/w;  nodes[o][1] = iop[j]/w;  nodes[o][2] = 0.;
                 dof2tk[o++] = 0;
-                nodes[o][0] = iop[i]/w;  nodes[o][1] = iop[j]/w;
+                nodes[o][0] = iop[i]/w;  nodes[o][1] = iop[j]/w;  nodes[o][2] = 0.;
                 dof2tk[o++] = 3;
               }
 
@@ -312,9 +413,9 @@ class Nedelec: public FieldShape {
 
           double x = nodes[o][0]; double y = nodes[o][1];
 
-          getChebyshevT(pm1, x, shape_x);
-          getChebyshevT(pm1, y, shape_y);
-          getChebyshevT(pm1, 1. - x - y, shape_l);
+          getChebyshevT(pm1, x, &shape_x[0]);
+          getChebyshevT(pm1, y, &shape_y[0]);
+          getChebyshevT(pm1, 1. - x - y, &shape_l[0]);
 
           for (int j = 0; j <= pm1; j++)
             for (int i = 0; i + j <= pm1; i++)
@@ -332,22 +433,20 @@ class Nedelec: public FieldShape {
 
         unsigned rank = mth::decomposeQR(T, Q, R);
       }
-      int getOrder() {return P;}
-    private:
-      int P;
     };
     class Tetrahedron : public apf::EntityShape
     {
     public:
+      int getOrder() {return P;}
       void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<double>&) const
       {
-	// TODO inform the user that this is not implemented and abort()
+      	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getValues not implemented for Nedelec Tetrahedron. Try getVectorValues. Aborting()!");
       }
       void getLocalGradients(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
       {
-	// TODO inform the user that this is not implemented and abort()
+      	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getLocalGradients not implemented for Nedelec Tetrahedron. Aborting()!");
       }
       int countNodes() const {return (P+3)*(P+2)*P/2;}
       void alignSharedNodes(apf::Mesh*,
@@ -355,26 +454,26 @@ class Nedelec: public FieldShape {
       {
 	(void)order;
       }
-      void getVectorValues(apf::Mesh* m, apf::MeshEntity* e,
-	  apf::Vector3 const& xi, apf::NewArray<apf::Vector3>& u) const
+      void getVectorValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const& xi, apf::NewArray<apf::Vector3>& shapes) const
       {
         const int pm1 = P - 1;
         const int p = P;
 
-        double* shape_x = new double[p];
-        double* shape_y = new double[p];
-        double* shape_z = new double[p];
-        double* shape_l = new double[p];
+      	apf::NewArray<double> shape_x(p);
+      	apf::NewArray<double> shape_y(p);
+      	apf::NewArray<double> shape_z(p);
+      	apf::NewArray<double> shape_l(p);
 
         int dof = countNodes();
         mth::Matrix<double> u(dof, dim);
 
         double x = xi[0]; double y = xi[1]; double z = xi[2];
 
-        getChebyshevT(pm1, x, shape_x);
-        getChebyshevT(pm1, y, shape_y);
-        getChebyshevT(pm1, z, shape_z);
-        getChebyshevT(pm1, 1. - x - y - z, shape_l);
+        getChebyshevT(pm1, x, &shape_x[0]);
+        getChebyshevT(pm1, y, &shape_y[0]);
+        getChebyshevT(pm1, z, &shape_z[0]);
+        getChebyshevT(pm1, 1. - x - y - z, &shape_l[0]);
 
         int n = 0;
         for (int k = 0; k <= pm1; k++)
@@ -399,20 +498,117 @@ class Nedelec: public FieldShape {
           u(n,0) = 0.;  u(n,1) = s*(z - c);  u(n,2) = -s*(y - c);  n++;
         }
 
-      	computeTi(); // TODO multiply u and QR in a for loop to compute vector shapes
+      	computeTi();
+        mth::Matrix<double> S(dof, dim);
+      	for(int i = 0; i < dim; i++) // S = Ti * u
+        {
+          apf::Vector<double> B(dof);
+          apf::Vector<double> X(dof);
+          for (int j = 0; j < dof; j++) B[j] = u(i,j); // populate b
+          mth::solveFromQR(Q, R, B, X);
+          for (int j = 0; j < dof; j++)  S(i,j) = X[j]; // populate S with x
+        }
+
+        shapes.allocate(dof);
+        for (int i = 0; i < dof; i++) // populate y
+        {
+      	  shapes(i) = apf::Vector3( S(i,0), S(i,1), S(i,2) );
+        }
       }
       void getLocalVectorCurls(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
-	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
+	  apf::Vector3 const& xi, apf::NewArray<apf::Vector3>& curl_shapes) const
       {
-      	// TODO: to be completed
+      	const int pm1 = P - 1;
+        const int p = P;
+
+      	apf::NewArray<double> shape_x(p);
+      	apf::NewArray<double> shape_y(p);
+      	apf::NewArray<double> shape_z(p);
+      	apf::NewArray<double> shape_l(p);
+      	apf::NewArray<double> dshape_x(p);
+      	apf::NewArray<double> dshape_y(p);
+      	apf::NewArray<double> dshape_z(p);
+      	apf::NewArray<double> dshape_l(p);
+
+      	int dof = countNodes();
+        mth::Matrix<double> u(dof, dim);
+
+        double x = xi[0]; double y = xi[1]; double z = xi[2];
+
+        getChebyshevT(pm1, x, &shape_x[0], &dshape_x[0]);
+        getChebyshevT(pm1, y, &shape_y[0], &dshape_y[0]);
+        getChebyshevT(pm1, z, &shape_z[0], &dshape_z[0]);
+        getChebyshevT(pm1, 1. - x - y - z, &shape_l[0], &dshape_l[0]);
+
+        int n = 0;
+        for (int k = 0; k <= pm1; k++)
+          for (int j = 0; j + k <= pm1; j++)
+            for (int i = 0; i + j + k <= pm1; i++)
+            {
+              int l = pm1-i-j-k;
+              const double dx = (dshape_x[i]*shape_l[l] -
+                             shape_x[i]*dshape_l[l])*shape_y[j]*shape_z[k];
+              const double dy = (dshape_y[j]*shape_l[l] -
+                             shape_y[j]*dshape_l[l])*shape_x[i]*shape_z[k];
+              const double dz = (dshape_z[k]*shape_l[l] -
+                             shape_z[k]*dshape_l[l])*shape_x[i]*shape_y[j];
+
+              u(n,0) =  0.;  u(n,1) =  dz;  u(n,2) = -dy;  n++;
+              u(n,0) = -dz;  u(n,1) =  0.;  u(n,2) =  dx;  n++;
+              u(n,0) =  dy;  u(n,1) = -dx;  u(n,2) =  0.;  n++;
+            }
+        for (int k = 0; k <= pm1; k++)
+          for (int j = 0; j + k <= pm1; j++)
+          {
+            int i = pm1 - j - k;
+            // s = shape_x(i)*shape_y(j)*shape_z(k);
+            // curl of s*(ip.y - c, -(ip.x - c), 0):
+            u(n,0) =  shape_x[i]*(x - c)*shape_y[j]*dshape_z[k];
+            u(n,1) =  shape_x[i]*shape_y[j]*(y - c)*dshape_z[k];
+            u(n,2) =  -((dshape_x[i]*(x - c) + shape_x[i])*shape_y[j]*shape_z[k] +
+                      (dshape_y(j)*(y - c) + shape_y[j])*shape_x[i]*shape_z[k]);
+            n++;
+            // curl of s*(ip.z - c, 0, -(ip.x - c)):
+            u(n,0) = -shape_x[i]*(x - c)*dshape_y[j]*shape_z[k];
+            u(n,1) = (shape_x[i]*shape_y[j]*(dshape_z[k]*(z - c) + shape_z[k]) +
+                     (dshape_x[i]*(x - c) + shape_x[i])*shape_y[j]*shape_z[k]);
+            u(n,2) = -shape_x[i]*dshape_y[j]*shape_z[k]*(z - c);
+            n++;
+          }
+        for (int k = 0; k <= pm1; k++)
+        {
+          int j = pm1 - k;
+          // curl of shape_y(j)*shape_z(k)*(0, ip.z - c, -(ip.y - c)):
+          u(n,0) = -((dshape_y[j]*(y - c) + shape_y[j])*shape_z[k] +
+                   shape_y[j]*(dshape_z[k]*(z - c) + shape_z[k]));
+          u(n,1) = 0.;
+          u(n,2) = 0.;  n++;
+        }
+
+        computeTi();
+        mth::Matrix<double> S(dof, dim);
+      	for(int i = 0; i < dim; i++) // S = Ti * u
+        {
+          apf::Vector<dof> B;
+          apf::Vector<double> X(dof);
+          for (int j = 0; j < dof; j++) B[j] = u(i,j); // populate b
+          mth::solveFromQR(Q, R, B, X);
+          for (int j = 0; j < dof; j++)  S(i,j) = X[j]; // populate S with x
+        }
+
+        curl_shapes.allocate(dof);
+        for (int i = 0; i < dof; i++) // populate y
+        {
+      	  curl_shapes(i) = apf::Vector3( S(i,0), S(i,1), S(i,2) );
+        }
       }
     private:
-      //mth::Matrix<double> Ti;
+      int P;
+      int dim = 3;
+      double c = 1./4.;
+      const double tk[18] = {1.,0.,0.,  -1.,1.,0.,  0.,-1.,0.,  0.,0.,1.,  -1.,0.,1.,  0.,-1.,1.}; // edge directions
       mth::Matrix<double> Q;
       mth::Matrix<double> R;
-      const double tk[18] = {1.,0.,0.,  -1.,1.,0.,  0.,-1.,0.,  0.,0.,1.,  -1.,0.,1.,  0.,-1.,1.}; // edge directions
-      const double c = 1./4.;
-      const int dim = 3;
       void computeTi()
       {
      	  int non = countNodes();
@@ -421,12 +617,12 @@ class Nedelec: public FieldShape {
         const double *fop = (P > 1) ? getOpenPoints(P - 2) : NULL;
         const double *iop = (P > 2) ? getOpenPoints(P - 3) : NULL;
 
-        const int pm1 = P - 1, pm2 = P - 2, pm3 = P - 3;
+        const int p = P, pm1 = P - 1, pm2 = P - 2, pm3 = P - 3;
 
-        double* shape_x = new double[P];
-        double* shape_y = new double[P];
-        double* shape_z = new double[P];
-        double* shape_l = new double[P];
+      	apf::NewArray<double> shape_x(p);
+      	apf::NewArray<double> shape_y(p);
+      	apf::NewArray<double> shape_z(p);
+      	apf::NewArray<double> shape_l(p);
 
        	apf::DynamicArray<apf::Vector3> nodes (non);
       	apf::DynamicArray<int> dof2tk (non);
@@ -527,10 +723,10 @@ class Nedelec: public FieldShape {
 
           double x = nodes[o][0]; double y = nodes[o][1]; double z = nodes[o][2];
 
-          getChebyshevT(pm1, x, shape_x);
-          getChebyshevT(pm1, y, shape_y);
-          getChebyshevT(pm1, z, shape_z);
-          getChebyshevT(pm1, 1. - x - y - z, shape_l);
+          getChebyshevT(pm1, x, &shape_x[0]);
+          getChebyshevT(pm1, y, &shape_y[0]);
+          getChebyshevT(pm1, z, &shape_z[0]);
+          getChebyshevT(pm1, 1. - x - y - z, &shape_l[0]);
 
           for (int k = 0; k <= pm1; k++)
             for (int j = 0; j + k <= pm1; j++)
@@ -558,9 +754,6 @@ class Nedelec: public FieldShape {
         unsigned rank = mth::decomposeQR(T, Q, R);
       }
     };
-    int getOrder() {return P;}
-    private:
-      int P;
 };
 
 };

From effdd237a598be64dd55cd3f5c0ae1bcddc54b14 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 24 Mar 2020 16:59:14 -0400
Subject: [PATCH 070/555] Minor changes related to Nedelec shapes

---
 apf/apfNedelec.cc | 23 +++++++++++++++--------
 apf/apfShape.h    |  9 +++++++++
 2 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index 1448f6265..04a988196 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -180,10 +180,13 @@ static void getChebyshevT(int order, double xi, double* u, double* d, double* dd
 }
 
 class Nedelec: public FieldShape {
+  private:
+    int P;
   public:
+    static int getFieldOrder() {return P;}
     const char* getName() const { return "Nedelec"; }
-    Nedelec(int order) : P(order) {
-    }
+    Nedelec(int order) : P(order)
+    {}
     class Vertex : public apf::EntityShape
     {
     public:
@@ -237,7 +240,7 @@ class Nedelec: public FieldShape {
     class Triangle : public apf::EntityShape
     {
     public:
-      int getOrder() {return P;}
+      int getOrder() {return Nedelec::getFieldOrder();}
       void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<double>&) const
       {
@@ -353,7 +356,6 @@ class Nedelec: public FieldShape {
         mth::solveFromQR(Q, R, curlu, X);
       }
     private:
-      int P; // polyonmial order
       int dim = 2; // reference element dim
       double c = 1./3.; // center of tri
       const double tk[8] = {1.,0.,  -1.,1.,  0.,-1.,  0.,1.};
@@ -431,13 +433,13 @@ class Nedelec: public FieldShape {
           }
         }
 
-        unsigned rank = mth::decomposeQR(T, Q, R);
+        mth::decomposeQR(T, Q, R);
       }
     };
     class Tetrahedron : public apf::EntityShape
     {
     public:
-      int getOrder() {return P;}
+      int getOrder() {return Nedelec::getFieldOrder();}
       void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<double>&) const
       {
@@ -603,7 +605,6 @@ class Nedelec: public FieldShape {
         }
       }
     private:
-      int P;
       int dim = 3;
       double c = 1./4.;
       const double tk[18] = {1.,0.,0.,  -1.,1.,0.,  0.,-1.,0.,  0.,0.,1.,  -1.,0.,1.,  0.,-1.,1.}; // edge directions
@@ -751,9 +752,15 @@ class Nedelec: public FieldShape {
           }
         }
 
-        unsigned rank = mth::decomposeQR(T, Q, R);
+        mth::decomposeQR(T, Q, R);
       }
     };
 };
 
+apf::FieldShape* getNedelec(int order)
+{
+  static Nedelec nedelec(order);
+  return &nedelec;
+}
+
 };
diff --git a/apf/apfShape.h b/apf/apfShape.h
index da907a6d7..9f0fc23bc 100644
--- a/apf/apfShape.h
+++ b/apf/apfShape.h
@@ -42,6 +42,15 @@ class EntityShape
         MeshEntity* e,
         Vector3 const& xi,
         NewArray<Vector3>& grads) const = 0;
+/** \brief evaluate element vector shape functions
+ \details this is use only for Nedelc
+ \param xi the parent element coordinates
+ \param values each entry is the vector shape function value for one node */
+    virtual void getVectorValues(
+        Mesh* m,
+        MeshEntity* e,
+        Vector3 const& xi,
+        NewArray<apf::Vector3>& values) const;
 /** \brief return the number of nodes affecting this element
     \details in a linear mesh, there are two nodes affecting
              and edge, three nodes affecting a triangle,

From cb8b34f7563f83f5c02034f6c9f74aa64f93b56c Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Wed, 25 Mar 2020 10:51:22 -0400
Subject: [PATCH 071/555] Fixed compilation errors.

---
 apf/apfNedelec.cc | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index 04a988196..e55a18e4a 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -24,7 +24,7 @@ enum {
   GAUSS_LOBATTO
 };
 
-static void getGaussLegendrePoints(int np, double* pts)
+void getGaussLegendrePoints(int np, double* pts)
 {
   switch (np)
   {
@@ -79,11 +79,11 @@ static void getGaussLegendrePoints(int np, double* pts)
   }
 }
 
-static void getGaussLobattoPoints(int /*np*/, double* /*pts*/)
+void getGaussLobattoPoints(int /*np*/, double* /*pts*/)
 { /* TODO implement Gauss Lobatto points. Later when needed. */
 };
 
-static const double* getPoints(int order, const int type)
+const double* getPoints(int order, const int type)
 {
   int np = order + 1;
   double* points = new double[np];
@@ -103,17 +103,17 @@ static const double* getPoints(int order, const int type)
   return points;
 }
 
-static const double* getOpenPoints(int order, const int type = GAUSS_LEGENDRE)
+const double* getOpenPoints(int order, const int type = GAUSS_LEGENDRE)
 {
   return getPoints(order, type);
 }
 
-static const double* getClosedPoints(int order, const int type = GAUSS_LOBATTO)
+const double* getClosedPoints(int order, const int type = GAUSS_LOBATTO)
 {
     return getPoints(order, type);
 }
 
-static void getChebyshevT(int order, double xi, double* u)
+void getChebyshevT(int order, double xi, double* u)
 {
   // TODO implement Chebyshev
   // recursive definition, z in [-1,1]
@@ -129,7 +129,7 @@ static void getChebyshevT(int order, double xi, double* u)
   }
 }
 
-static void getChebyshevT(int order, double xi, double* u, double* d)
+void getChebyshevT(int order, double xi, double* u, double* d)
 {
   // recursive definition, z in [-1,1]
   // T_0(z) = 1,  T_1(z) = z
@@ -152,7 +152,7 @@ static void getChebyshevT(int order, double xi, double* u, double* d)
   }
 }
 
-static void getChebyshevT(int order, double xi, double* u, double* d, double* dd)
+void getChebyshevT(int order, double xi, double* u, double* d, double* dd)
 {
   // recursive definition, z in [-1,1]
   // T_0(z) = 1,  T_1(z) = z
@@ -295,9 +295,9 @@ class Nedelec: public FieldShape {
         mth::Matrix<double> S(dof, dim);
       	for(int i = 0; i < dim; i++) // S = Ti * u
         {
-          apf::Vector<dof> B;
-          apf::Vector<double> X(dof);
-          for (int j = 0; j < dof; j++) B[j] = u(i,j); // populate b
+          mth::Vector<double> B (dof);
+          mth::Vector<double> X (dof);
+          for (int j = 0; j < dof; j++) B[j] = u(i,j); // populate b in QR x = b
           mth::solveFromQR(Q, R, B, X);
           for (int j = 0; j < dof; j++)  S(i,j) = X[j]; // populate S with x
         }
@@ -322,7 +322,7 @@ class Nedelec: public FieldShape {
       	apf::NewArray<double> dshape_l(p);
 
       	int dof = countNodes();
-        apf::Vector<double> curlu(dof);
+        mth::Vector<double> curlu(dof);
 
         double x = xi[0]; double y = xi[1];
 
@@ -336,7 +336,7 @@ class Nedelec: public FieldShape {
           {
             int l = pm1-i-j;
             const double dx = (dshape_x[i]*shape_l[l] -
-                          shape_x(i)*dshape_l(l)) * shape_y(j);
+                          shape_x[i]*dshape_l[l]) * shape_y[j];
             const double dy = (dshape_y[j]*shape_l[l] -
                           shape_y[j]*dshape_l[l]) * shape_x[i];
 
@@ -352,7 +352,7 @@ class Nedelec: public FieldShape {
         }
 
         computeTi();
-        apf::Vector<double> X(dof);
+        mth::Vector<double> X(dof);
         mth::solveFromQR(Q, R, curlu, X);
       }
     private:
@@ -504,8 +504,8 @@ class Nedelec: public FieldShape {
         mth::Matrix<double> S(dof, dim);
       	for(int i = 0; i < dim; i++) // S = Ti * u
         {
-          apf::Vector<double> B(dof);
-          apf::Vector<double> X(dof);
+          mth::Vector<double> B (dof);
+          mth::Vector<double> X (dof);
           for (int j = 0; j < dof; j++) B[j] = u(i,j); // populate b
           mth::solveFromQR(Q, R, B, X);
           for (int j = 0; j < dof; j++)  S(i,j) = X[j]; // populate S with x
@@ -591,8 +591,8 @@ class Nedelec: public FieldShape {
         mth::Matrix<double> S(dof, dim);
       	for(int i = 0; i < dim; i++) // S = Ti * u
         {
-          apf::Vector<dof> B;
-          apf::Vector<double> X(dof);
+          mth::Vector<double> B(dof);
+          mth::Vector<double> X(dof);
           for (int j = 0; j < dof; j++) B[j] = u(i,j); // populate b
           mth::solveFromQR(Q, R, B, X);
           for (int j = 0; j < dof; j++)  S(i,j) = X[j]; // populate S with x

From b9d3621d1d85e177b97890dbdbd57e635a16b49c Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Wed, 25 Mar 2020 11:08:38 -0400
Subject: [PATCH 072/555] Fixed compilation errors.

---
 apf/apfNedelec.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index e55a18e4a..b28d9ed5c 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -305,7 +305,7 @@ class Nedelec: public FieldShape {
         shapes.allocate(dof);
         for (int i = 0; i < dof; i++) // populate y
         {
-      	  shapes(i) = apf::Vector3( S(i,0), S(i,1), 0.0 );
+      	  shapes[i] = apf::Vector3( S(i,0), S(i,1), 0.0 );
         }
       }
       void getLocalVectorCurls(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
@@ -514,7 +514,7 @@ class Nedelec: public FieldShape {
         shapes.allocate(dof);
         for (int i = 0; i < dof; i++) // populate y
         {
-      	  shapes(i) = apf::Vector3( S(i,0), S(i,1), S(i,2) );
+      	  shapes[i] = apf::Vector3( S(i,0), S(i,1), S(i,2) );
         }
       }
       void getLocalVectorCurls(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
@@ -568,7 +568,7 @@ class Nedelec: public FieldShape {
             u(n,0) =  shape_x[i]*(x - c)*shape_y[j]*dshape_z[k];
             u(n,1) =  shape_x[i]*shape_y[j]*(y - c)*dshape_z[k];
             u(n,2) =  -((dshape_x[i]*(x - c) + shape_x[i])*shape_y[j]*shape_z[k] +
-                      (dshape_y(j)*(y - c) + shape_y[j])*shape_x[i]*shape_z[k]);
+                      (dshape_y[j]*(y - c) + shape_y[j])*shape_x[i]*shape_z[k]);
             n++;
             // curl of s*(ip.z - c, 0, -(ip.x - c)):
             u(n,0) = -shape_x[i]*(x - c)*dshape_y[j]*shape_z[k];
@@ -601,7 +601,7 @@ class Nedelec: public FieldShape {
         curl_shapes.allocate(dof);
         for (int i = 0; i < dof; i++) // populate y
         {
-      	  curl_shapes(i) = apf::Vector3( S(i,0), S(i,1), S(i,2) );
+      	  curl_shapes[i] = apf::Vector3( S(i,0), S(i,1), S(i,2) );
         }
       }
     private:

From 156d9d825a9cc6d735a27354209e314593f3e9b8 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Wed, 25 Mar 2020 18:05:00 -0400
Subject: [PATCH 073/555] Attempts to fix the issue with order

But this way of doing things order P will alway be the same for all the
nedelec shapes.
For example: if we define a shape by using
"apf::FieldShape* nedelec2 = apf::getNedelec(2)"
and then define another one using
"apf::FieldShape* nedelec3 = apf::getNedelec(3)"
then "getOrder()" returns 3 for both of them!
---
 apf/apfNedelec.cc | 45 ++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 38 insertions(+), 7 deletions(-)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index b28d9ed5c..d757c88a3 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -180,12 +180,13 @@ void getChebyshevT(int order, double xi, double* u, double* d, double* dd)
 }
 
 class Nedelec: public FieldShape {
-  private:
-    int P;
   public:
-    static int getFieldOrder() {return P;}
+    static int P;
     const char* getName() const { return "Nedelec"; }
-    Nedelec(int order) : P(order)
+    int getOrder() {return P;}
+    /* Nedelec(int order) : P(order) */
+    /* {} */
+    Nedelec()
     {}
     class Vertex : public apf::EntityShape
     {
@@ -240,7 +241,8 @@ class Nedelec: public FieldShape {
     class Triangle : public apf::EntityShape
     {
     public:
-      int getOrder() {return Nedelec::getFieldOrder();}
+      /* int getOrder() {return Nedelec::getFieldOrder();} */
+      int getOrder() {return P;}
       void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<double>&) const
       {
@@ -439,7 +441,8 @@ class Nedelec: public FieldShape {
     class Tetrahedron : public apf::EntityShape
     {
     public:
-      int getOrder() {return Nedelec::getFieldOrder();}
+      /* int getOrder() {return Nedelec::getFieldOrder();} */
+      int getOrder() {return P;}
       void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<double>&) const
       {
@@ -755,11 +758,39 @@ class Nedelec: public FieldShape {
         mth::decomposeQR(T, Q, R);
       }
     };
+    EntityShape* getEntityShape(int type)
+    {
+      static Edge edge;
+      static Triangle triangle;
+      static Tetrahedron tet;
+      static EntityShape* shapes[apf::Mesh::TYPES] =
+      {NULL,       //vertex
+       &edge,      //edge
+       &triangle,  //triangle
+       NULL,       //quad
+       &tet,       //tetrahedron
+       NULL,       //hex
+       NULL,       //prism
+       NULL        //pyramid
+      };
+      return shapes[type];
+    }
+    bool hasNodesIn(int)
+    {
+      // TODO complete this
+      return true;
+    }
+    int countNodesOn(int)
+    {
+      // TODO complete this
+      return 0;
+    }
 };
 
 apf::FieldShape* getNedelec(int order)
 {
-  static Nedelec nedelec(order);
+  Nedelec::P = order;
+  static Nedelec nedelec;
   return &nedelec;
 }
 

From 8c29aa6ef9b92ff4e201fe8eaa3973614b1799ff Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Wed, 25 Mar 2020 19:04:26 -0400
Subject: [PATCH 074/555] Uses templates for the order P

The drawback of this approach is that different order nedelec shapes has
to be specialized manually. But since this is done inside getNedlec
routine, we can just specialize as many as we want and clients don't
have to worry about this. For now the specializations are done for
orders 1 to 10.
---
 apf/apfNedelec.cc | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index d757c88a3..b98cf0bb7 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -179,9 +179,9 @@ void getChebyshevT(int order, double xi, double* u, double* d, double* dd)
   }
 }
 
+template<int P>
 class Nedelec: public FieldShape {
   public:
-    static int P;
     const char* getName() const { return "Nedelec"; }
     int getOrder() {return P;}
     /* Nedelec(int order) : P(order) */
@@ -241,7 +241,6 @@ class Nedelec: public FieldShape {
     class Triangle : public apf::EntityShape
     {
     public:
-      /* int getOrder() {return Nedelec::getFieldOrder();} */
       int getOrder() {return P;}
       void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<double>&) const
@@ -441,7 +440,6 @@ class Nedelec: public FieldShape {
     class Tetrahedron : public apf::EntityShape
     {
     public:
-      /* int getOrder() {return Nedelec::getFieldOrder();} */
       int getOrder() {return P;}
       void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<double>&) const
@@ -789,9 +787,23 @@ class Nedelec: public FieldShape {
 
 apf::FieldShape* getNedelec(int order)
 {
-  Nedelec::P = order;
-  static Nedelec nedelec;
-  return &nedelec;
+  PCU_ALWAYS_ASSERT_VERBOSE(order >= 1,
+      "order is expected to be bigger than or equal to 1!");
+  PCU_ALWAYS_ASSERT_VERBOSE(order <= 10,
+      "order is expected to be less than or equal to 10!");
+  static Nedelec<1>  ND1;
+  static Nedelec<2>  ND2;
+  static Nedelec<3>  ND3;
+  static Nedelec<4>  ND4;
+  static Nedelec<5>  ND5;
+  static Nedelec<6>  ND6;
+  static Nedelec<7>  ND7;
+  static Nedelec<8>  ND8;
+  static Nedelec<9>  ND9;
+  static Nedelec<10> ND10;
+  static FieldShape* const nedelecShapes[10] =
+  {&ND1, &ND2, &ND3, &ND4, &ND5, &ND6, &ND7, &ND8, &ND9, &ND10};
+  return nedelecShapes[order];
 }
 
 };

From 379dfb8e7acbfac4d03844fc57ac937c9bdd5ad2 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Wed, 25 Mar 2020 22:53:15 -0400
Subject: [PATCH 075/555] Re-arranges computations of Ti s

This now uses static tables to avoid recomputing Q and R if they are
already computed.
---
 apf/apfNedelec.cc | 556 ++++++++++++++++++++++++++--------------------
 1 file changed, 320 insertions(+), 236 deletions(-)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index b98cf0bb7..e196e43c1 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -19,6 +19,7 @@
 
 namespace apf {
 
+static unsigned const MAX_ND_ORDER = 10;
 enum {
   GAUSS_LEGENDRE,
   GAUSS_LOBATTO
@@ -179,6 +180,300 @@ void getChebyshevT(int order, double xi, double* u, double* d, double* dd)
   }
 }
 
+static inline int countTriNodes(int P)
+{
+  return P*(P+2);
+}
+static inline int countTetNodes(int P)
+{
+  return (P+3)*(P+2)*P/2;
+}
+
+static void computeTriangleTi(
+    int P, /*order*/
+    mth::Matrix<double>& Q, /*Q in QR factorization of Ti*/
+    mth::Matrix<double>& R) /*R in QR factorization of Ti*/
+{
+  const double tk[8] = {1.,0.,  -1.,1.,  0.,-1.,  0.,1.};
+  const double c = 1./3.;
+  int non = countTriNodes(P);
+
+  const double *eop = getOpenPoints(P - 1);
+  const double *iop = (P > 1) ? getOpenPoints(P - 2) : NULL;
+
+  const int p = P, pm1 = P - 1, pm2 = P - 2;
+  apf::NewArray<double> shape_x(p);
+  apf::NewArray<double> shape_y(p);
+  apf::NewArray<double> shape_l(p);
+
+  apf::DynamicArray<apf::Vector3> nodes (non);
+  apf::DynamicArray<int> dof2tk (non);
+
+  int o = 0;
+  // Edge loops to get nodes and dof2tk for edges
+  for (int i = 0; i < P; i++)  // (0,1)
+  {
+    nodes[o][0] = eop[i];  nodes[o][1] = 0.;  nodes[o][2] = 0.;
+    dof2tk[o++] = 0;
+  }
+  for (int i = 0; i < P; i++)  // (1,2)
+  {
+    nodes[o][0] = eop[pm1-i];  nodes[o][1] = eop[i];  nodes[o][2] = 0.;
+    dof2tk[o++] = 1;
+  }
+  for (int i = 0; i < P; i++)  // (2,0)
+  {
+    nodes[o][0] = 0.;  nodes[o][1] = eop[pm1-i];  nodes[o][2] = 0.;
+    dof2tk[o++] = 2;
+  }
+
+  // Face loops to get nodes and dof2tk for faces
+  for (int j = 0; j <= pm2; j++) {
+    for (int i = 0; i + j <= pm2; i++)
+      {
+        double w = iop[i] + iop[j] + iop[pm2-i-j];
+        nodes[o][0] = iop[i]/w;  nodes[o][1] = iop[j]/w;  nodes[o][2] = 0.;
+        dof2tk[o++] = 0;
+        nodes[o][0] = iop[i]/w;  nodes[o][1] = iop[j]/w;  nodes[o][2] = 0.;
+        dof2tk[o++] = 3;
+      }
+  }
+
+  // Populate T
+  mth::Matrix<double> T(non,non); // T(i,j)
+  for (int m = 0; m < non; m++)
+  {
+    const double *tm = tk + 2*dof2tk[m];
+    o = 0;
+
+    double x = nodes[o][0]; double y = nodes[o][1];
+
+    getChebyshevT(pm1, x, &shape_x[0]);
+    getChebyshevT(pm1, y, &shape_y[0]);
+    getChebyshevT(pm1, 1. - x - y, &shape_l[0]);
+
+    for (int j = 0; j <= pm1; j++)
+      for (int i = 0; i + j <= pm1; i++)
+      {
+        double s = shape_x[i]*shape_y[j]*shape_l[pm1-i-j];
+        T(o++, m) = s * tm[0];
+        T(o++, m) = s * tm[1];
+      }
+    for (int j = 0; j <= pm1; j++)
+    {
+      T(o++, m) =
+        shape_x[pm1-j]*shape_y[j]*((y - c)*tm[0] - (x - c)*tm[1]);
+    }
+  }
+  mth::decomposeQR(T, Q, R);
+}
+
+static void computeTetTi(
+    int P, /*order*/
+    mth::Matrix<double>& Q, /*Q in QR factorization of Ti*/
+    mth::Matrix<double>& R) /*R in QR factorization of Ti*/
+{
+  int non = countTetNodes(P);
+  const double c = 1./4.;
+  const double tk[18] = { /* edge directions in a tet */
+     1. ,  0.,  0.,
+    -1. ,  1.,  0.,
+     0. , -1.,  0.,
+     0. ,  0.,  1.,
+    -1. ,  0.,  1.,
+     0.,  -1.,  1.};
+  const double *eop = getOpenPoints(P - 1);
+  const double *fop = (P > 1) ? getOpenPoints(P - 2) : NULL;
+  const double *iop = (P > 2) ? getOpenPoints(P - 3) : NULL;
+
+  const int p = P, pm1 = P - 1, pm2 = P - 2, pm3 = P - 3;
+
+  apf::NewArray<double> shape_x(p);
+  apf::NewArray<double> shape_y(p);
+  apf::NewArray<double> shape_z(p);
+  apf::NewArray<double> shape_l(p);
+
+  apf::DynamicArray<apf::Vector3> nodes (non);
+  apf::DynamicArray<int> dof2tk (non);
+  nodes.setSize(non);
+  dof2tk.setSize(non);
+
+  int o = 0;
+  // Edge loops to get nodes and dof2tk for edges
+  for (int i = 0; i < P; i++)  // (0,1)
+  {
+    nodes[o][0] = eop[i];  nodes[o][1] = 0.;  nodes[o][2] = 0.;
+    dof2tk[o++] = 0;
+  }
+  for (int i = 0; i < P; i++)  // (1,2)
+  {
+    nodes[o][0] = eop[pm1-i];  nodes[o][1] = eop[i];  nodes[o][2] = 0.;
+    dof2tk[o++] = 1;
+  }
+  for (int i = 0; i < P; i++)  // (2,0)
+  {
+    nodes[o][0] = 0.;  nodes[o][1] = eop[pm1-i];  nodes[o][2] = 0.;
+    dof2tk[o++] = 2;
+  }
+  for (int i = 0; i < P; i++)  // (0,3)
+  {
+    nodes[o][0] = 0.;  nodes[o][1] = 0.;  nodes[o][2] = eop[i];
+    dof2tk[o++] = 3;
+  }
+  for (int i = 0; i < P; i++)  // (1,3)
+  {
+    nodes[o][0] = eop[pm1-i];  nodes[o][1] = 0.;  nodes[o][2] = eop[i];
+    dof2tk[o++] = 4;
+  }
+  for (int i = 0; i < P; i++)  // (2,3)
+  {
+    nodes[o][0] = 0.;  nodes[o][1] = eop[pm1-i];  nodes[o][2] = eop[i];
+    dof2tk[o++] = 5;
+  }
+
+  // Face loops to get nodes and dof2tk for faces
+  // (0,1,2)
+  for (int j = 0; j <= pm2; j++) {
+    for (int i = 0; i + j <= pm2; i++)
+    {
+      double w = fop[i] + fop[j] + fop[pm2-i-j];
+      nodes[o][0] = fop[i]/w;  nodes[o][1] = fop[j]/w;  nodes[o][2] = 0.;
+      dof2tk[o++] = 0;
+      nodes[o][0] = fop[i]/w;  nodes[o][1] = fop[j]/w;  nodes[o][2] = 0.;
+      dof2tk[o++] = 2;
+    }
+  }
+  // (0,1,3)
+  for (int j = 0; j <= pm2; j++) {
+    for (int i = 0; i + j <= pm2; i++)
+    {
+      double w = fop[i] + fop[j] + fop[pm2-i-j];
+      nodes[o][0] = fop[i]/w;  nodes[o][1] = 0.;  nodes[o][2] = fop[j]/w;
+      dof2tk[o++] = 0;
+      nodes[o][0] = fop[i]/w;  nodes[o][1] = 0.;  nodes[o][2] = fop[j]/w;
+      dof2tk[o++] = 3;
+    }
+  }
+  // (1,2,3)
+  for (int j = 0; j <= pm2; j++) {
+    for (int i = 0; i + j <= pm2; i++)
+    {
+      double w = fop[i] + fop[j] + fop[pm2-i-j];
+      nodes[o][0] = fop[pm2-i-j]/w;  nodes[o][1] = fop[i]/w;  nodes[o][2] = fop[j]/w;
+      dof2tk[o++] = 1;
+      nodes[o][0] = fop[pm2-i-j]/w;  nodes[o][1] = fop[i]/w;  nodes[o][2] = fop[j]/w;
+      dof2tk[o++] = 4;
+    }
+  }
+  // (0,2,3)
+  for (int j = 0; j <= pm2; j++) {
+    for (int i = 0; i + j <= pm2; i++)
+    {
+      double w = fop[i] + fop[j] + fop[pm2-i-j];
+      nodes[o][0] = 0.;  nodes[o][1] = fop[i]/w;  nodes[o][2] = fop[j]/w;
+      dof2tk[o++] = 2;
+      nodes[o][0] = 0.;  nodes[o][1] = fop[i]/w;  nodes[o][2] = fop[j]/w;
+      dof2tk[o++] = 3;
+    }
+  }
+
+  // Region loops to get nodes and dof2tk for regions
+  for (int k = 0; k <= pm3; k++) {
+    for (int j = 0; j + k <= pm3; j++) {
+      for (int i = 0; i + j + k <= pm3; i++) {
+        double w = iop[i] + iop[j] + iop[k] + iop[pm3-i-j-k];
+        nodes[o][0] = iop[i]/w;  nodes[o][1] = iop[j]/w;  nodes[o][2] = iop[k]/w;
+        dof2tk[o++] = 0;
+        nodes[o][0] = iop[i]/w;  nodes[o][1] = iop[j]/w;  nodes[o][2] = iop[k]/w;
+        dof2tk[o++] = 2;
+        nodes[o][0] = iop[i]/w;  nodes[o][1] = iop[j]/w;  nodes[o][2] = iop[k]/w;
+        dof2tk[o++] = 3;
+      }
+    }
+  }
+
+  // Populate T
+  mth::Matrix<double> T(non,non); // T(i,j)
+  for (int m = 0; m < non; m++)
+  {
+    const double *tm = tk + 3*dof2tk[m];
+    o = 0;
+
+    double x = nodes[o][0]; double y = nodes[o][1]; double z = nodes[o][2];
+
+    getChebyshevT(pm1, x, &shape_x[0]);
+    getChebyshevT(pm1, y, &shape_y[0]);
+    getChebyshevT(pm1, z, &shape_z[0]);
+    getChebyshevT(pm1, 1. - x - y - z, &shape_l[0]);
+
+    for (int k = 0; k <= pm1; k++) {
+      for (int j = 0; j + k <= pm1; j++) {
+        for (int i = 0; i + j + k <= pm1; i++)
+        {
+          double s = shape_x[i]*shape_y[j]*shape_z[k]*shape_l[pm1-i-j-k];
+          T(o++, m) = s * tm[0];
+          T(o++, m) = s * tm[1];
+          T(o++, m) = s * tm[2];
+        }
+      }
+    }
+    for (int k = 0; k <= pm1; k++) {
+      for (int j = 0; j + k <= pm1; j++) {
+        {
+          double s = shape_x[pm1-j-k]*shape_y[j]*shape_z[k];
+          T(o++, m) = s*((y - c)*tm[0] - (x - c)*tm[1]);
+          T(o++, m) = s*((z - c)*tm[0] - (x - c)*tm[2]);
+        }
+      }
+    }
+    for (int k = 0; k <= pm1; k++)
+    {
+      T(o++, m) =
+        shape_y[pm1-k]*shape_z[k]*((z - c)*tm[1] - (y - c)*tm[2]);
+    }
+  }
+  mth::decomposeQR(T, Q, R);
+}
+
+static void getTi(
+    int P,
+    int type,
+    mth::Matrix<double>& Q,
+    mth::Matrix<double>& R)
+{
+
+  bool cond = (type == apf::Mesh::TRIANGLE || type == apf::Mesh::TET);
+  PCU_ALWAYS_ASSERT_VERBOSE(cond,
+      "type should be either apf::Mesh::TRIANGLE or apf::Mesh::TET!");
+
+  static apf::NewArray<double> transformQ[apf::Mesh::TYPES][MAX_ND_ORDER+1];
+  static apf::NewArray<double> transformR[apf::Mesh::TYPES][MAX_ND_ORDER+1];
+  int n = type == apf::Mesh::TRIANGLE ? countTriNodes(P) : countTetNodes(P);
+
+  // get the transform matrices if the are not already computed
+  if (!transformQ[type][P].allocated()) {
+    mth::Matrix<double> LQ(n,n);
+    mth::Matrix<double> LR(n,n);
+    type == apf::Mesh::TRIANGLE ?
+    	  computeTriangleTi(P, LQ, LR) : computeTetTi(P, LQ, LR);
+
+    for (int i = 0; i < n; i++) {
+      for (int j = 0; j < n; j++) {
+	transformQ[type][P][i*n+j] = LQ(i,j);
+	transformR[type][P][i*n+j] = LR(i,j);
+      }
+    }
+  }
+
+  // set Q and R using transformQ and transformR
+  for (int i = 0; i < n; i++) {
+    for (int j = 0; j < n; j++) {
+      Q(i,j) = transformQ[type][P][i*n+j];
+      R(i,j) = transformR[type][P][i*n+j];
+    }
+  }
+}
+
 template<int P>
 class Nedelec: public FieldShape {
   public:
@@ -240,6 +535,9 @@ class Nedelec: public FieldShape {
     };
     class Triangle : public apf::EntityShape
     {
+    private:
+      const int dim = 2; // reference element dim
+      const double c = 1./3.; // center of tri
     public:
       int getOrder() {return P;}
       void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
@@ -252,7 +550,7 @@ class Nedelec: public FieldShape {
       {
       	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getLocalGradients not implemented for Nedelec Triangle. Aborting()!");
       }
-      int countNodes() const {return P*(P+2);}
+      int countNodes() const {return countTriNodes(P);}
       void alignSharedNodes(apf::Mesh*,
 	  apf::MeshEntity*, apf::MeshEntity*, int order[])
       {
@@ -292,7 +590,10 @@ class Nedelec: public FieldShape {
         }
 
 
-      	computeTi();
+	mth::Matrix<double> Q(dof, dof);
+	mth::Matrix<double> R(dof, dof);
+	getTi(P, apf::Mesh::TRIANGLE, Q, R);
+
         mth::Matrix<double> S(dof, dim);
       	for(int i = 0; i < dim; i++) // S = Ti * u
         {
@@ -352,93 +653,19 @@ class Nedelec: public FieldShape {
                      (dshape_y[j]*(y - c) + shape_y[j]) * shape_x[i]);
         }
 
-        computeTi();
+	mth::Matrix<double> Q(dof, dof);
+	mth::Matrix<double> R(dof, dof);
+	getTi(P, apf::Mesh::TRIANGLE, Q, R);
+
         mth::Vector<double> X(dof);
         mth::solveFromQR(Q, R, curlu, X);
       }
-    private:
-      int dim = 2; // reference element dim
-      double c = 1./3.; // center of tri
-      const double tk[8] = {1.,0.,  -1.,1.,  0.,-1.,  0.,1.};
-      mth::Matrix<double> Q;
-      mth::Matrix<double> R;
-      void computeTi()
-      {
-     	  int non = countNodes();
-
-        const double *eop = getOpenPoints(P - 1);
-        const double *iop = (P > 1) ? getOpenPoints(P - 2) : NULL;
-
-        const int p = P, pm1 = P - 1, pm2 = P - 2;
-
-      	apf::NewArray<double> shape_x(p);
-      	apf::NewArray<double> shape_y(p);
-      	apf::NewArray<double> shape_l(p);
-
-       	apf::DynamicArray<apf::Vector3> nodes (non);
-      	apf::DynamicArray<int> dof2tk (non);
-
-        int o = 0;
-      	// Edge loops to get nodes and dof2tk for edges
-        for (int i = 0; i < P; i++)  // (0,1)
-        {
-          nodes[o][0] = eop[i];  nodes[o][1] = 0.;  nodes[o][2] = 0.;
-          dof2tk[o++] = 0;
-        }
-        for (int i = 0; i < P; i++)  // (1,2)
-        {
-          nodes[o][0] = eop[pm1-i];  nodes[o][1] = eop[i];  nodes[o][2] = 0.;
-          dof2tk[o++] = 1;
-        }
-        for (int i = 0; i < P; i++)  // (2,0)
-        {
-          nodes[o][0] = 0.;  nodes[o][1] = eop[pm1-i];  nodes[o][2] = 0.;
-          dof2tk[o++] = 2;
-        }
-
-      	// Region loops to get nodes and dof2tk for regions
-        for (int j = 0; j <= pm2; j++)
-          for (int i = 0; i + j <= pm2; i++)
-              {
-                double w = iop[i] + iop[j] + iop[pm2-i-j];
-                nodes[o][0] = iop[i]/w;  nodes[o][1] = iop[j]/w;  nodes[o][2] = 0.;
-                dof2tk[o++] = 0;
-                nodes[o][0] = iop[i]/w;  nodes[o][1] = iop[j]/w;  nodes[o][2] = 0.;
-                dof2tk[o++] = 3;
-              }
-
-      	// Populate T
-      	mth::Matrix<double> T(non,non); // T(i,j)
-        for (int m = 0; m < non; m++)
-        {
-          const double *tm = tk + 2*dof2tk[m];
-          o = 0;
-
-          double x = nodes[o][0]; double y = nodes[o][1];
-
-          getChebyshevT(pm1, x, &shape_x[0]);
-          getChebyshevT(pm1, y, &shape_y[0]);
-          getChebyshevT(pm1, 1. - x - y, &shape_l[0]);
-
-          for (int j = 0; j <= pm1; j++)
-            for (int i = 0; i + j <= pm1; i++)
-            {
-              double s = shape_x[i]*shape_y[j]*shape_l[pm1-i-j];
-              T(o++, m) = s * tm[0];
-              T(o++, m) = s * tm[1];
-            }
-          for (int j = 0; j <= pm1; j++)
-          {
-            T(o++, m) =
-              shape_x[pm1-j]*shape_y[j]*((y - c)*tm[0] - (x - c)*tm[1]);
-          }
-        }
-
-        mth::decomposeQR(T, Q, R);
-      }
     };
     class Tetrahedron : public apf::EntityShape
     {
+    private:
+      const int dim = 3;
+      const double c = 1./4.;
     public:
       int getOrder() {return P;}
       void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
@@ -451,7 +678,7 @@ class Nedelec: public FieldShape {
       {
       	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getLocalGradients not implemented for Nedelec Tetrahedron. Aborting()!");
       }
-      int countNodes() const {return (P+3)*(P+2)*P/2;}
+      int countNodes() const {return countTetNodes(P);}
       void alignSharedNodes(apf::Mesh*,
 	  apf::MeshEntity*, apf::MeshEntity*, int order[])
       {
@@ -501,7 +728,10 @@ class Nedelec: public FieldShape {
           u(n,0) = 0.;  u(n,1) = s*(z - c);  u(n,2) = -s*(y - c);  n++;
         }
 
-      	computeTi();
+	mth::Matrix<double> Q(dof, dof);
+	mth::Matrix<double> R(dof, dof);
+	getTi(P, apf::Mesh::TET, Q, R);
+
         mth::Matrix<double> S(dof, dim);
       	for(int i = 0; i < dim; i++) // S = Ti * u
         {
@@ -588,7 +818,11 @@ class Nedelec: public FieldShape {
           u(n,2) = 0.;  n++;
         }
 
-        computeTi();
+
+	mth::Matrix<double> Q(dof, dof);
+	mth::Matrix<double> R(dof, dof);
+	getTi(P, apf::Mesh::TET, Q, R);
+
         mth::Matrix<double> S(dof, dim);
       	for(int i = 0; i < dim; i++) // S = Ti * u
         {
@@ -605,156 +839,6 @@ class Nedelec: public FieldShape {
       	  curl_shapes[i] = apf::Vector3( S(i,0), S(i,1), S(i,2) );
         }
       }
-    private:
-      int dim = 3;
-      double c = 1./4.;
-      const double tk[18] = {1.,0.,0.,  -1.,1.,0.,  0.,-1.,0.,  0.,0.,1.,  -1.,0.,1.,  0.,-1.,1.}; // edge directions
-      mth::Matrix<double> Q;
-      mth::Matrix<double> R;
-      void computeTi()
-      {
-     	  int non = countNodes();
-
-        const double *eop = getOpenPoints(P - 1);
-        const double *fop = (P > 1) ? getOpenPoints(P - 2) : NULL;
-        const double *iop = (P > 2) ? getOpenPoints(P - 3) : NULL;
-
-        const int p = P, pm1 = P - 1, pm2 = P - 2, pm3 = P - 3;
-
-      	apf::NewArray<double> shape_x(p);
-      	apf::NewArray<double> shape_y(p);
-      	apf::NewArray<double> shape_z(p);
-      	apf::NewArray<double> shape_l(p);
-
-       	apf::DynamicArray<apf::Vector3> nodes (non);
-      	apf::DynamicArray<int> dof2tk (non);
-	      nodes.setSize(non);
-      	dof2tk.setSize(non);
-
-        int o = 0;
-      	// Edge loops to get nodes and dof2tk for edges
-        for (int i = 0; i < P; i++)  // (0,1)
-        {
-          nodes[o][0] = eop[i];  nodes[o][1] = 0.;  nodes[o][2] = 0.;
-          dof2tk[o++] = 0;
-        }
-        for (int i = 0; i < P; i++)  // (1,2)
-        {
-          nodes[o][0] = eop[pm1-i];  nodes[o][1] = eop[i];  nodes[o][2] = 0.;
-          dof2tk[o++] = 1;
-        }
-        for (int i = 0; i < P; i++)  // (2,0)
-        {
-          nodes[o][0] = 0.;  nodes[o][1] = eop[pm1-i];  nodes[o][2] = 0.;
-          dof2tk[o++] = 2;
-        }
-        for (int i = 0; i < P; i++)  // (0,3)
-        {
-          nodes[o][0] = 0.;  nodes[o][1] = 0.;  nodes[o][2] = eop[i];
-          dof2tk[o++] = 3;
-        }
-        for (int i = 0; i < P; i++)  // (1,3)
-        {
-          nodes[o][0] = eop[pm1-i];  nodes[o][1] = 0.;  nodes[o][2] = eop[i];
-          dof2tk[o++] = 4;
-        }
-        for (int i = 0; i < P; i++)  // (2,3)
-        {
-          nodes[o][0] = 0.;  nodes[o][1] = eop[pm1-i];  nodes[o][2] = eop[i];
-          dof2tk[o++] = 5;
-        }
-
-      	// Face loops to get nodes and dof2tk for faces
-        for (int j = 0; j <= pm2; j++)  // (0,1,2)
-          for (int i = 0; i + j <= pm2; i++)
-          {
-            double w = fop[i] + fop[j] + fop[pm2-i-j];
-            nodes[o][0] = fop[i]/w;  nodes[o][1] = fop[j]/w;  nodes[o][2] = 0.;
-            dof2tk[o++] = 0;
-            nodes[o][0] = fop[i]/w;  nodes[o][1] = fop[j]/w;  nodes[o][2] = 0.;
-            dof2tk[o++] = 2;
-          }
-        for (int j = 0; j <= pm2; j++)  // (0,1,3)
-          for (int i = 0; i + j <= pm2; i++)
-          {
-            double w = fop[i] + fop[j] + fop[pm2-i-j];
-            nodes[o][0] = fop[i]/w;  nodes[o][1] = 0.;  nodes[o][2] = fop[j]/w;
-            dof2tk[o++] = 0;
-            nodes[o][0] = fop[i]/w;  nodes[o][1] = 0.;  nodes[o][2] = fop[j]/w;
-            dof2tk[o++] = 3;
-          }
-        for (int j = 0; j <= pm2; j++)  // (1,2,3)
-          for (int i = 0; i + j <= pm2; i++)
-          {
-            double w = fop[i] + fop[j] + fop[pm2-i-j];
-            nodes[o][0] = fop[pm2-i-j]/w;  nodes[o][1] = fop[i]/w;  nodes[o][2] = fop[j]/w;
-            dof2tk[o++] = 1;
-            nodes[o][0] = fop[pm2-i-j]/w;  nodes[o][1] = fop[i]/w;  nodes[o][2] = fop[j]/w;
-            dof2tk[o++] = 4;
-          }
-        for (int j = 0; j <= pm2; j++)  // (0,2,3)
-          for (int i = 0; i + j <= pm2; i++)
-          {
-            double w = fop[i] + fop[j] + fop[pm2-i-j];
-            nodes[o][0] = 0.;  nodes[o][1] = fop[i]/w;  nodes[o][2] = fop[j]/w;
-            dof2tk[o++] = 2;
-            nodes[o][0] = 0.;  nodes[o][1] = fop[i]/w;  nodes[o][2] = fop[j]/w;
-            dof2tk[o++] = 3;
-          }
-
-      	// Region loops to get nodes and dof2tk for regions
-        for (int k = 0; k <= pm3; k++)
-          for (int j = 0; j + k <= pm3; j++)
-            for (int i = 0; i + j + k <= pm3; i++)
-              {
-                double w = iop[i] + iop[j] + iop[k] + iop[pm3-i-j-k];
-                nodes[o][0] = iop[i]/w;  nodes[o][1] = iop[j]/w;  nodes[o][2] = iop[k]/w;
-                dof2tk[o++] = 0;
-                nodes[o][0] = iop[i]/w;  nodes[o][1] = iop[j]/w;  nodes[o][2] = iop[k]/w;
-                dof2tk[o++] = 2;
-                nodes[o][0] = iop[i]/w;  nodes[o][1] = iop[j]/w;  nodes[o][2] = iop[k]/w;
-                dof2tk[o++] = 3;
-              }
-
-      	// Populate T
-      	mth::Matrix<double> T(non,non); // T(i,j)
-        for (int m = 0; m < non; m++)
-        {
-          const double *tm = tk + 3*dof2tk[m];
-          o = 0;
-
-          double x = nodes[o][0]; double y = nodes[o][1]; double z = nodes[o][2];
-
-          getChebyshevT(pm1, x, &shape_x[0]);
-          getChebyshevT(pm1, y, &shape_y[0]);
-          getChebyshevT(pm1, z, &shape_z[0]);
-          getChebyshevT(pm1, 1. - x - y - z, &shape_l[0]);
-
-          for (int k = 0; k <= pm1; k++)
-            for (int j = 0; j + k <= pm1; j++)
-              for (int i = 0; i + j + k <= pm1; i++)
-              {
-                double s = shape_x[i]*shape_y[j]*shape_z[k]*shape_l[pm1-i-j-k];
-                T(o++, m) = s * tm[0];
-                T(o++, m) = s * tm[1];
-                T(o++, m) = s * tm[2];
-              }
-          for (int k = 0; k <= pm1; k++)
-            for (int j = 0; j + k <= pm1; j++)
-              {
-                double s = shape_x[pm1-j-k]*shape_y[j]*shape_z[k];
-                T(o++, m) = s*((y - c)*tm[0] - (x - c)*tm[1]);
-                T(o++, m) = s*((z - c)*tm[0] - (x - c)*tm[2]);
-              }
-          for (int k = 0; k <= pm1; k++)
-          {
-            T(o++, m) =
-              shape_y[pm1-k]*shape_z[k]*((z - c)*tm[1] - (y - c)*tm[2]);
-          }
-        }
-
-        mth::decomposeQR(T, Q, R);
-      }
     };
     EntityShape* getEntityShape(int type)
     {

From 6af07b31b85a58049f06af858199c4bf74ee207f Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Wed, 25 Mar 2020 22:57:38 -0400
Subject: [PATCH 076/555] Removes `getGlobalVectorGradients`

---
 apf/apfElement.cc | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/apf/apfElement.cc b/apf/apfElement.cc
index 731ace661..82c89c602 100644
--- a/apf/apfElement.cc
+++ b/apf/apfElement.cc
@@ -85,14 +85,6 @@ void Element::getGlobalGradients(Vector3 const& local,
     globalGradients[i] = jinv * localGradients[i];
 }
 
-void Element::getGlobalVectorGradients(Vector3 const& local,
-                                 NewArray<Matrix3x3>& globalGradients)
-{
-  // TODO
-  // the same as above but for vector shape functions
-  // using getLocalVectorGradients in apfNedelec.cc
-}
-
 void Element::getComponents(Vector3 const& xi, double* c)
 {
   NewArray<double> shapeValues;

From a51a4e565178b1ecbfdab576e53239cf8b750f96 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Thu, 26 Mar 2020 00:10:22 -0400
Subject: [PATCH 077/555] Implements hasNodesIn and countNodes on + cleanup

Also adds comments as to how the number of nodes is computed.
---
 apf/apfNedelec.cc | 46 ++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 38 insertions(+), 8 deletions(-)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index e196e43c1..c3f4e8116 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -116,7 +116,6 @@ const double* getClosedPoints(int order, const int type = GAUSS_LOBATTO)
 
 void getChebyshevT(int order, double xi, double* u)
 {
-  // TODO implement Chebyshev
   // recursive definition, z in [-1,1]
   // T_0(z) = 1,  T_1(z) = z
   // T_{n+1}(z) = 2*z*T_n(z) - T_{n-1}(z)
@@ -180,13 +179,31 @@ void getChebyshevT(int order, double xi, double* u, double* d, double* dd)
   }
 }
 
+// This is all nodes, including the nodes associated with bounding edges
 static inline int countTriNodes(int P)
 {
-  return P*(P+2);
+  // each node on an edge has 1 dof
+  // each node on a face has 2 dofs
+  // each term in the following can be understood as follows
+  // e*dofs*nodes
+  // e      := # of entities of that dimension
+  // dofs   := # of dofs associated with entities of that dimension
+  // nodes  := # of nodes associated with entities of that dimension
+  return 3*1*P + 1*2*P*(P-1)/2;
 }
+// This is all nodes, including the nodes associated with bounding edges and faces
 static inline int countTetNodes(int P)
 {
-  return (P+3)*(P+2)*P/2;
+  // each node on an edge has 1 dof
+  // each node on a face has 2 dofs
+  // each node on a tet has 3 dofs
+  // each term in the following can be understood as follows
+  // e*dofs*nodes
+  // e      := # of entities of that dimension
+  // dofs   := # of dofs associated with entities of that dimension
+  // nodes  := # of nodes associated with entities of that dimension
+  if (P<2) return 6*1*P + 4*2*P*(P-1)/2;
+  return 6*1*P + 4*2*P*(P-1)/2 + 1*3*P*(P-1)*(P-2)/6;
 }
 
 static void computeTriangleTi(
@@ -857,14 +874,27 @@ class Nedelec: public FieldShape {
       };
       return shapes[type];
     }
-    bool hasNodesIn(int)
+    // For the following to member functions we only need to
+    // consider the interior nodes, i.e.,
+    // Faces: no need to count the nodes associated with bounding edges
+    // Tets: no need to count the nodes associated with bounding edges/faces
+    // TODO: The above description is consistent with how things are done
+    // for other fields in pumi. We need to make sure this will not cause
+    // any problems for Nedelec fields
+    bool hasNodesIn(int dimension)
     {
-      // TODO complete this
-      return true;
+      if (dimension == 1) return true;
+      if (dimension == 2) return P > 1;
+      if (dimension == 3) return P > 2;
+      // if not returned by now dimension should be 0, so return false
+      return false;
     }
-    int countNodesOn(int)
+    int countNodesOn(int type)
     {
-      // TODO complete this
+      if (type == apf::Mesh::EDGE) return P;
+      if (type == apf::Mesh::TRIANGLE) return 2*P*(P-1)/2;
+      if (type == apf::Mesh::TET && P>2) return 3*P*(P-1)*(P-2)/6;
+      // for any other case (type and P combination) return 0;
       return 0;
     }
 };

From 332f96356e2ff6b1389faff93fbcbc6b1e7ca59b Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Thu, 26 Mar 2020 23:36:05 -0400
Subject: [PATCH 078/555] Makes getVectorValues pure virtual in EntityShape

getVectorValues has been added to handle Nedlec elements (these have
vector shape functions). To keep everything consistent we added
getVectorValues to the EntityShape class as a pure virtual member
function. All the subclasses of EntityShape are have an (empty)
implementation of this functions.
---
 apf/apf.cc           |  6 ---
 apf/apf.h            |  8 ----
 apf/apfElement.h     |  2 -
 apf/apfHierarchic.cc | 14 ++++++-
 apf/apfIPShape.cc    | 30 +++++++++++++++
 apf/apfShape.cc      | 90 ++++++++++++++++++++++++++++++++++++++++++++
 apf/apfShape.h       |  2 +-
 crv/crvBezier.cc     | 30 +++++++++++++++
 8 files changed, 164 insertions(+), 18 deletions(-)

diff --git a/apf/apf.cc b/apf/apf.cc
index c6d2476a1..c981988f7 100644
--- a/apf/apf.cc
+++ b/apf/apf.cc
@@ -416,12 +416,6 @@ void getVectorShapeValues(Element* e, Vector3 const& local,
   e->getShape()->getVectorValues(e->getMesh(), e->getEntity(), local,values);
 }
 
-void getVectorShapeGrads(Element* e, Vector3 const& local,
-    NewArray<Matrix3x3>& grads)
-{
-  e->getGlobalVectorGradients(local,grads);
-}
-
 FieldShape* getShape(Field* f)
 {
   return f->getShape();
diff --git a/apf/apf.h b/apf/apf.h
index 410d1bf2e..9e9e567e3 100644
--- a/apf/apf.h
+++ b/apf/apf.h
@@ -609,14 +609,6 @@ void getShapeGrads(Element* e, Vector3 const& local,
 void getVectorShapeValues(Element* e, Vector3 const& local,
     NewArray<Vector3>& values);
 
-/** \brief Returns the vector shape function gradients at a point
-  *
-  * \details used only for Nedelec shapes
-  */
-void getVectorShapeGrads(Element* e, Vector3 const& local,
-    NewArray<Matrix3x3>& grads);
-
-
 /** \brief Retrieve the apf::FieldShape used by a field
   */
 FieldShape* getShape(Field* f);
diff --git a/apf/apfElement.h b/apf/apfElement.h
index 953fc08f8..f100d42e8 100644
--- a/apf/apfElement.h
+++ b/apf/apfElement.h
@@ -25,8 +25,6 @@ class Element
     virtual ~Element();
     void getGlobalGradients(Vector3 const& local,
                             NewArray<Vector3>& globalGradients);
-    void getGlobalVectorGradients(Vector3 const& local,
-                            NewArray<Matrix3x3>& globalGradients);
     int getType() {return mesh->getType(entity);}
     int getDimension() {return Mesh::typeDimension[getType()];}
     int getOrder() {return field->getShape()->getOrder();}
diff --git a/apf/apfHierarchic.cc b/apf/apfHierarchic.cc
index 830a68aac..c864a8224 100644
--- a/apf/apfHierarchic.cc
+++ b/apf/apfHierarchic.cc
@@ -32,6 +32,8 @@ class HVertex : public EntityShape {
     }
     void getLocalGradients(
         Mesh*, MeshEntity*, Vector3 const&, NewArray<Vector3>&) const {}
+    void getVectorValues(
+        Mesh*, MeshEntity*, Vector3 const&, NewArray<Vector3>&) const {}
     int countNodes() const { return 1; }
 };
 
@@ -51,6 +53,8 @@ class HEdge2 : public EntityShape {
       dN[1] = Vector3(0.5, 0.0, 0.0);
       dN[2] = Vector3(-0.5 * c0 * xi[0], 0.0, 0.0);
     }
+    void getVectorValues(
+        Mesh*, MeshEntity*, Vector3 const&, NewArray<Vector3>&) const {}
     int countNodes() const { return 3; }
 };
 
@@ -72,6 +76,8 @@ class HEdge3 : public EntityShape {
       dN[2] = Vector3(-0.5 * c0 * xi[0], 0.0, 0.0);
       dN[3] = Vector3(0.25 * c0 * (1.0 - 3.0*xi[0]*xi[0]), 0.0, 0.0);
     }
+    void getVectorValues(
+        Mesh*, MeshEntity*, Vector3 const&, NewArray<Vector3>&) const {}
     int countNodes() const { return 4; }
 };
 
@@ -105,6 +111,8 @@ class HTriangle2 : public EntityShape {
       dN[4] = Vector3(c0 * xi[1], c0 * xi[0], 0.0);
       dN[5] = Vector3(-c0 * xi[1], c0 * (1.0 - xi[0] - 2.0*xi[1]), 0.0);
     }
+    void getVectorValues(
+        Mesh*, MeshEntity*, Vector3 const&, NewArray<Vector3>&) const {}
     int countNodes() const { return 6; }
 };
 
@@ -152,7 +160,7 @@ class HTriangle3 : public EntityShape {
       N[9] = l0*l1*l2;
     }
     void getLocalGradients(
-        Mesh* m, MeshEntity* e, Vector3 const& xi, NewArray<Vector3>& dN) const {
+        Mesh* m, MeshEntity* e, Vector3 const& xi, NewArray<Vector3>& dN) const    {
       dN.allocate(10);
 
       /* edge orientations */
@@ -197,6 +205,8 @@ class HTriangle3 : public EntityShape {
       /* face */
       dN[9] = dl0*l1*l2 + dl1*l0*l2 + dl2*l0*l1;
     }
+    void getVectorValues(
+        Mesh*, MeshEntity*, Vector3 const&, NewArray<Vector3>&) const {}
     void alignSharedNodes(Mesh*, MeshEntity*, MeshEntity*, int order[])
     {
       /* unlike Lagrange shape functions, hierarchic 'modes' do not
@@ -247,6 +257,8 @@ class HTetrahedron2 : public EntityShape {
       dN[8] = Vector3( xi[2], 0.0, xi[0] ) * c0;
       dN[9] = Vector3( 0.0, xi[2], xi[1] ) * c0;
     }
+    void getVectorValues(
+        Mesh*, MeshEntity*, Vector3 const&, NewArray<Vector3>&) const {}
     int countNodes() const {return 10;}
 };
 
diff --git a/apf/apfIPShape.cc b/apf/apfIPShape.cc
index 3d6bce1b3..e55679bfb 100644
--- a/apf/apfIPShape.cc
+++ b/apf/apfIPShape.cc
@@ -162,6 +162,12 @@ class VoronoiShape : public IPBase
         {
           fail("gradients not defined for Voronoi shapes");
         }
+        void getVectorValues(Mesh*, MeshEntity*,
+            Vector3 const&,
+            NewArray<Vector3>&) const
+        {
+          fail("getVectorValues not defined for Voronoi shapes");
+        }
         int countNodes() const
         {
           return points.size();
@@ -227,6 +233,12 @@ class ConstantIPFit : public IPBase
           grads.allocate(1);
           grads[0] = Vector3(0,0,0);
         }
+        void getVectorValues(Mesh*, MeshEntity*,
+            Vector3 const&,
+            NewArray<Vector3>&) const
+        {
+          fail("getVectorValues not defined for ConstantIPFit shapes");
+        }
         int countNodes() const {return 1;}
     };
     class Tetrahedron : public EntityShape
@@ -244,6 +256,12 @@ class ConstantIPFit : public IPBase
           grads.allocate(1);
           grads[0] = Vector3(0,0,0);
         }
+        void getVectorValues(Mesh*, MeshEntity*,
+            Vector3 const&,
+            NewArray<Vector3>&) const
+        {
+          fail("getVectorValues not defined for ConstantIPFit shapes");
+        }
         int countNodes() const {return 1;}
     };
     EntityShape* getEntityShape(int type)
@@ -308,6 +326,12 @@ class LinearIPFit : public IPBase
         {
           fail("grads not implemented yet");
         }
+        void getVectorValues(Mesh*, MeshEntity*,
+            Vector3 const&,
+            NewArray<Vector3>&) const
+        {
+          fail("getVectorValues not defined for LinearIPFit shapes");
+        }
         int countNodes() const {return 3;}
     };
     class Tetrahedron : public EntityShape
@@ -338,6 +362,12 @@ class LinearIPFit : public IPBase
         {
           fail("grads not implemented yet");
         }
+        void getVectorValues(Mesh*, MeshEntity*,
+            Vector3 const&,
+            NewArray<Vector3>&) const
+        {
+          fail("getVectorValues not defined for LinearIPFit shapes");
+        }
         int countNodes() const {return 4;}
     };
     EntityShape* getEntityShape(int type)
diff --git a/apf/apfShape.cc b/apf/apfShape.cc
index 7add80ff7..566f3ad75 100644
--- a/apf/apfShape.cc
+++ b/apf/apfShape.cc
@@ -86,6 +86,11 @@ class Linear : public FieldShape
             Vector3 const&, NewArray<Vector3>&) const
         {
         }
+        void getVectorValues(Mesh*, MeshEntity*,
+            Vector3 const&, NewArray<Vector3>&) const
+	{
+	  fail("getVectorValues not defined for nodal shapes");
+	}
         int countNodes() const {return 1;}
     };
     class Edge : public EntityShape
@@ -105,6 +110,11 @@ class Linear : public FieldShape
           grads[0] = Vector3(-0.5,0,0);
           grads[1] = Vector3( 0.5,0,0);
         }
+        void getVectorValues(Mesh*, MeshEntity*,
+            Vector3 const&, NewArray<Vector3>&) const
+	{
+	  fail("getVectorValues not defined for nodal shapes");
+	}
         int countNodes() const {return 2;}
     };
     class Triangle : public EntityShape
@@ -126,6 +136,11 @@ class Linear : public FieldShape
           grads[1] = Vector3( 1, 0,0);
           grads[2] = Vector3( 0, 1,0);
         }
+        void getVectorValues(Mesh*, MeshEntity*,
+            Vector3 const&, NewArray<Vector3>&) const
+	{
+	  fail("getVectorValues not defined for nodal shapes");
+	}
         int countNodes() const {return 3;}
     };
     class Quad : public EntityShape
@@ -158,6 +173,11 @@ class Linear : public FieldShape
           grads[2] = Vector3( l1y, l1x,0)/4;
           grads[3] = Vector3(-l1y, l0x,0)/4;
         }
+        void getVectorValues(Mesh*, MeshEntity*,
+            Vector3 const&, NewArray<Vector3>&) const
+	{
+	  fail("getVectorValues not defined for nodal shapes");
+	}
         int countNodes() const {return 4;}
     };
     class Tetrahedron : public EntityShape
@@ -181,6 +201,11 @@ class Linear : public FieldShape
           grads[2] = Vector3( 0, 1, 0);
           grads[3] = Vector3( 0, 0, 1);
         }
+        void getVectorValues(Mesh*, MeshEntity*,
+            Vector3 const&, NewArray<Vector3>&) const
+	{
+	  fail("getVectorValues not defined for nodal shapes");
+	}
         int countNodes() const {return 4;}
     };
     class Prism : public EntityShape
@@ -228,6 +253,11 @@ class Linear : public FieldShape
           grads[4] = tg[1] * up   + eg[1] * nt[1];
           grads[5] = tg[2] * up   + eg[1] * nt[2];
         }
+        void getVectorValues(Mesh*, MeshEntity*,
+            Vector3 const&, NewArray<Vector3>&) const
+	{
+	  fail("getVectorValues not defined for nodal shapes");
+	}
         int countNodes() const {return 6;}
     };
     class Pyramid : public EntityShape
@@ -265,6 +295,11 @@ class Linear : public FieldShape
           grads[3] = Vector3(-l1y * l0z,  l0x * l0z, -l0x * l1y) / 8;
           grads[4] = Vector3(0,0,0.5);
         }
+        void getVectorValues(Mesh*, MeshEntity*,
+            Vector3 const&, NewArray<Vector3>&) const
+	{
+	  fail("getVectorValues not defined for nodal shapes");
+	}
         int countNodes() const {return 5;}
     };
     class Hexahedron : public EntityShape
@@ -309,6 +344,11 @@ class Linear : public FieldShape
           grads[6] = Vector3( l1y * l1z,  l1x * l1z,  l1x * l1y) / 8;
           grads[7] = Vector3(-l1y * l1z,  l0x * l1z,  l0x * l1y) / 8;
         }
+        void getVectorValues(Mesh*, MeshEntity*,
+            Vector3 const&, NewArray<Vector3>&) const
+	{
+	  fail("getVectorValues not defined for nodal shapes");
+	}
         int countNodes() const {return 8;}
     };
     EntityShape* getEntityShape(int type)
@@ -376,6 +416,11 @@ class QuadraticBase : public FieldShape
           grads[1] = Vector3((2*xi[0]+1)/2.0,0,0);
           grads[2] = Vector3(-2*xi[0],0,0);
         }
+        void getVectorValues(Mesh*, MeshEntity*,
+            Vector3 const&, NewArray<Vector3>&) const
+	{
+	  fail("getVectorValues not defined for nodal shapes");
+	}
         int countNodes() const {return 3;}
     };
     class Triangle : public EntityShape
@@ -406,6 +451,11 @@ class QuadraticBase : public FieldShape
           grads[4] = Vector3(4*xi[1],4*xi[0],0);
           grads[5] = Vector3(-4*xi[1],4*(xi2-xi[1]),0);
         }
+        void getVectorValues(Mesh*, MeshEntity*,
+            Vector3 const&, NewArray<Vector3>&) const
+	{
+	  fail("getVectorValues not defined for nodal shapes");
+	}
         int countNodes() const {return 6;}
     };
     class Tetrahedron : public EntityShape
@@ -445,6 +495,11 @@ class QuadraticBase : public FieldShape
           grads[8] = Vector3(4*xi[2],0,4*xi[0]);
           grads[9] = Vector3(0,4*xi[2],4*xi[1]);
         }
+        void getVectorValues(Mesh*, MeshEntity*,
+            Vector3 const&, NewArray<Vector3>&) const
+	{
+	  fail("getVectorValues not defined for nodal shapes");
+	}
         int countNodes() const {return 10;}
     };
     EntityShape* getEntityShape(int type)
@@ -544,6 +599,11 @@ class LagrangeQuadratic : public QuadraticBase
               -2.0*e*(1-(n*n)),
               -2.0*n*(1-(e*e)), 0.0);
         }
+        void getVectorValues(Mesh*, MeshEntity*,
+            Vector3 const&, NewArray<Vector3>&) const
+	{
+	  fail("getVectorValues not defined for nodal shapes");
+	}
         int countNodes() const {return 9;}
     };
     EntityShape* getEntityShape(int type)
@@ -616,6 +676,11 @@ class SerendipityQuadratic : public QuadraticBase
         grads[6] = Vector3(-xi[0] - xi[0]*xi[1],  0.5 - xi[0]*xi[0]/2.0, 0.0);
         grads[7] = Vector3(xi[1]*xi[1]/2.0 - 0.5,   xi[0]*xi[1] - xi[1], 0.0);
       }
+      void getVectorValues(Mesh*, MeshEntity*,
+	  Vector3 const&, NewArray<Vector3>&) const
+      {
+	  fail("getVectorValues not defined for nodal shapes");
+      }
       int countNodes() const {return 8;}
     };
     EntityShape* getEntityShape(int type)
@@ -661,6 +726,11 @@ class LagrangeCubic : public FieldShape
             Vector3 const&, NewArray<Vector3>&) const
         {
         }
+        void getVectorValues(Mesh*, MeshEntity*,
+            Vector3 const&, NewArray<Vector3>&) const
+	{
+	  fail("getVectorValues not defined for nodal shapes");
+	}
         int countNodes() const {return 1;}
     };
     class Edge : public EntityShape
@@ -686,6 +756,11 @@ class LagrangeCubic : public FieldShape
           dN[2] = Vector3(27./16.*(3.*xi*xi-2./3.*xi-1.), 0, 0);
           dN[3] = Vector3(27./16.*(-3.*xi*xi-2./3.*xi+1.), 0, 0);
         }
+        void getVectorValues(Mesh*, MeshEntity*,
+            Vector3 const&, NewArray<Vector3>&) const
+	{
+	  fail("getVectorValues not defined for nodal shapes");
+	}
         int countNodes() const {return 4;}
     };
     class Triangle : public EntityShape
@@ -730,6 +805,11 @@ class LagrangeCubic : public FieldShape
           dN[8] = (gl3*(3.*l1*l1-l1) + gl1*(6.*l3*l1-l3))*9./2.;
           dN[9] = (gl1*l2*l3+gl2*l1*l3+gl3*l1*l2)*27.;
         }
+        void getVectorValues(Mesh*, MeshEntity*,
+            Vector3 const&, NewArray<Vector3>&) const
+	{
+	  fail("getVectorValues not defined for nodal shapes");
+	}
         int countNodes() const {return 10;}
         void alignSharedNodes(Mesh* m, MeshEntity* elem,
             MeshEntity* edge, int order[]) {
@@ -811,6 +891,11 @@ class LagrangeCubic : public FieldShape
           dN[18] = (gl1*l2*l3 + gl2*l1*l3 + gl3*l1*l2)*27.;
           dN[19] = (gl0*l2*l3 + gl2*l0*l3 + gl3*l0*l2)*27.;
         }
+        void getVectorValues(Mesh*, MeshEntity*,
+            Vector3 const&, NewArray<Vector3>&) const
+	{
+	  fail("getVectorValues not defined for nodal shapes");
+	}
         int countNodes() const {return 20;}
         void alignSharedNodes(Mesh* m, MeshEntity* elem,
             MeshEntity* edge, int order[]) {
@@ -933,6 +1018,11 @@ class Constant : public FieldShape
           grads.allocate(1);
           grads[0] = Vector3( 0, 0, 0);
         }
+        void getVectorValues(Mesh*, MeshEntity*,
+            Vector3 const&, NewArray<Vector3>&) const
+	{
+	  fail("getVectorValues not defined for constant shapes");
+	}
         int countNodes() const {return 1;}
         int getDimension() const {return D;}
     };
diff --git a/apf/apfShape.h b/apf/apfShape.h
index 9f0fc23bc..1f65030a1 100644
--- a/apf/apfShape.h
+++ b/apf/apfShape.h
@@ -50,7 +50,7 @@ class EntityShape
         Mesh* m,
         MeshEntity* e,
         Vector3 const& xi,
-        NewArray<apf::Vector3>& values) const;
+        NewArray<apf::Vector3>& values) const = 0;
 /** \brief return the number of nodes affecting this element
     \details in a linear mesh, there are two nodes affecting
              and edge, three nodes affecting a triangle,
diff --git a/crv/crvBezier.cc b/crv/crvBezier.cc
index 6007d669a..99c84f8ac 100644
--- a/crv/crvBezier.cc
+++ b/crv/crvBezier.cc
@@ -90,6 +90,11 @@ class Bezier : public apf::FieldShape
         apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
     {
     }
+    void getVectorValues(apf::Mesh*, apf::MeshEntity*,
+        apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
+    {
+      fail("getVectorValues is not implemented for Bezier shapes");
+    }
     int countNodes() const {return 1;}
     void alignSharedNodes(apf::Mesh*,
         apf::MeshEntity*, apf::MeshEntity*, int order[])
@@ -112,6 +117,11 @@ class Bezier : public apf::FieldShape
       grads.allocate(P+1);
       bezierGrads[apf::Mesh::EDGE](P,xi,grads);
     }
+    void getVectorValues(apf::Mesh*, apf::MeshEntity*,
+        apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
+    {
+      fail("getVectorValues is not implemented for Bezier shapes");
+    }
     int countNodes() const {return P+1;}
     void alignSharedNodes(apf::Mesh*,
         apf::MeshEntity*, apf::MeshEntity*, int order[])
@@ -146,6 +156,11 @@ class Bezier : public apf::FieldShape
         BlendedTriangleGetLocalGradients(m,e,xi,grads);
 
     }
+    void getVectorValues(apf::Mesh*, apf::MeshEntity*,
+        apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
+    {
+      fail("getVectorValues is not implemented for Bezier shapes");
+    }
     int countNodes() const {return getNumControlPoints(apf::Mesh::TRIANGLE,P);}
     void alignSharedNodes(apf::Mesh* m,
         apf::MeshEntity* elem, apf::MeshEntity* shared, int order[])
@@ -179,6 +194,11 @@ class Bezier : public apf::FieldShape
         BlendedTetGetLocalGradients(m,e,xi,grads);
       }
     }
+    void getVectorValues(apf::Mesh*, apf::MeshEntity*,
+        apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
+    {
+      fail("getVectorValues is not implemented for Bezier shapes");
+    }
     int countNodes() const {
       if(!useBlending(apf::Mesh::TET))
         return (P+1)*(P+2)*(P+3)/6;
@@ -372,6 +392,11 @@ class GregorySurface4 : public apf::FieldShape
         BlendedTriangleGetLocalGradients(m,e,xi,grads);
 
     }
+    void getVectorValues(apf::Mesh*, apf::MeshEntity*,
+        apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
+    {
+      fail("getVectorValues is not implemented for Gregory shapes");
+    }
     int countNodes() const
     {
       return 18;
@@ -513,6 +538,11 @@ class GregorySurface4 : public apf::FieldShape
         BlendedTetGetLocalGradients(m,e,xi,grads);
       }
     }
+    void getVectorValues(apf::Mesh*, apf::MeshEntity*,
+        apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
+    {
+      fail("getVectorValues is not implemented for Gregory shapes");
+    }
     int countNodes() const {
       if(!useBlending(apf::Mesh::TET))
         return 47;

From 1f0ded0a3c8bf7f0259cd2e68ea674adbaa05024 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Fri, 27 Mar 2020 22:18:45 -0400
Subject: [PATCH 079/555] Fixed bugs.

---
 apf/apfNedelec.cc | 45 +++++++++++++++++++++++++--------------------
 apf/apfShape.h    |  2 +-
 2 files changed, 26 insertions(+), 21 deletions(-)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index c3f4e8116..34551c2d1 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -16,6 +16,7 @@
 #include <PCU.h>
 
 #include <iostream>
+using namespace std;
 
 namespace apf {
 
@@ -263,7 +264,7 @@ static void computeTriangleTi(
     const double *tm = tk + 2*dof2tk[m];
     o = 0;
 
-    double x = nodes[o][0]; double y = nodes[o][1];
+    double x = nodes[m][0]; double y = nodes[m][1];
 
     getChebyshevT(pm1, x, &shape_x[0]);
     getChebyshevT(pm1, y, &shape_y[0]);
@@ -317,32 +318,32 @@ static void computeTetTi(
 
   int o = 0;
   // Edge loops to get nodes and dof2tk for edges
-  for (int i = 0; i < P; i++)  // (0,1)
+  for (int i = 0; i < p; i++)  // (0,1)
   {
     nodes[o][0] = eop[i];  nodes[o][1] = 0.;  nodes[o][2] = 0.;
     dof2tk[o++] = 0;
   }
-  for (int i = 0; i < P; i++)  // (1,2)
+  for (int i = 0; i < p; i++)  // (1,2)
   {
     nodes[o][0] = eop[pm1-i];  nodes[o][1] = eop[i];  nodes[o][2] = 0.;
     dof2tk[o++] = 1;
   }
-  for (int i = 0; i < P; i++)  // (2,0)
+  for (int i = 0; i < p; i++)  // (2,0)
   {
     nodes[o][0] = 0.;  nodes[o][1] = eop[pm1-i];  nodes[o][2] = 0.;
     dof2tk[o++] = 2;
   }
-  for (int i = 0; i < P; i++)  // (0,3)
+  for (int i = 0; i < p; i++)  // (0,3)
   {
     nodes[o][0] = 0.;  nodes[o][1] = 0.;  nodes[o][2] = eop[i];
     dof2tk[o++] = 3;
   }
-  for (int i = 0; i < P; i++)  // (1,3)
+  for (int i = 0; i < p; i++)  // (1,3)
   {
     nodes[o][0] = eop[pm1-i];  nodes[o][1] = 0.;  nodes[o][2] = eop[i];
     dof2tk[o++] = 4;
   }
-  for (int i = 0; i < P; i++)  // (2,3)
+  for (int i = 0; i < p; i++)  // (2,3)
   {
     nodes[o][0] = 0.;  nodes[o][1] = eop[pm1-i];  nodes[o][2] = eop[i];
     dof2tk[o++] = 5;
@@ -416,7 +417,7 @@ static void computeTetTi(
     const double *tm = tk + 3*dof2tk[m];
     o = 0;
 
-    double x = nodes[o][0]; double y = nodes[o][1]; double z = nodes[o][2];
+    double x = nodes[m][0]; double y = nodes[m][1]; double z = nodes[m][2];
 
     getChebyshevT(pm1, x, &shape_x[0]);
     getChebyshevT(pm1, y, &shape_y[0]);
@@ -449,6 +450,7 @@ static void computeTetTi(
         shape_y[pm1-k]*shape_z[k]*((z - c)*tm[1] - (y - c)*tm[2]);
     }
   }
+
   mth::decomposeQR(T, Q, R);
 }
 
@@ -474,10 +476,13 @@ static void getTi(
     type == apf::Mesh::TRIANGLE ?
     	  computeTriangleTi(P, LQ, LR) : computeTetTi(P, LQ, LR);
 
+    transformQ[type][P].allocate(n*n);
+    transformR[type][P].allocate(n*n);
+
     for (int i = 0; i < n; i++) {
       for (int j = 0; j < n; j++) {
-	transformQ[type][P][i*n+j] = LQ(i,j);
-	transformR[type][P][i*n+j] = LR(i,j);
+	      transformQ[type][P][i*n+j] = LQ(i,j);
+      	transformR[type][P][i*n+j] = LR(i,j);
       }
     }
   }
@@ -616,9 +621,9 @@ class Nedelec: public FieldShape {
         {
           mth::Vector<double> B (dof);
           mth::Vector<double> X (dof);
-          for (int j = 0; j < dof; j++) B[j] = u(i,j); // populate b in QR x = b
+          for (int j = 0; j < dof; j++) B[j] = u(j,i); // populate b in QR x = b
           mth::solveFromQR(Q, R, B, X);
-          for (int j = 0; j < dof; j++)  S(i,j) = X[j]; // populate S with x
+          for (int j = 0; j < dof; j++)  S(j,i) = X[j]; // populate S with x
         }
 
         shapes.allocate(dof);
@@ -745,18 +750,18 @@ class Nedelec: public FieldShape {
           u(n,0) = 0.;  u(n,1) = s*(z - c);  u(n,2) = -s*(y - c);  n++;
         }
 
-	mth::Matrix<double> Q(dof, dof);
-	mth::Matrix<double> R(dof, dof);
-	getTi(P, apf::Mesh::TET, Q, R);
+      	mth::Matrix<double> Q(dof, dof);
+      	mth::Matrix<double> R(dof, dof);
+      	getTi(P, apf::Mesh::TET, Q, R);
 
         mth::Matrix<double> S(dof, dim);
       	for(int i = 0; i < dim; i++) // S = Ti * u
         {
           mth::Vector<double> B (dof);
           mth::Vector<double> X (dof);
-          for (int j = 0; j < dof; j++) B[j] = u(i,j); // populate b
+          for (int j = 0; j < dof; j++) B[j] = u(j,i); // populate b
           mth::solveFromQR(Q, R, B, X);
-          for (int j = 0; j < dof; j++)  S(i,j) = X[j]; // populate S with x
+          for (int j = 0; j < dof; j++)  S(j,i) = X[j]; // populate S with x
         }
 
         shapes.allocate(dof);
@@ -845,9 +850,9 @@ class Nedelec: public FieldShape {
         {
           mth::Vector<double> B(dof);
           mth::Vector<double> X(dof);
-          for (int j = 0; j < dof; j++) B[j] = u(i,j); // populate b
+          for (int j = 0; j < dof; j++) B[j] = u(j,i); // populate b
           mth::solveFromQR(Q, R, B, X);
-          for (int j = 0; j < dof; j++)  S(i,j) = X[j]; // populate S with x
+          for (int j = 0; j < dof; j++)  S(j,i) = X[j]; // populate S with x
         }
 
         curl_shapes.allocate(dof);
@@ -917,7 +922,7 @@ apf::FieldShape* getNedelec(int order)
   static Nedelec<10> ND10;
   static FieldShape* const nedelecShapes[10] =
   {&ND1, &ND2, &ND3, &ND4, &ND5, &ND6, &ND7, &ND8, &ND9, &ND10};
-  return nedelecShapes[order];
+  return nedelecShapes[order-1];
 }
 
 };
diff --git a/apf/apfShape.h b/apf/apfShape.h
index 1f65030a1..230455e38 100644
--- a/apf/apfShape.h
+++ b/apf/apfShape.h
@@ -43,7 +43,7 @@ class EntityShape
         Vector3 const& xi,
         NewArray<Vector3>& grads) const = 0;
 /** \brief evaluate element vector shape functions
- \details this is use only for Nedelc
+ \details this is used only for Nedelec
  \param xi the parent element coordinates
  \param values each entry is the vector shape function value for one node */
     virtual void getVectorValues(

From 619571da7e3ea72e70a29ae5433f2b7bdf0a53ad Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 31 Mar 2020 13:23:58 -0400
Subject: [PATCH 080/555] Implements getElementNodeXis for simplex types

This is to get all the nodes associated with the simplex, including the
nodes on bounding entities. This is useful for field transfer from FEM
codes.
---
 apf/apfShape.cc | 79 +++++++++++++++++++++++++++++++++++++++++++++++++
 apf/apfShape.h  |  6 ++++
 2 files changed, 85 insertions(+)

diff --git a/apf/apfShape.cc b/apf/apfShape.cc
index 566f3ad75..a4e4c09a3 100644
--- a/apf/apfShape.cc
+++ b/apf/apfShape.cc
@@ -1069,4 +1069,83 @@ int countElementNodes(FieldShape* s, int type)
   return s->getEntityShape(type)->countNodes();
 }
 
+void getElementNodeXis(FieldShape* s, int type,
+    apf::NewArray<apf::Vector3>& xis)
+{
+  PCU_ALWAYS_ASSERT_VERBOSE(isSimplex(type),
+      "Only implemented for simplex types!");
+
+  static apf::Vector3 const vert_vert_xi[1] = {
+      apf::Vector3(0,0,0)
+  };
+  static apf::Vector3 const edge_vert_xi[2] = {
+      apf::Vector3(-1,0,0),
+      apf::Vector3(1,0,0),
+  };
+  static apf::Vector3 const tri_vert_xi[3] = {
+      apf::Vector3(0,0,0),
+      apf::Vector3(1,0,0),
+      apf::Vector3(0,1,0),
+  };
+  static apf::Vector3 const tet_vert_xi[4] = {
+      apf::Vector3(0,0,0),
+      apf::Vector3(1,0,0),
+      apf::Vector3(0,1,0),
+      apf::Vector3(0,0,1),
+  };
+
+  static apf::Vector3 const* const elem_vert_xi[apf::Mesh::TYPES] = {
+      vert_vert_xi,
+      edge_vert_xi,
+      tri_vert_xi,
+      0, /* quad */
+      tet_vert_xi,
+      0, /* hex */
+      0, /* prism */
+      0  /* pyramid */
+  };
+
+  if (!xis.allocated())
+    xis.allocate(countElementNodes(s, type));
+  else
+    xis.resize(countElementNodes(s, type));
+
+  int td = apf::Mesh::typeDimension[type];
+  apf::Vector3 childXi, parentXi;
+  int evi = 0;
+
+  int row = 0;
+  for(int d = 0; d <= td; ++d){
+    int nDown = apf::Mesh::adjacentCount[type][d];
+    int bt = apf::Mesh::simplexTypes[d];
+    apf::EntityShape* shape = apf::getLagrange(1)->getEntityShape(bt);
+    int non = s->countNodesOn(bt);
+    for(int j = 0; j < nDown; ++j){
+      for(int x = 0; x < non; ++x){
+        s->getNodeXi(bt, x, childXi);
+        apf::NewArray<double> shape_vals;
+
+        shape->getValues(0, 0, childXi, shape_vals);
+        parentXi.zero();
+        evi = j;
+        for (int i = 0; i < apf::Mesh::adjacentCount[bt][0]; ++i) {
+          printf("i,x,j,bt,d    %d,%d,%d,%d,%d\n", i, x, j, bt, d);
+          if(bt == apf::Mesh::EDGE && type == apf::Mesh::TRIANGLE)
+            evi = apf::tri_edge_verts[j][i];
+          else if(bt == apf::Mesh::EDGE && type == apf::Mesh::TET)
+            evi = apf::tet_edge_verts[j][i];
+          else if(bt == apf::Mesh::TRIANGLE && type == apf::Mesh::TET)
+            evi = apf::tet_tri_verts[j][i];
+          else if(bt == type)
+            evi = i;
+          parentXi += elem_vert_xi[type][evi] * shape_vals[i];
+        }
+        xis[row] = parentXi;
+        ++row;
+      }
+    }
+  }
+  PCU_ALWAYS_ASSERT(row == s->getEntityShape(type)->countNodes());
+}
+
 }//namespace apf
diff --git a/apf/apfShape.h b/apf/apfShape.h
index 230455e38..b70cd59ab 100644
--- a/apf/apfShape.h
+++ b/apf/apfShape.h
@@ -159,6 +159,12 @@ FieldShape* getShapeByName(const char* name);
   \param type select from apf::Mesh::Type */
 int countElementNodes(FieldShape* s, int type);
 
+/** \brief gets the xi coordinates for all the nodes
+  \details order follows downward adjacency
+  \param type select from apf::Mesh::Type */
+void getElementNodeXis(FieldShape* s, int type,
+    apf::NewArray<apf::Vector3>& xis);
+
 /** \brief Reparameterize from boundary entity to element
   \details This function converts a point in the local
   parametric space of a boundary mesh entity into the

From 3cce1a39799b1a029a96ebcb30ef026d94272a00 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Thu, 2 Apr 2020 19:43:50 -0400
Subject: [PATCH 081/555] Adds necessary code to use Nedelec Shape

More member functions needed to be implemented inside Nedelec class.
More specifically alignSharedNodes for Nedelec::Triangle and
Nedelec::Tetrahedron entity shapes.
---
 apf/CMakeLists.txt           |  2 +
 apf/apf.cc                   | 83 +++++++++++++++++++++++++++---------
 apf/apfElement.cc            | 29 ++++++++++---
 apf/apfElement.h             |  2 +
 apf/apfElementOf.h           | 14 +++---
 apf/apfField.h               | 10 ++++-
 apf/apfMatrixField.h         |  1 +
 apf/apfMixedVectorElement.cc | 31 ++++++++++++++
 apf/apfMixedVectorElement.h  | 29 +++++++++++++
 apf/apfMixedVectorField.cc   | 24 +++++++++++
 apf/apfMixedVectorField.h    | 32 ++++++++++++++
 apf/apfNedelec.cc            |  1 +
 apf/apfPackedField.h         |  1 +
 apf/apfScalarField.h         |  1 +
 apf/apfShape.cc              |  5 +++
 apf/apfShape.h               |  2 +
 apf/apfVectorField.h         |  1 +
 17 files changed, 235 insertions(+), 33 deletions(-)
 create mode 100644 apf/apfMixedVectorElement.cc
 create mode 100644 apf/apfMixedVectorElement.h
 create mode 100644 apf/apfMixedVectorField.cc
 create mode 100644 apf/apfMixedVectorField.h

diff --git a/apf/CMakeLists.txt b/apf/CMakeLists.txt
index 2d0fa0276..43484f1c7 100644
--- a/apf/CMakeLists.txt
+++ b/apf/CMakeLists.txt
@@ -29,6 +29,8 @@ set(SOURCES
   apfVector.cc
   apfVectorElement.cc
   apfVectorField.cc
+  apfMixedVectorElement.cc
+  apfMixedVectorField.cc
   apfPackedField.cc
   apfNumbering.cc
   apfMixedNumbering.cc
diff --git a/apf/apf.cc b/apf/apf.cc
index c981988f7..8481feb8a 100644
--- a/apf/apf.cc
+++ b/apf/apf.cc
@@ -10,6 +10,8 @@
 #include "apfScalarElement.h"
 #include "apfVectorField.h"
 #include "apfVectorElement.h"
+#include "apfMixedVectorField.h"
+#include "apfMixedVectorElement.h"
 #include "apfMatrixField.h"
 #include "apfMatrixElement.h"
 #include "apfPackedField.h"
@@ -71,16 +73,24 @@ Field* makeField(
 {
   PCU_ALWAYS_ASSERT( ! m->findField(name));
   Field* f = 0;
-  if (valueType == SCALAR)
-    f = new ScalarField();
-  else if (valueType == VECTOR)
-    f = new VectorField();
-  else if (valueType == MATRIX)
-    f = new MatrixField();
-  else if (valueType == PACKED)
-    f = new PackedField(components);
-  else
-    fail("invalid valueType in field construction\n");
+  // Cases with Vector shape functions
+  if (shape->isVectorShape()) {
+    PCU_ALWAYS_ASSERT(valueType == SCALAR);
+    f = new MixedVectorField();
+  }
+  // Cases with Scalar shahpe funtions
+  else {
+    if (valueType == SCALAR)
+      f = new ScalarField();
+    else if (valueType == VECTOR)
+      f = new VectorField();
+    else if (valueType == MATRIX)
+      f = new MatrixField();
+    else if (valueType == PACKED)
+      f = new PackedField(components);
+    else
+      fail("invalid valueType in field construction\n");
+  }
   f->init(name,m,shape,data);
   m->addField(f);
   return f;
@@ -166,16 +176,29 @@ void destroyField(Field* f)
 
 void setScalar(Field* f, MeshEntity* e, int node, double value)
 {
-  ScalarField* field = static_cast<ScalarField*>(f);
-  double tmp[1] = {value};
-  field->setNodeValue(e,node,tmp);
+  if (f->getShape()->isVectorShape()) {
+    MixedVectorField* field = static_cast<MixedVectorField*>(f);
+    double tmp[1] = {value};
+    field->setNodeValue(e,node,tmp);
+  }
+  else {
+    ScalarField* field = static_cast<ScalarField*>(f);
+    double tmp[1] = {value};
+    field->setNodeValue(e,node,tmp);
+  }
 }
 
 double getScalar(Field* f, MeshEntity* e, int node)
 {
-  ScalarField* field = static_cast<ScalarField*>(f);
   double value[1];
-  field->getNodeValue(e,node,value);
+  if (f->getShape()->isVectorShape()) {
+    MixedVectorField* field = static_cast<MixedVectorField*>(f);
+    field->getNodeValue(e,node,value);
+  }
+  else {
+    ScalarField* field = static_cast<ScalarField*>(f);
+    field->getNodeValue(e,node,value);
+  }
   return value[0];
 }
 
@@ -258,24 +281,46 @@ void getGrad(Element* e, Vector3 const& param, Vector3& g)
 
 void getVector(Element* e, Vector3 const& param, Vector3& value)
 {
-  VectorElement* element = static_cast<VectorElement*>(e);
-  value = element->getValue(param);
+  // Cases with vector shape functions first
+  if (e->getFieldShape()->isVectorShape()) {
+    MixedVectorElement* element = static_cast<MixedVectorElement*>(e);
+    value = element->getValue(param);
+  }
+  // Cases with scalar shape functions
+  else {
+    VectorElement* element = static_cast<VectorElement*>(e);
+    value = element->getValue(param);
+  }
 }
 
 double getDiv(Element* e, Vector3 const& param)
 {
+  // Make sure this in not called for cases with vector shapes
+  PCU_ALWAYS_ASSERT_VERBOSE(!e->getFieldShape()->isVectorShape(),
+      "Not implemented for fields with vector shape functions.");
   VectorElement* element = static_cast<VectorElement*>(e);
   return element->div(param);
 }
 
 void getCurl(Element* e, Vector3 const& param, Vector3& c)
 {
-  VectorElement* element = static_cast<VectorElement*>(e);
-  return element->curl(param,c);
+  // Cases with vector shape functions first
+  if (e->getFieldShape()->isVectorShape()) {
+    MixedVectorElement* element = static_cast<MixedVectorElement*>(e);
+    return element->curl(param,c);
+  }
+  // Cases with scalar shape functions
+  else {
+    VectorElement* element = static_cast<VectorElement*>(e);
+    return element->curl(param,c);
+  }
 }
 
 void getVectorGrad(Element* e, Vector3 const& param, Matrix3x3& deriv)
 {
+  // Make sure this in not called for cases with vector shapes
+  PCU_ALWAYS_ASSERT_VERBOSE(!e->getFieldShape()->isVectorShape(),
+      "Not implemented for fields with vector shape functions.");
   VectorElement* element = static_cast<VectorElement*>(e);
   return element->grad(param,deriv);
 }
diff --git a/apf/apfElement.cc b/apf/apfElement.cc
index 82c89c602..1a7e5f8f4 100644
--- a/apf/apfElement.cc
+++ b/apf/apfElement.cc
@@ -5,6 +5,8 @@
  * BSD license as described in the LICENSE file in the top-level directory.
  */
 
+#include <pcu_util.h>
+
 #include "apfElement.h"
 #include "apfShape.h"
 #include "apfMesh.h"
@@ -75,6 +77,8 @@ Matrix3x3 getJacobianInverse(Matrix3x3 J, int dim)
 void Element::getGlobalGradients(Vector3 const& local,
                                  NewArray<Vector3>& globalGradients)
 {
+  PCU_ALWAYS_ASSERT_VERBOSE(!field->getShape()->isVectorShape(),
+      "Not implemented for vector shape functions!");
   Matrix3x3 J;
   parent->getJacobian(local,J);
   Matrix3x3 jinv = getJacobianInverse(J, getDimension());
@@ -87,13 +91,26 @@ void Element::getGlobalGradients(Vector3 const& local,
 
 void Element::getComponents(Vector3 const& xi, double* c)
 {
-  NewArray<double> shapeValues;
-  shape->getValues(mesh, entity, xi, shapeValues);
-  for (int ci = 0; ci < nc; ++ci)
-    c[ci] = 0;
-  for (int ni = 0; ni < nen; ++ni)
+  // handle cases with vector shape functions
+  if (field->getShape()->isVectorShape()) {
+    NewArray<Vector3> shapeValues;
+    shape->getVectorValues(mesh, entity, xi, shapeValues);
+    for (int ci = 0; ci < 3; ci++)
+      c[ci] = 0.;
+    for (int ni = 0; ni < nen; ni++)
+      for (int ci = 0; ci < 3; ci++)
+      	c[ci] += nodeData[ni] * shapeValues[ni][ci];
+  }
+  // handle cases with scalar shape functions
+  else {
+    NewArray<double> shapeValues;
+    shape->getValues(mesh, entity, xi, shapeValues);
     for (int ci = 0; ci < nc; ++ci)
-      c[ci] += nodeData[ni * nc + ci] * shapeValues[ni];
+      c[ci] = 0;
+    for (int ni = 0; ni < nen; ++ni)
+      for (int ci = 0; ci < nc; ++ci)
+	c[ci] += nodeData[ni * nc + ci] * shapeValues[ni];
+  }
 }
 
 void Element::getNodeData()
diff --git a/apf/apfElement.h b/apf/apfElement.h
index f100d42e8..54318d32b 100644
--- a/apf/apfElement.h
+++ b/apf/apfElement.h
@@ -15,6 +15,7 @@
 namespace apf {
 
 class EntityShape;
+class FieldShape;
 class VectorElement;
 
 class Element
@@ -32,6 +33,7 @@ class Element
     MeshEntity* getEntity() {return entity;}
     Mesh* getMesh() {return mesh;}
     EntityShape* getShape() {return shape;}
+    FieldShape* getFieldShape() {return field->getShape();}
     void getComponents(Vector3 const& xi, double* c);
   protected:
     void init(Field* f, MeshEntity* e, VectorElement* p);
diff --git a/apf/apfElementOf.h b/apf/apfElementOf.h
index 9b20565d7..afedf6111 100644
--- a/apf/apfElementOf.h
+++ b/apf/apfElementOf.h
@@ -14,22 +14,22 @@
 
 namespace apf {
 
-template <class T>
+template <class T, class S = T>
 class ElementOf : public Element
 {
   public:
-    ElementOf(FieldOf<T>* f, MeshEntity* e):
+    ElementOf(FieldOf<S>* f, MeshEntity* e):
       Element(f,e)
     {
     }
-    ElementOf(FieldOf<T>* f, VectorElement* p):
+    ElementOf(FieldOf<S>* f, VectorElement* p):
       Element(f,p)
     {
     }
     virtual ~ElementOf() {}
-    T* getNodeValues()
+    S* getNodeValues()
     {
-      return reinterpret_cast<T*>(&(this->nodeData[0]));
+      return reinterpret_cast<S*>(&(this->nodeData[0]));
     }
     T getValue(Vector3 const& local)
     {
@@ -37,10 +37,10 @@ class ElementOf : public Element
       getComponents(local, reinterpret_cast<double*>(value));
       return value[0];
     }
-    void getValues(NewArray<T>& values)
+    void getValues(NewArray<S>& values)
     {
       values.allocate(nen);
-      T* nodeValues = getNodeValues();
+      S* nodeValues = getNodeValues();
       for (int i=0; i < nen; ++i)
         values[i] = nodeValues[i];
     }
diff --git a/apf/apfField.h b/apf/apfField.h
index c68fc50a8..39b31718c 100644
--- a/apf/apfField.h
+++ b/apf/apfField.h
@@ -25,7 +25,7 @@ class FieldBase
               FieldData* d);
     virtual ~FieldBase();
     /* returns the number of components per node:
-       1 for a scalar field
+       1 for a scalar field and Nedelec type fields
        3 for a vector field
        9 for a matrix field, etc. */
     virtual int countComponents() const = 0;
@@ -54,7 +54,15 @@ class Field : public FieldBase
 {
   public:
     virtual Element* getElement(VectorElement* e) = 0;
+    // Think of this as the dof types, that is:
+    // ScalarField will have 1 dof per node
+    // VectorField will have 3 dof per noed
+    // MixedVectorField (eg Nedelec) will have 1 dof per node
     virtual int getValueType() const = 0;
+    // Think of this as the number of components for the shape, that is:
+    // Regular shape function will have 1 value per node, but Nedelec type
+    // shapes will have 3 values per node
+    virtual int getShapeType() const = 0;
     virtual int getScalarType() {return Mesh::DOUBLE;}
     FieldDataOf<double>* getData();
     virtual void project(Field* from) = 0;
diff --git a/apf/apfMatrixField.h b/apf/apfMatrixField.h
index 61b3b1bac..3ba9ec677 100644
--- a/apf/apfMatrixField.h
+++ b/apf/apfMatrixField.h
@@ -19,6 +19,7 @@ class MatrixField : public FieldOf<Matrix3x3>
     virtual ~MatrixField() {}
     virtual Element* getElement(VectorElement* e);
     virtual int getValueType() const {return MATRIX;}
+    virtual int getShapeType() const {return SCALAR;}
     virtual int countComponents() const;
 };
 
diff --git a/apf/apfMixedVectorElement.cc b/apf/apfMixedVectorElement.cc
new file mode 100644
index 000000000..9d08d579c
--- /dev/null
+++ b/apf/apfMixedVectorElement.cc
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2011 Scientific Computation Research Center
+ *
+ * This work is open source software, licensed under the terms of the
+ * BSD license as described in the LICENSE file in the top-level directory.
+ */
+
+#include "apfVectorElement.h"
+#include "apfMixedVectorElement.h"
+#include "apfMixedVectorField.h"
+
+namespace apf {
+
+MixedVectorElement::MixedVectorElement(MixedVectorField* f, MeshEntity* e):
+  ElementOf<Vector3, double>(f,e)
+{
+}
+
+MixedVectorElement::MixedVectorElement(MixedVectorField* f, VectorElement* p):
+  ElementOf<Vector3, double>(f,p)
+{
+}
+
+void MixedVectorElement::curl(Vector3 const& xi, Vector3& c)
+{
+  //TODO to be completed using the curl of nedelec shapes and the nodeData
+  (void) xi;
+  (void) c;
+}
+
+}//namespace apf
diff --git a/apf/apfMixedVectorElement.h b/apf/apfMixedVectorElement.h
new file mode 100644
index 000000000..35d3f6107
--- /dev/null
+++ b/apf/apfMixedVectorElement.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2011 Scientific Computation Research Center
+ *
+ * This work is open source software, licensed under the terms of the
+ * BSD license as described in the LICENSE file in the top-level directory.
+ */
+
+#ifndef APFMIXEDVECTORELEMENT_H
+#define APFMIXEDVECTORELEMENT_H
+
+#include "apfElementOf.h"
+#include "apfMatrix.h"
+
+namespace apf {
+
+class MixedVectorField;
+
+class MixedVectorElement : public ElementOf<Vector3, double>
+{
+  public:
+    MixedVectorElement(MixedVectorField* f, MeshEntity* e);
+    MixedVectorElement(MixedVectorField* f, VectorElement* p);
+    virtual ~MixedVectorElement() {}
+    void curl(Vector3 const& xi, Vector3& c);
+};
+
+}//namespace apf
+
+#endif
diff --git a/apf/apfMixedVectorField.cc b/apf/apfMixedVectorField.cc
new file mode 100644
index 000000000..f9786c828
--- /dev/null
+++ b/apf/apfMixedVectorField.cc
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2011 Scientific Computation Research Center
+ *
+ * This work is open source software, licensed under the terms of the
+ * BSD license as described in the LICENSE file in the top-level directory.
+ */
+
+#include "apfMixedVectorField.h"
+#include "apfMixedVectorElement.h"
+#include "apfVectorElement.h"
+
+namespace apf {
+
+Element* MixedVectorField::getElement(VectorElement* e)
+{
+  return new MixedVectorElement(this,e);
+}
+
+int MixedVectorField::countComponents() const
+{
+  return 1;
+}
+
+}//namespace apf
diff --git a/apf/apfMixedVectorField.h b/apf/apfMixedVectorField.h
new file mode 100644
index 000000000..aa4d5495a
--- /dev/null
+++ b/apf/apfMixedVectorField.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2011 Scientific Computation Research Center
+ *
+ * This work is open source software, licensed under the terms of the
+ * BSD license as described in the LICENSE file in the top-level directory.
+ */
+
+#ifndef APFMIXEDVECTORFIELD_H
+#define APFMIXEDVECTORFIELD_H
+
+#include "apfFieldOf.h"
+#include "apf.h"
+
+namespace apf {
+
+/* This is used for fields with vector shape functions (e.g. Nedelec)
+ * They are special in the sense that the dofs (or nodal values)
+ * are scalar but the shape functions are vectors!
+ */
+class MixedVectorField : public FieldOf<double>
+{
+  public:
+    virtual ~MixedVectorField() {}
+    virtual Element* getElement(VectorElement* e);
+    virtual int getValueType() const {return SCALAR;}
+    virtual int getShapeType() const {return VECTOR;}
+    virtual int countComponents() const;
+};
+
+}//namespace apf
+
+#endif
diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index 34551c2d1..e7013ab58 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -501,6 +501,7 @@ class Nedelec: public FieldShape {
   public:
     const char* getName() const { return "Nedelec"; }
     int getOrder() {return P;}
+    bool isVectorShape() {return true;}
     /* Nedelec(int order) : P(order) */
     /* {} */
     Nedelec()
diff --git a/apf/apfPackedField.h b/apf/apfPackedField.h
index 5236b5f64..f293c9b50 100644
--- a/apf/apfPackedField.h
+++ b/apf/apfPackedField.h
@@ -20,6 +20,7 @@ class PackedField : public Field
     virtual ~PackedField() {}
     virtual Element* getElement(VectorElement* e);
     virtual int getValueType() const {return PACKED;}
+    virtual int getShapeType() const {return SCALAR;}
     virtual int countComponents() const {return components;}
     virtual void project(Field* from);
     virtual void axpy(double a, Field* x);
diff --git a/apf/apfScalarField.h b/apf/apfScalarField.h
index cdfffb90d..866ad4094 100644
--- a/apf/apfScalarField.h
+++ b/apf/apfScalarField.h
@@ -19,6 +19,7 @@ class ScalarField : public FieldOf<double>
     virtual ~ScalarField() {}
     virtual Element* getElement(VectorElement* e);
     virtual int getValueType() const {return SCALAR;}
+    virtual int getShapeType() const {return SCALAR;}
     virtual int countComponents() const;
 };
 
diff --git a/apf/apfShape.cc b/apf/apfShape.cc
index a4e4c09a3..1f088e37a 100644
--- a/apf/apfShape.cc
+++ b/apf/apfShape.cc
@@ -39,6 +39,11 @@ void FieldShape::getNodeXi(int, int, Vector3&)
   fail("unimplemented getNodeXi called");
 }
 
+bool FieldShape::isVectorShape()
+{
+  return false;
+}
+
 void FieldShape::registerSelf(const char* name_)
 {
   std::string name = name_;
diff --git a/apf/apfShape.h b/apf/apfShape.h
index b70cd59ab..160ba6d0e 100644
--- a/apf/apfShape.h
+++ b/apf/apfShape.h
@@ -103,6 +103,8 @@ class FieldShape
   \param node index from element node ordering
   \param xi parent element coordinates */
     virtual void getNodeXi(int type, int node, Vector3& xi);
+/** \brief Returns true if the shape functions are vectors */
+    virtual bool isVectorShape();
 /** \brief Get a unique string for this shape function scheme */
     virtual const char* getName() const = 0;
     void registerSelf(const char* name);
diff --git a/apf/apfVectorField.h b/apf/apfVectorField.h
index c2f705a51..36a3dc076 100644
--- a/apf/apfVectorField.h
+++ b/apf/apfVectorField.h
@@ -19,6 +19,7 @@ class VectorField : public FieldOf<Vector3>
     virtual ~VectorField() {}
     virtual Element* getElement(VectorElement* e);
     virtual int getValueType() const {return VECTOR;}
+    virtual int getShapeType() const {return SCALAR;}
     virtual int countComponents() const;
 };
 

From 114986c3b042053e9f4143e8a1d161303bbbcfc6 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Mon, 6 Apr 2020 14:54:40 -0400
Subject: [PATCH 082/555] Fixes declaration of transpose in namespace mth

---
 mth/mth.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mth/mth.h b/mth/mth.h
index bda16d1eb..93e70a492 100644
--- a/mth/mth.h
+++ b/mth/mth.h
@@ -38,7 +38,7 @@ Vector<T,N> reject(Vector<T,N> const& a, Vector<T,N> const& b);
 
 /** \brief transpose of a static matrix */
 template <class T, unsigned M, unsigned N>
-Matrix<T,M,N> transpose(Matrix<T,M,N> const& a);
+void transpose(Matrix<T,M,N> const& a, Matrix<T,N,M>& b);
 
 /** \brief determinant of a 2 by 2 matrix */
 template <class T>

From 22d88c8748cea6630f55debaf8369a9efef19e2d Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Mon, 6 Apr 2020 17:36:51 -0400
Subject: [PATCH 083/555] alignSharedNodes for Nedelec Tet and Tri

---
 apf/apf.h         |  2 +-
 apf/apfNedelec.cc | 96 ++++++++++++++++++++++++++++++++++++++++++++---
 2 files changed, 91 insertions(+), 7 deletions(-)

diff --git a/apf/apf.h b/apf/apf.h
index 9e9e567e3..677e3849f 100644
--- a/apf/apf.h
+++ b/apf/apf.h
@@ -43,7 +43,7 @@ template <class T> class ReductionOp;
 template <class T> class ReductionSum;
 
 /** \brief Base class for applying operations to make a Field consistent
-  * in parallel 
+  * in parallel
   * \details This function gets applied pairwise to the Field values
   * from every partition, resulting in a single unique value.  No guarantees
   * are made about the order in which this function is applied to the
diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index e7013ab58..0b4b8e2f8 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -574,10 +574,19 @@ class Nedelec: public FieldShape {
       	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getLocalGradients not implemented for Nedelec Triangle. Aborting()!");
       }
       int countNodes() const {return countTriNodes(P);}
-      void alignSharedNodes(apf::Mesh*,
-	  apf::MeshEntity*, apf::MeshEntity*, int order[])
+      void alignSharedNodes(apf::Mesh* m,
+	  apf::MeshEntity* elem, apf::MeshEntity* shared, int order[])
       {
-	(void)order;
+	      int which, rotate;
+	      bool flip;
+        getAlignment(m,elem,shared,which,flip,rotate);
+        int n = getOrder();
+        if(!flip)
+          for(int i = 0; i < n; ++i)
+            order[i] = i;
+        else
+          for(int i = 0; i < n; ++i)
+            order[i] = n-1-i;
       }
       void getVectorValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const& xi, apf::NewArray<apf::Vector3>& shapes) const
@@ -702,10 +711,85 @@ class Nedelec: public FieldShape {
       	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getLocalGradients not implemented for Nedelec Tetrahedron. Aborting()!");
       }
       int countNodes() const {return countTetNodes(P);}
-      void alignSharedNodes(apf::Mesh*,
-	  apf::MeshEntity*, apf::MeshEntity*, int order[])
+      void alignSharedNodes(apf::Mesh* m,
+	  apf::MeshEntity* elem, apf::MeshEntity* shared, int order[])
       {
-	(void)order;
+	      int which, rotate;
+	      bool flip;
+        getAlignment(m,elem,shared,which,flip,rotate);
+        if(m->getType(shared) == apf::Mesh::EDGE)
+        {
+          int n = countNodes();
+          if(!flip)
+            for(int i = 0; i < n; ++i)
+              order[i] = i;
+          else
+            for(int i = 0; i < n; ++i)
+              order[i] = n-1-i;
+          return;
+        }
+        //must be a triangle
+        int k = getOrder();
+        int non = k*(k-1); // nodes on the face only
+        int unique_non = non/2; // bc 2 dofs per node on the face
+        if ( k > 1 ) // face nodes appear for k > 1
+        {
+          int i = 0; int size = 1; // Determine size of nodes matrix
+          while(i < unique_non) {
+            i = i + size;
+            size++;
+          }
+          size--;
+
+          mth::Matrix<double> Nodes(size,size); // populate nodes matrix
+          Nodes.zero();
+          int num = 0;
+          for ( int r = size-1; r >= 0; r--)
+            for ( int c = size - r -1; c < size; c++)
+              Nodes(r,c) = (num++);
+
+          // CASES
+          if(rotate == 1) {
+            for (int r = size-1; r >= 0; r--) {    // horizontal swaps
+              int left = size-r-1; int right = size-1;
+              for(int range = left; range <= left + (right-left)/2; range++) {
+                            std::swap( Nodes(r,range), Nodes(r,left+right-range) );
+              }
+            }
+            mth::Matrix<double> temp(size, size);
+            mth::transpose(Nodes,temp);
+            for (int r = 0; r < size; r++)
+              for(int c = 0; c < size; c++)
+                Nodes(r,c) = temp(r,c);
+          }
+          if(rotate == 2) {
+            for (int c = size-1; c >= 0; c--) {    // vertical swaps
+                    int left = size-c-1; int right = size-1;
+                    for(int range = left; range <= left + (right-left)/2; range++) {
+                            std::swap( Nodes(range,c), Nodes(left+right-range,c) );
+                    }
+            }
+            mth::Matrix<double> temp(size, size);
+            mth::transpose(Nodes,temp);
+            for (int r = 0; r < size; r++)
+              for(int c = 0; c < size; c++)
+                Nodes(r,c) = temp(r,c);
+          }
+          if(flip)
+          {
+            mth::Matrix<double> temp(size, size);
+            mth::transpose(Nodes,temp);
+            for (int r = 0; r < size; r++)
+              for(int c = 0; c < size; c++)
+                Nodes(r,c) = temp(r,c);
+          }
+          // get the ordered list
+          int index = 0;
+          for ( int r = size-1; r >= 0; r--)
+            for (int c = size-r-1 ; c < size; c++) {
+              order[i++] = Nodes(r,c)*2;    order[i++] = Nodes(r,c)*2 + 1;
+            }
+        }
       }
       void getVectorValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const& xi, apf::NewArray<apf::Vector3>& shapes) const

From 50392a467878aab506de72c1df62c291dd542298 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Mon, 6 Apr 2020 22:00:07 -0400
Subject: [PATCH 084/555] Fixed bugs in alignSharedNodes for Nedelec Tet.

---
 apf/apfNedelec.cc | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index 0b4b8e2f8..b042c883c 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -734,12 +734,7 @@ class Nedelec: public FieldShape {
         int unique_non = non/2; // bc 2 dofs per node on the face
         if ( k > 1 ) // face nodes appear for k > 1
         {
-          int i = 0; int size = 1; // Determine size of nodes matrix
-          while(i < unique_non) {
-            i = i + size;
-            size++;
-          }
-          size--;
+          int size = (-1 + sqrt(1+8*unique_non) )/2;
 
           mth::Matrix<double> Nodes(size,size); // populate nodes matrix
           Nodes.zero();
@@ -784,7 +779,7 @@ class Nedelec: public FieldShape {
                 Nodes(r,c) = temp(r,c);
           }
           // get the ordered list
-          int index = 0;
+          int i = 0;
           for ( int r = size-1; r >= 0; r--)
             for (int c = size-r-1 ; c < size; c++) {
               order[i++] = Nodes(r,c)*2;    order[i++] = Nodes(r,c)*2 + 1;

From 43585ac914e849f2089e6e17d0eeddf132a3b5c6 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Tue, 7 Apr 2020 13:33:32 -0400
Subject: [PATCH 085/555] implements getNodeXi for Nedelec edge,tet,tri.

---
 apf/apfNedelec.cc | 52 +++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 50 insertions(+), 2 deletions(-)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index b042c883c..019a97a3d 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -500,7 +500,6 @@ template<int P>
 class Nedelec: public FieldShape {
   public:
     const char* getName() const { return "Nedelec"; }
-    int getOrder() {return P;}
     bool isVectorShape() {return true;}
     /* Nedelec(int order) : P(order) */
     /* {} */
@@ -529,6 +528,7 @@ class Nedelec: public FieldShape {
     class Edge : public apf::EntityShape
     {
     public:
+      int getOrder() {return P;}
       void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<double>&) const
       {
@@ -539,7 +539,7 @@ class Nedelec: public FieldShape {
       {
       	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getLocalGradients not implemented for Nedelec Edges. Aborting()!");
       }
-      int countNodes() const {return 0; /* TODO update this*/}
+      int countNodes() const {return P;}
       void alignSharedNodes(apf::Mesh*,
 	  apf::MeshEntity*, apf::MeshEntity*, int order[])
       {
@@ -982,6 +982,54 @@ class Nedelec: public FieldShape {
       // for any other case (type and P combination) return 0;
       return 0;
     }
+    int getOrder() {return P;}
+    void getNodeXi(int type, int node, Vector3& xi)
+    {
+      if(type == Mesh::EDGE)
+      {
+        const double *eop = (P > 0) ? getOpenPoints(P-1) : NULL;
+        xi = Vector3( -1 + 2*eop[node], 0., 0. ); // map from [0,1] to [-1,1]
+        return;
+      }
+      else if (type == Mesh::TRIANGLE)
+      {
+        const double *iop = (P > 1) ? getOpenPoints(P - 2) : NULL;
+        int pm2 = P - 2; int c = 0;
+
+        for (int j = 0; j <= pm2; j++) {
+          for (int i = 0; i + j <= pm2; i++) {
+            if (node/2 == c) {  // since 2 dofs per node on the face
+              double w = iop[i] + iop[j] + iop[pm2-i-j];
+              xi = Vector3( iop[i]/w, iop[j]/w, 0. );
+              return;
+            }
+            else
+              c++;
+          }
+        }
+      }
+      else if (type == Mesh::TET)
+      {
+        const double *iop = (P > 2) ? getOpenPoints(P - 3) : NULL;
+        int pm3 = P - 3; int c = 0;
+
+        for (int k = 0; k <= pm3; k++) {
+          for (int j = 0; j + k <= pm3; j++) {
+            for (int i = 0; i + j + k <= pm3; i++) {
+              if( node/3 == c) {  // since 3 dofs per node on interior tet
+                double w = iop[i] + iop[j] + iop[k] + iop[pm3-i-j-k];
+                xi = Vector3( iop[i]/w, iop[j]/w,  iop[k]/w );
+                return;
+              }
+              else
+                c++;
+            }
+          }
+        }
+      }
+      else
+        xi = Vector3(0,0,0);
+    }
 };
 
 apf::FieldShape* getNedelec(int order)

From c7769685feb67ae97a066cb8e0195e3d0f64bf51 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 7 Apr 2020 14:27:16 -0400
Subject: [PATCH 086/555] Adds getNodeTangent to FieldShape class

---
 apf/apfShape.h | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/apf/apfShape.h b/apf/apfShape.h
index 160ba6d0e..ba28375a2 100644
--- a/apf/apfShape.h
+++ b/apf/apfShape.h
@@ -103,6 +103,12 @@ class FieldShape
   \param node index from element node ordering
   \param xi parent element coordinates */
     virtual void getNodeXi(int type, int node, Vector3& xi);
+/** \brief Get the parent element tangent vector of an element node
+  \param type element type, select from apf::Mesh::Type
+  \param node index from element node ordering
+  \param t parent element tangent
+  \details this is only applicable for vector shapes */
+    virtual void getNodeTangent(int type, int node, Vector3& t);
 /** \brief Returns true if the shape functions are vectors */
     virtual bool isVectorShape();
 /** \brief Get a unique string for this shape function scheme */

From 37fe37ab69dcd9da6c667b0558a07c1fe51d9a78 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 7 Apr 2020 14:52:54 -0400
Subject: [PATCH 087/555] Adds a TODO for getVector for MixedVectorElement

---
 apf/apf.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/apf/apf.cc b/apf/apf.cc
index 8481feb8a..6b942d9ea 100644
--- a/apf/apf.cc
+++ b/apf/apf.cc
@@ -285,6 +285,7 @@ void getVector(Element* e, Vector3 const& param, Vector3& value)
   if (e->getFieldShape()->isVectorShape()) {
     MixedVectorElement* element = static_cast<MixedVectorElement*>(e);
     value = element->getValue(param);
+    // TODO: do we need to apply Piola Transform here?
   }
   // Cases with scalar shape functions
   else {

From 828295b6fa50c113766da5ac2314f1af7ad91b42 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Tue, 7 Apr 2020 22:32:11 -0400
Subject: [PATCH 088/555] implements getNodeTangent for edge, tet, tri.

---
 apf/apfNedelec.cc | 26 ++++++++++++++++++++++++++
 apf/apfShape.cc   |  5 +++++
 2 files changed, 31 insertions(+)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index 019a97a3d..382d0c241 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -1030,6 +1030,32 @@ class Nedelec: public FieldShape {
       else
         xi = Vector3(0,0,0);
     }
+    void getNodeTangent(int type, int node, Vector3& t)
+    {
+      if(type == Mesh::EDGE)
+      {
+        t = Vector3( 1., 0., 0.);
+        return;
+      }
+      else if(type == Mesh::TRIANGLE)
+      {
+        PCU_ALWAYS_ASSERT_VERBOSE(P >= 2,
+            "face nodes appear only for order bigger than or equal to 2!");
+        (node % 2 == 0) ? t = Vector3(1., 0., 0.) : t = Vector3(0., 1., 0.);
+        return;
+      }
+      else if(type == Mesh::TET)
+      {
+        PCU_ALWAYS_ASSERT_VERBOSE(P >= 3,
+            "volume nodes appear only for order bigger than or equal to 3!");
+       if (node % 3 == 0) t = Vector3(1., 0., 0.);
+       else if (node % 3 == 1) t = Vector3(0., -1., 0.);
+       else t = Vector3(0., 0., 1.);
+       return;
+      }
+      else
+        t = Vector3(0, 0, 0);
+    }
 };
 
 apf::FieldShape* getNedelec(int order)
diff --git a/apf/apfShape.cc b/apf/apfShape.cc
index 1f088e37a..d8fae3711 100644
--- a/apf/apfShape.cc
+++ b/apf/apfShape.cc
@@ -39,6 +39,11 @@ void FieldShape::getNodeXi(int, int, Vector3&)
   fail("unimplemented getNodeXi called");
 }
 
+void FieldShape::getNodeTangent(int, int, Vector3&)
+{
+  fail("unimplemented getNodeTangent called");
+}
+
 bool FieldShape::isVectorShape()
 {
   return false;

From 2e862805b378e742f91091d81606f3becf16e9c4 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Wed, 8 Apr 2020 13:28:39 -0400
Subject: [PATCH 089/555] implements Piola transformation for Nedelec.

---
 apf/apf.cc        | 25 ++++++++++++++++++++++++-
 apf/apf.h         |  1 +
 apf/apfElement.cc |  3 ++-
 apf/apfNedelec.cc | 18 ++++++++++++------
 apf/apfShape.cc   |  5 +++++
 apf/apfShape.h    |  3 +++
 6 files changed, 47 insertions(+), 8 deletions(-)

diff --git a/apf/apf.cc b/apf/apf.cc
index 6b942d9ea..996622c86 100644
--- a/apf/apf.cc
+++ b/apf/apf.cc
@@ -459,7 +459,30 @@ void getShapeGrads(Element* e, Vector3 const& local,
 void getVectorShapeValues(Element* e, Vector3 const& local,
     NewArray<Vector3>& values)
 {
-  e->getShape()->getVectorValues(e->getMesh(), e->getEntity(), local,values);
+  NewArray<Vector3> vvals(values.size());
+  e->getShape()->getVectorValues(e->getMesh(), e->getEntity(), local, vvals);
+
+  // Perform Piola transformation
+  apf::MeshElement* me = apf::createMeshElement( e->getMesh(), e->getEntity() );
+  if( e->getShape()->getRefDim()  ==  e->getDimension() ) // i.e. J is square
+  {
+    apf::Matrix3x3 Jinv;
+    apf::getJacobianInv(me, local, Jinv);
+
+    // u(x_hat) * J(x_hat)^{-1}
+    for( size_t i = 0; i < values.size(); i++ ) {
+      for ( int j = 0; j < 3; j++ ) {
+        values[i][j] = 0.;
+        for ( int k = 0; k < 3; k++ )
+          values[i][j] += vvals[i][k] * Jinv[k][j];
+      }
+    }
+  }
+  else
+  {
+    // TODO
+  }
+  apf::destroyMeshElement(me);
 }
 
 FieldShape* getShape(Field* f)
diff --git a/apf/apf.h b/apf/apf.h
index 677e3849f..991972853 100644
--- a/apf/apf.h
+++ b/apf/apf.h
@@ -605,6 +605,7 @@ void getShapeGrads(Element* e, Vector3 const& local,
 
 /** \brief Returns the vector shape function values at a point
  *  \details used only for Nedelec shapes
+ *  (Piola transformation used to map from parent to physical coordinates)
   */
 void getVectorShapeValues(Element* e, Vector3 const& local,
     NewArray<Vector3>& values);
diff --git a/apf/apfElement.cc b/apf/apfElement.cc
index 1a7e5f8f4..5cdacc434 100644
--- a/apf/apfElement.cc
+++ b/apf/apfElement.cc
@@ -94,7 +94,8 @@ void Element::getComponents(Vector3 const& xi, double* c)
   // handle cases with vector shape functions
   if (field->getShape()->isVectorShape()) {
     NewArray<Vector3> shapeValues;
-    shape->getVectorValues(mesh, entity, xi, shapeValues);
+    //shape->getVectorValues(mesh, entity, xi, shapeValues); //TODO remove this
+    getVectorShapeValues(this, xi, shapeValues);
     for (int ci = 0; ci < 3; ci++)
       c[ci] = 0.;
     for (int ni = 0; ni < nen; ni++)
diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index 382d0c241..409e92f6f 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -527,8 +527,12 @@ class Nedelec: public FieldShape {
     };
     class Edge : public apf::EntityShape
     {
+    private:
+      const int dim = 1; // ref elem dim
+      const double c = 0.; // center of edge
     public:
       int getOrder() {return P;}
+      int getRefDim() {return dim;}
       void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<double>&) const
       {
@@ -563,6 +567,7 @@ class Nedelec: public FieldShape {
       const double c = 1./3.; // center of tri
     public:
       int getOrder() {return P;}
+      int getRefDim() {return dim;}
       void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<double>&) const
       {
@@ -622,9 +627,9 @@ class Nedelec: public FieldShape {
         }
 
 
-	mth::Matrix<double> Q(dof, dof);
-	mth::Matrix<double> R(dof, dof);
-	getTi(P, apf::Mesh::TRIANGLE, Q, R);
+	      mth::Matrix<double> Q(dof, dof);
+      	mth::Matrix<double> R(dof, dof);
+      	getTi(P, apf::Mesh::TRIANGLE, Q, R);
 
         mth::Matrix<double> S(dof, dim);
       	for(int i = 0; i < dim; i++) // S = Ti * u
@@ -685,9 +690,9 @@ class Nedelec: public FieldShape {
                      (dshape_y[j]*(y - c) + shape_y[j]) * shape_x[i]);
         }
 
-	mth::Matrix<double> Q(dof, dof);
-	mth::Matrix<double> R(dof, dof);
-	getTi(P, apf::Mesh::TRIANGLE, Q, R);
+      	mth::Matrix<double> Q(dof, dof);
+      	mth::Matrix<double> R(dof, dof);
+      	getTi(P, apf::Mesh::TRIANGLE, Q, R);
 
         mth::Vector<double> X(dof);
         mth::solveFromQR(Q, R, curlu, X);
@@ -699,6 +704,7 @@ class Nedelec: public FieldShape {
       const int dim = 3;
       const double c = 1./4.;
     public:
+      int getRefDim() {return dim;}
       int getOrder() {return P;}
       void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<double>&) const
diff --git a/apf/apfShape.cc b/apf/apfShape.cc
index d8fae3711..b661a0855 100644
--- a/apf/apfShape.cc
+++ b/apf/apfShape.cc
@@ -44,6 +44,11 @@ void FieldShape::getNodeTangent(int, int, Vector3&)
   fail("unimplemented getNodeTangent called");
 }
 
+int EntityShape::getRefDim()
+{
+  fail("unimplemented getRefDim called");
+}
+
 bool FieldShape::isVectorShape()
 {
   return false;
diff --git a/apf/apfShape.h b/apf/apfShape.h
index ba28375a2..8122a8457 100644
--- a/apf/apfShape.h
+++ b/apf/apfShape.h
@@ -74,6 +74,9 @@ class EntityShape
     help of apf::getAlignment */
     virtual void alignSharedNodes(Mesh* m,
         MeshEntity* elem, MeshEntity* shared, int order[]);
+/** \brief Get the parent element dimension
+ * \details this is not always applicable */
+    virtual int getRefDim();
 };
 
 /** \brief Describes field distribution and shape functions

From f56a0e6e898412160092b58ce105fda8f8c7860c Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Thu, 9 Apr 2020 16:35:38 -0400
Subject: [PATCH 090/555] implements getCurl & getCurlShapeValues for 3D ND

---
 apf/apf.cc                   | 55 +++++++++++++++++++++++++++++++-----
 apf/apf.h                    |  7 +++++
 apf/apfElement.cc            |  2 +-
 apf/apfMixedVectorElement.cc | 15 ++++++++--
 apf/apfNedelec.cc            |  9 ++----
 apf/apfShape.cc              | 11 ++++----
 apf/apfShape.h               | 12 ++++++--
 7 files changed, 86 insertions(+), 25 deletions(-)

diff --git a/apf/apf.cc b/apf/apf.cc
index 996622c86..398ec22da 100644
--- a/apf/apf.cc
+++ b/apf/apf.cc
@@ -27,6 +27,10 @@
 #include <pcu_util.h>
 #include <lionPrint.h>
 
+#include "mth.h"
+#include "mth_def.h"
+
+using namespace std;
 namespace apf {
 
 void destroyMesh(Mesh* m)
@@ -285,7 +289,6 @@ void getVector(Element* e, Vector3 const& param, Vector3& value)
   if (e->getFieldShape()->isVectorShape()) {
     MixedVectorElement* element = static_cast<MixedVectorElement*>(e);
     value = element->getValue(param);
-    // TODO: do we need to apply Piola Transform here?
   }
   // Cases with scalar shape functions
   else {
@@ -463,28 +466,66 @@ void getVectorShapeValues(Element* e, Vector3 const& local,
   e->getShape()->getVectorValues(e->getMesh(), e->getEntity(), local, vvals);
 
   // Perform Piola transformation
-  apf::MeshElement* me = apf::createMeshElement( e->getMesh(), e->getEntity() );
-  if( e->getShape()->getRefDim()  ==  e->getDimension() ) // i.e. J is square
+  if( e->getDimension() == e->getMesh()->getDimension() ) // i.e. J is square
   {
     apf::Matrix3x3 Jinv;
-    apf::getJacobianInv(me, local, Jinv);
+    apf::getJacobianInv( e->getParent(), local, Jinv );
+    apf::Matrix3x3 JinvT = apf::transpose(Jinv); 
 
     // u(x_hat) * J(x_hat)^{-1}
     for( size_t i = 0; i < values.size(); i++ ) {
       for ( int j = 0; j < 3; j++ ) {
         values[i][j] = 0.;
         for ( int k = 0; k < 3; k++ )
-          values[i][j] += vvals[i][k] * Jinv[k][j];
+          values[i][j] += vvals[i][k] * JinvT[k][j];
       }
     }
   }
   else
   {
-    // TODO
+    // TODO when reference dimension != mesh space dimension. Psedoinverse needed.
   }
-  apf::destroyMeshElement(me);
 }
 
+void getCurlShapeValues(Element* e, Vector3 const& local,
+    NewArray<Vector3>& values)
+{
+  NewArray<Vector3> cvals(values.size());
+  e->getShape()->getLocalVectorCurls(e->getMesh(), e->getEntity(), local, cvals);
+
+  // Perform Piola transformation
+  if (e->getDimension() == 3)
+  {
+    apf::Matrix3x3 J;
+    apf::getJacobian( e->getParent(), local, J);
+    double jdet = apf::getJacobianDeterminant(J, e->getDimension() );
+
+    // mult J * cvals^T and divide by jdet
+    mth::Matrix <double> cvalsT(e->getDimension(), cvals.size()); // cvals transpose
+    for (int i = 0; i < e->getDimension(); i++)
+      for (size_t j = 0; j < cvals.size(); j++)
+        cvalsT(i,j) = cvals[j][i];
+
+    mth::Matrix <double> JT(e->getDimension(), e->getDimension()); // J transpose
+    for (int i = 0; i < e->getDimension(); i++)
+      for (int j = 0; j < e->getDimension(); j++)
+        JT(i,j) = J[j][i];
+
+    mth::Matrix <double> physCurlShapes(e->getDimension(), cvals.size());
+    mth::multiply(JT, cvalsT, physCurlShapes);
+    physCurlShapes *= 1./jdet;
+
+    for (size_t i = 0; i < values.size(); i++)
+      for (int j = 0; j < e->getDimension(); j++)
+        values[i][j] = physCurlShapes(j,i);
+  }
+  else
+  {
+    // TODO for 2d
+  }
+
+
+}
 FieldShape* getShape(Field* f)
 {
   return f->getShape();
diff --git a/apf/apf.h b/apf/apf.h
index 991972853..5c5b02755 100644
--- a/apf/apf.h
+++ b/apf/apf.h
@@ -610,6 +610,13 @@ void getShapeGrads(Element* e, Vector3 const& local,
 void getVectorShapeValues(Element* e, Vector3 const& local,
     NewArray<Vector3>& values);
 
+/** \brief Returns the vector curl shape function values at a point
+ *  \details used only for Nedelec shapes
+ *  (Piola transformation used to map from parent to physical coordinates)
+  */
+void getCurlShapeValues(Element* e, Vector3 const& local,
+    NewArray<Vector3>& values);
+
 /** \brief Retrieve the apf::FieldShape used by a field
   */
 FieldShape* getShape(Field* f);
diff --git a/apf/apfElement.cc b/apf/apfElement.cc
index 5cdacc434..0b9df9436 100644
--- a/apf/apfElement.cc
+++ b/apf/apfElement.cc
@@ -94,7 +94,7 @@ void Element::getComponents(Vector3 const& xi, double* c)
   // handle cases with vector shape functions
   if (field->getShape()->isVectorShape()) {
     NewArray<Vector3> shapeValues;
-    //shape->getVectorValues(mesh, entity, xi, shapeValues); //TODO remove this
+    shapeValues.allocate(nen);
     getVectorShapeValues(this, xi, shapeValues);
     for (int ci = 0; ci < 3; ci++)
       c[ci] = 0.;
diff --git a/apf/apfMixedVectorElement.cc b/apf/apfMixedVectorElement.cc
index 9d08d579c..3d975e5ea 100644
--- a/apf/apfMixedVectorElement.cc
+++ b/apf/apfMixedVectorElement.cc
@@ -5,6 +5,8 @@
  * BSD license as described in the LICENSE file in the top-level directory.
  */
 
+#include <pcu_util.h>
+
 #include "apfVectorElement.h"
 #include "apfMixedVectorElement.h"
 #include "apfMixedVectorField.h"
@@ -23,9 +25,16 @@ MixedVectorElement::MixedVectorElement(MixedVectorField* f, VectorElement* p):
 
 void MixedVectorElement::curl(Vector3 const& xi, Vector3& c)
 {
-  //TODO to be completed using the curl of nedelec shapes and the nodeData
-  (void) xi;
-  (void) c;
+  PCU_ALWAYS_ASSERT_VERBOSE( field->getShape()->isVectorShape(),
+      "Not applicable for non-vector shape functions!");
+    NewArray<Vector3> curlShapeValues;
+    curlShapeValues.allocate(nen);
+    getCurlShapeValues(this, xi, curlShapeValues);
+    for (int ci = 0; ci < 3; ci++)
+      c[ci] = 0.;
+    for (int ni = 0; ni < nen; ni++)
+      for (int ci = 0; ci < 3; ci++)
+      	c[ci] += nodeData[ni] * curlShapeValues[ni][ci];
 }
 
 }//namespace apf
diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index 409e92f6f..0a9f346e3 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -532,7 +532,6 @@ class Nedelec: public FieldShape {
       const double c = 0.; // center of edge
     public:
       int getOrder() {return P;}
-      int getRefDim() {return dim;}
       void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<double>&) const
       {
@@ -567,7 +566,6 @@ class Nedelec: public FieldShape {
       const double c = 1./3.; // center of tri
     public:
       int getOrder() {return P;}
-      int getRefDim() {return dim;}
       void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<double>&) const
       {
@@ -704,7 +702,6 @@ class Nedelec: public FieldShape {
       const int dim = 3;
       const double c = 1./4.;
     public:
-      int getRefDim() {return dim;}
       int getOrder() {return P;}
       void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<double>&) const
@@ -927,9 +924,9 @@ class Nedelec: public FieldShape {
         }
 
 
-	mth::Matrix<double> Q(dof, dof);
-	mth::Matrix<double> R(dof, dof);
-	getTi(P, apf::Mesh::TET, Q, R);
+      	mth::Matrix<double> Q(dof, dof);
+	      mth::Matrix<double> R(dof, dof);
+	      getTi(P, apf::Mesh::TET, Q, R);
 
         mth::Matrix<double> S(dof, dim);
       	for(int i = 0; i < dim; i++) // S = Ti * u
diff --git a/apf/apfShape.cc b/apf/apfShape.cc
index b661a0855..c49b00e31 100644
--- a/apf/apfShape.cc
+++ b/apf/apfShape.cc
@@ -34,6 +34,12 @@ FieldShape::~FieldShape()
 {
 }
 
+void EntityShape::getLocalVectorCurls(Mesh* , MeshEntity* ,
+	  Vector3 const&, NewArray<Vector3>&) const
+{
+  fail("unimplemented getLocalVectorCurls called");
+}
+
 void FieldShape::getNodeXi(int, int, Vector3&)
 {
   fail("unimplemented getNodeXi called");
@@ -44,11 +50,6 @@ void FieldShape::getNodeTangent(int, int, Vector3&)
   fail("unimplemented getNodeTangent called");
 }
 
-int EntityShape::getRefDim()
-{
-  fail("unimplemented getRefDim called");
-}
-
 bool FieldShape::isVectorShape()
 {
   return false;
diff --git a/apf/apfShape.h b/apf/apfShape.h
index 8122a8457..1b02b110f 100644
--- a/apf/apfShape.h
+++ b/apf/apfShape.h
@@ -51,6 +51,15 @@ class EntityShape
         MeshEntity* e,
         Vector3 const& xi,
         NewArray<apf::Vector3>& values) const = 0;
+/** \brief evaluate element vector curl shape functions
+ \details this is used only for Nedelec
+ \param xi the parent element coordinates
+ \param values each entry is the vector shape function value for one node */
+    virtual void getLocalVectorCurls(
+        Mesh* m,
+        MeshEntity* e,
+        Vector3 const& xi,
+        NewArray<apf::Vector3>& values) const;
 /** \brief return the number of nodes affecting this element
     \details in a linear mesh, there are two nodes affecting
              and edge, three nodes affecting a triangle,
@@ -74,9 +83,6 @@ class EntityShape
     help of apf::getAlignment */
     virtual void alignSharedNodes(Mesh* m,
         MeshEntity* elem, MeshEntity* shared, int order[]);
-/** \brief Get the parent element dimension
- * \details this is not always applicable */
-    virtual int getRefDim();
 };
 
 /** \brief Describes field distribution and shape functions

From 0860a840bcc6895dd09430e9a4c8f14e49ab0732 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Fri, 10 Apr 2020 16:08:50 -0400
Subject: [PATCH 091/555] fixes bugs in alignSharedNodes for ND shapes.

---
 apf/apfNedelec.cc | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index 0a9f346e3..8d9bc5ac0 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -583,13 +583,12 @@ class Nedelec: public FieldShape {
 	      int which, rotate;
 	      bool flip;
         getAlignment(m,elem,shared,which,flip,rotate);
-        int n = getOrder();
         if(!flip)
-          for(int i = 0; i < n; ++i)
+          for(int i = 0; i < P; ++i)
             order[i] = i;
         else
-          for(int i = 0; i < n; ++i)
-            order[i] = n-1-i;
+          for(int i = 0; i < P; ++i)
+            order[i] = P-1-i;
       }
       void getVectorValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const& xi, apf::NewArray<apf::Vector3>& shapes) const
@@ -722,20 +721,18 @@ class Nedelec: public FieldShape {
         getAlignment(m,elem,shared,which,flip,rotate);
         if(m->getType(shared) == apf::Mesh::EDGE)
         {
-          int n = countNodes();
           if(!flip)
-            for(int i = 0; i < n; ++i)
+            for(int i = 0; i < P; ++i)
               order[i] = i;
           else
-            for(int i = 0; i < n; ++i)
-              order[i] = n-1-i;
+            for(int i = 0; i < P; ++i)
+              order[i] = P-1-i;
           return;
         }
         //must be a triangle
-        int k = getOrder();
-        int non = k*(k-1); // nodes on the face only
+        int non = P*(P-1); // nodes on the face only
         int unique_non = non/2; // bc 2 dofs per node on the face
-        if ( k > 1 ) // face nodes appear for k > 1
+        if ( P > 1 ) // face nodes appear for k > 1
         {
           int size = (-1 + sqrt(1+8*unique_non) )/2;
 

From 10df87b841433d1f2469f9c6203ddc92c8bb6aff Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Fri, 10 Apr 2020 18:10:31 -0400
Subject: [PATCH 092/555] Updates reorderData to account for - indexes

This is to handle Vector Shape (e.g. Nedelec) properly. This change
should not affect regular Nodal fields.
---
 apf/apfFieldData.cc | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/apf/apfFieldData.cc b/apf/apfFieldData.cc
index 2cc1a2c8b..70aea4fd2 100644
--- a/apf/apfFieldData.cc
+++ b/apf/apfFieldData.cc
@@ -198,13 +198,18 @@ void FieldDataOf<T>::getNodeComponents(MeshEntity* e, int node, T* components)
     components[i] = allComponents[node*nc+i];
 }
 
+// Note that the values in the array "order" are allowed to be negative
+// for cases with vector shape functions such as Nedelec Shapes.
+// For such cases the absolute value is used to locate the data,
+// and the sign is multiplied to the data value at that location.
 template <class T>
 void reorderData(T const dataIn[], T dataOut[], int const order[], int nc, int nn)
 {
   for (int i = 0; i < nn; ++i) {
-    int oi = order[i];
+    int oi = order[i] >= 0 ? order[i] : -order[i];
     for (int j = 0; j < nc; ++j)
-      dataOut[oi * nc + j] = dataIn[i * nc + j];
+      dataOut[oi * nc + j] =
+      	order[i] >= 0 ? dataIn[i * nc + j] : -dataIn[i * nc + j];
   }
 }
 

From 883f2e9f94c8a62332b882423e0e2058a52d918b Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Sun, 12 Apr 2020 19:32:05 -0400
Subject: [PATCH 093/555] fixes the neg ND edge dofs for misaligned edges

---
 apf/apfNedelec.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index 8d9bc5ac0..36a6e1e46 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -588,7 +588,7 @@ class Nedelec: public FieldShape {
             order[i] = i;
         else
           for(int i = 0; i < P; ++i)
-            order[i] = P-1-i;
+            order[i] = -(P-1-i);
       }
       void getVectorValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const& xi, apf::NewArray<apf::Vector3>& shapes) const
@@ -726,7 +726,7 @@ class Nedelec: public FieldShape {
               order[i] = i;
           else
             for(int i = 0; i < P; ++i)
-              order[i] = P-1-i;
+              order[i] = -(P-1-i);
           return;
         }
         //must be a triangle

From b3b072fd801bcb627fd80c457bb019820cc31ef4 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Mon, 13 Apr 2020 14:34:31 -0400
Subject: [PATCH 094/555] Treats vector shapes differently in getElementData

Note that vector shapes for edges have to account for ordering and
directions since even for first order nedelc elements
---
 apf/apfFieldData.cc | 33 +++++++++++++++++++++++----------
 1 file changed, 23 insertions(+), 10 deletions(-)

diff --git a/apf/apfFieldData.cc b/apf/apfFieldData.cc
index 70aea4fd2..98f8b71ff 100644
--- a/apf/apfFieldData.cc
+++ b/apf/apfFieldData.cc
@@ -237,16 +237,29 @@ int FieldDataOf<T>::getElementData(MeshEntity* entity, NewArray<T>& data)
       int na = mesh->getDownward(entity,d,a);
       for (int i = 0; i < na; ++i) {
         int nan = fs->countNodesOn(mesh->getType(a[i]));
-        if (nan > 1 && ed != d) { /* multiple shared nodes, check alignment */
-          order.setSize(nen); /* nen >= nan */
-          adata.setSize(nen); /* setSize is no-op for the same size */
-          es->alignSharedNodes(mesh, entity, a[i], &order[0]);
-          get(a[i], &adata[0]);
-          reorderData<T>(&adata[0], &data[n], &order[0], nc, nan);
-        } else if (nan) { /* non-zero set of nodes, either one
-                             or not shared */
-          get(a[i], &data[n]);
-        }
+        // for vector shapes (i.e., nedelec) direction matters for nan>=1
+        if (fs->isVectorShape()) {
+	  if (nan >= 1 && ed != d) {
+	    order.setSize(nen);
+	    adata.setSize(nen);
+	    es->alignSharedNodes(mesh, entity, a[i], &order[0]);
+	    get(a[i], &adata[0]);
+	    reorderData<T>(&adata[0], &data[n], &order[0], nc, nan);
+	  }
+	}
+        // for non vector shapes direction matters for nan>1
+	else {
+	  if (nan > 1 && ed != d) { /* multiple shared nodes, check alignment */
+	    order.setSize(nen); /* nen >= nan */
+	    adata.setSize(nen); /* setSize is no-op for the same size */
+	    es->alignSharedNodes(mesh, entity, a[i], &order[0]);
+	    get(a[i], &adata[0]);
+	    reorderData<T>(&adata[0], &data[n], &order[0], nc, nan);
+	  } else if (nan) { /* non-zero set of nodes, either one
+			      or not shared */
+	    get(a[i], &data[n]);
+	  }
+	}
         n += nc * nan;
       }
     }

From f148bfcc2b45b6c00f978b513303b57b9ac5ac60 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Mon, 13 Apr 2020 14:40:19 -0400
Subject: [PATCH 095/555] Updates the reorderData

The previous version would not negate the value of the 0 index. So we
have to follow a notation similar to mfem. For example in the case of
edges in a tet we have

different local/global dirs     same local/global dirs
........., -3, -2, -1        |         0, 1, 2, ..........
where
the value is the same at 0 and -1 except for a minus sign
the value is the same at 1 and -2 except for a minus sign
the value is the same at 2 and -3 except for a minus sign
and so on.

alignSharedNodes in apfNedelec EntityShapes have to be updated to
account for this.
---
 apf/apfFieldData.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/apf/apfFieldData.cc b/apf/apfFieldData.cc
index 98f8b71ff..0e3d249ec 100644
--- a/apf/apfFieldData.cc
+++ b/apf/apfFieldData.cc
@@ -200,13 +200,13 @@ void FieldDataOf<T>::getNodeComponents(MeshEntity* e, int node, T* components)
 
 // Note that the values in the array "order" are allowed to be negative
 // for cases with vector shape functions such as Nedelec Shapes.
-// For such cases the absolute value is used to locate the data,
+// For such cases the absolute value of (order+1) is used to locate the data,
 // and the sign is multiplied to the data value at that location.
 template <class T>
 void reorderData(T const dataIn[], T dataOut[], int const order[], int nc, int nn)
 {
   for (int i = 0; i < nn; ++i) {
-    int oi = order[i] >= 0 ? order[i] : -order[i];
+    int oi = order[i] >= 0 ? order[i] : -(order[i]+1);
     for (int j = 0; j < nc; ++j)
       dataOut[oi * nc + j] =
       	order[i] >= 0 ? dataIn[i * nc + j] : -dataIn[i * nc + j];

From cbaf3e89ed637ea93f6fb7bbeb4c8431fe3a8569 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Mon, 13 Apr 2020 15:41:20 -0400
Subject: [PATCH 096/555] implements MFEM node ordering for neg ND dofs

---
 apf/apfNedelec.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index 36a6e1e46..23d6760dd 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -588,7 +588,7 @@ class Nedelec: public FieldShape {
             order[i] = i;
         else
           for(int i = 0; i < P; ++i)
-            order[i] = -(P-1-i);
+            order[i] = -(P-1-i)-1; //following MFEM ordering
       }
       void getVectorValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const& xi, apf::NewArray<apf::Vector3>& shapes) const
@@ -726,7 +726,7 @@ class Nedelec: public FieldShape {
               order[i] = i;
           else
             for(int i = 0; i < P; ++i)
-              order[i] = -(P-1-i);
+              order[i] = -(P-1-i)-1; //following MFEM ordering
           return;
         }
         //must be a triangle

From a23a10d7ccfd55e7e870b8b9da96a38928fa7e1c Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 14 Apr 2020 22:29:34 -0400
Subject: [PATCH 097/555] Changes edge tangent vector from (100) to (200)

Edges are parametrized from -1 to 1 so the size of the tangent vector is
actually 2. With this, there is no need for the factor of 2 when
computing the dofs on edges for nedelec shapes.
---
 apf/apfNedelec.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index 23d6760dd..946047483 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -1034,7 +1034,10 @@ class Nedelec: public FieldShape {
     {
       if(type == Mesh::EDGE)
       {
-        t = Vector3( 1., 0., 0.);
+      	// Edges are parametrized from -1 to 1.
+      	// Having the 2 here enables us to avoid multiplying dofs by 2 for
+      	// when computing them for edges
+        t = Vector3( 2., 0., 0.);
         return;
       }
       else if(type == Mesh::TRIANGLE)

From 9f8e62a0f54e77ca17c9cea56b2522c674d14209 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Wed, 15 Apr 2020 11:57:52 -0400
Subject: [PATCH 098/555] Adds to dos to handle alignment for Nedelec shape

---
 apf/apfFieldData.cc | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/apf/apfFieldData.cc b/apf/apfFieldData.cc
index 0e3d249ec..8fd586ed8 100644
--- a/apf/apfFieldData.cc
+++ b/apf/apfFieldData.cc
@@ -240,10 +240,16 @@ int FieldDataOf<T>::getElementData(MeshEntity* entity, NewArray<T>& data)
         // for vector shapes (i.e., nedelec) direction matters for nan>=1
         if (fs->isVectorShape()) {
 	  if (nan >= 1 && ed != d) {
-	    order.setSize(nen);
+	    // The first  nen ints would tell you the first component
+	    // The second nen ints would tell you the second component
+	    // The last   nen ints would tell you the whether to add them
+	    order.setSize(3*nen);
 	    adata.setSize(nen);
+	    // TODO: alignSharedNodes need to be updated for this extra info
 	    es->alignSharedNodes(mesh, entity, a[i], &order[0]);
 	    get(a[i], &adata[0]);
+	    // TODO: We would want to have a different reorder here to handle
+	    // the fact that order now includes some extra info
 	    reorderData<T>(&adata[0], &data[n], &order[0], nc, nan);
 	  }
 	}

From 2c0899a0c359f90e92c21cd1bb721c7239e6f948 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Thu, 16 Apr 2020 12:18:50 -0400
Subject: [PATCH 099/555] Adds 7th edge tangent list to use for tri/tet

This is to make implementation of the alignSharedNodes easier.
Now all the faces will have tangent pointing away from the corner.
---
 apf/apfNedelec.cc | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index 946047483..436f57117 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -293,13 +293,16 @@ static void computeTetTi(
 {
   int non = countTetNodes(P);
   const double c = 1./4.;
-  const double tk[18] = { /* edge directions in a tet */
+  const double tk[21] = { /* edge directions in a tet */
      1. ,  0.,  0.,
     -1. ,  1.,  0.,
      0. , -1.,  0.,
      0. ,  0.,  1.,
     -1. ,  0.,  1.,
-     0.,  -1.,  1.};
+     0.,  -1.,  1.,
+     // this last one is the negative of the third one
+     // and it is used for face and tet tangents only
+     0.,   1.,  0.};
   const double *eop = getOpenPoints(P - 1);
   const double *fop = (P > 1) ? getOpenPoints(P - 2) : NULL;
   const double *iop = (P > 2) ? getOpenPoints(P - 3) : NULL;
@@ -358,7 +361,7 @@ static void computeTetTi(
       nodes[o][0] = fop[i]/w;  nodes[o][1] = fop[j]/w;  nodes[o][2] = 0.;
       dof2tk[o++] = 0;
       nodes[o][0] = fop[i]/w;  nodes[o][1] = fop[j]/w;  nodes[o][2] = 0.;
-      dof2tk[o++] = 2;
+      dof2tk[o++] = 6;
     }
   }
   // (0,1,3)
@@ -389,7 +392,7 @@ static void computeTetTi(
     {
       double w = fop[i] + fop[j] + fop[pm2-i-j];
       nodes[o][0] = 0.;  nodes[o][1] = fop[i]/w;  nodes[o][2] = fop[j]/w;
-      dof2tk[o++] = 2;
+      dof2tk[o++] = 6;
       nodes[o][0] = 0.;  nodes[o][1] = fop[i]/w;  nodes[o][2] = fop[j]/w;
       dof2tk[o++] = 3;
     }
@@ -403,7 +406,7 @@ static void computeTetTi(
         nodes[o][0] = iop[i]/w;  nodes[o][1] = iop[j]/w;  nodes[o][2] = iop[k]/w;
         dof2tk[o++] = 0;
         nodes[o][0] = iop[i]/w;  nodes[o][1] = iop[j]/w;  nodes[o][2] = iop[k]/w;
-        dof2tk[o++] = 2;
+        dof2tk[o++] = 6;
         nodes[o][0] = iop[i]/w;  nodes[o][1] = iop[j]/w;  nodes[o][2] = iop[k]/w;
         dof2tk[o++] = 3;
       }
@@ -1052,7 +1055,7 @@ class Nedelec: public FieldShape {
         PCU_ALWAYS_ASSERT_VERBOSE(P >= 3,
             "volume nodes appear only for order bigger than or equal to 3!");
        if (node % 3 == 0) t = Vector3(1., 0., 0.);
-       else if (node % 3 == 1) t = Vector3(0., -1., 0.);
+       else if (node % 3 == 1) t = Vector3(0., 1., 0.);
        else t = Vector3(0., 0., 1.);
        return;
       }

From bdd7deb9104830db6a09b7c45a590cc6109a465f Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Thu, 16 Apr 2020 18:31:43 -0400
Subject: [PATCH 100/555] alignSharedNodes & reorder data for ND face nodes.

---
 apf/apf.cc          |   2 +-
 apf/apfFieldData.cc |  30 ++++++++-
 apf/apfNedelec.cc   | 155 +++++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 181 insertions(+), 6 deletions(-)

diff --git a/apf/apf.cc b/apf/apf.cc
index 398ec22da..271b9436d 100644
--- a/apf/apf.cc
+++ b/apf/apf.cc
@@ -470,7 +470,7 @@ void getVectorShapeValues(Element* e, Vector3 const& local,
   {
     apf::Matrix3x3 Jinv;
     apf::getJacobianInv( e->getParent(), local, Jinv );
-    apf::Matrix3x3 JinvT = apf::transpose(Jinv); 
+    apf::Matrix3x3 JinvT = apf::transpose(Jinv);
 
     // u(x_hat) * J(x_hat)^{-1}
     for( size_t i = 0; i < values.size(); i++ ) {
diff --git a/apf/apfFieldData.cc b/apf/apfFieldData.cc
index 8fd586ed8..45e71ad79 100644
--- a/apf/apfFieldData.cc
+++ b/apf/apfFieldData.cc
@@ -3,7 +3,8 @@
 #include "apfShape.h"
 #include <pcu_util.h>
 #include <cstdlib>
-
+using namespace std;
+#include <iostream>
 namespace apf {
 
 FieldData::~FieldData()
@@ -213,6 +214,26 @@ void reorderData(T const dataIn[], T dataOut[], int const order[], int nc, int n
   }
 }
 
+// This is only used to reorder the data for interior face nodes on a face of
+// a Nedelec tet, where each node on the face contains 2 dof values.
+template <class T>
+void reorderFaceData(T const dataIn[], T dataOut[], int const order[], int nc, int nn)
+{
+  for (int i = 0; i < nn; ++i) {
+    if(order[2*nn+i])
+    {
+      dataOut[i*nc] = (order[i] >= 0) ? dataIn[ order[i] ] : -dataIn[ -(order[i]+1) ];
+    }
+    else
+      dataOut[i*nc] = 0.;
+
+    if(order[3*nn+i])
+    {
+      dataOut[i*nc] += (order[1*nn+i] >= 0) ? dataIn[ order[1*nn+i] ] : -dataIn[ -(order[1*nn+i]+1) ];
+    }
+  }
+}
+
 template <class T>
 int FieldDataOf<T>::getElementData(MeshEntity* entity, NewArray<T>& data)
 {
@@ -243,14 +264,17 @@ int FieldDataOf<T>::getElementData(MeshEntity* entity, NewArray<T>& data)
 	    // The first  nen ints would tell you the first component
 	    // The second nen ints would tell you the second component
 	    // The last   nen ints would tell you the whether to add them
-	    order.setSize(3*nen);
+	    order.setSize(4*nen);
 	    adata.setSize(nen);
 	    // TODO: alignSharedNodes need to be updated for this extra info
 	    es->alignSharedNodes(mesh, entity, a[i], &order[0]);
 	    get(a[i], &adata[0]);
 	    // TODO: We would want to have a different reorder here to handle
 	    // the fact that order now includes some extra info
-	    reorderData<T>(&adata[0], &data[n], &order[0], nc, nan);
+      if (mesh->getType(a[i]) == apf::Mesh::TRIANGLE)
+        reorderFaceData<T>(&adata[0], &data[n], &order[0], nc, nan);
+      else
+	      reorderData<T>(&adata[0], &data[n], &order[0], nc, nan);
 	  }
 	}
         // for non vector shapes direction matters for nan>1
diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index 436f57117..0713f29e8 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -20,6 +20,152 @@ using namespace std;
 
 namespace apf {
 
+int* alignFaceNodes(int init_order[], int nodes, int r, bool f)
+{
+  int size = 4*nodes;
+  int* final_order = new int[size];
+
+  if (r == 0 && !f)     // CASE 1
+  {
+    int ind = 0;
+    for (int i = 0; i < nodes; i++)
+      final_order[i] = init_order[i];
+    ind += nodes;
+    for (int i = 0 ; i < nodes; i++)
+      final_order[i+ind] = init_order[i];
+    ind += nodes;
+    for (int i = 0; i < nodes; i++)
+      final_order[i+ind] = 1;
+    ind += nodes;
+    for (int i = 0; i < nodes; i++)
+      final_order[i+ind] = 0;
+  }
+  else if (r == 1 && !f) // CASE 2
+  {
+    int ind = 0;
+    int offset = 0;
+    for (int i = 0; i < nodes; i++)
+    {
+      final_order[i] = -init_order[offset] - 1;
+      final_order[i+1] = -init_order[offset] - 1;
+      offset += 2;
+      i++;
+    }
+    ind += nodes;
+    offset = 1;
+    for (int i = 0 ; i < nodes; i++)
+    {
+      final_order[i+ind] = init_order[offset];
+      final_order[i+1+ind] = init_order[offset];
+      offset += 2;
+      i++;
+    }
+    ind += nodes;
+    for (int i = 0; i < nodes; i++)
+      final_order[i+ind] = 1;
+    ind += nodes;
+    for (int i = 0; i < nodes; i++)
+      final_order[i+ind] = (i%2) ? 0 : 1;
+  }
+  else if (r == 2 && !f) // CASE 3
+  {
+    int ind = 0;
+    int offset = 0;
+    for (int i = 0; i < nodes; i++)
+    {
+      final_order[i] = init_order[offset];
+      final_order[i+1] = init_order[offset];
+      offset += 2;
+      i++;
+    }
+    ind += nodes;
+    offset = 1;
+    for (int i = 0 ; i < nodes; i++)
+    {
+      final_order[i+ind] = -init_order[offset]-1;
+      final_order[i+1+ind] = -init_order[offset]-1;
+      offset += 2;
+      i++;
+    }
+    ind += nodes;
+    for (int i = 0; i < nodes; i++)
+      final_order[i+ind] = (i%2) ? 1 : 0;
+    ind += nodes;
+    for (int i = 0; i < nodes; i++)
+      final_order[i+ind] = 1;
+  }
+  else if (r == 0 && f)  // CASE 4
+  {
+    int ind = 0;
+    int offset = 0;
+    for (int i = 0; i < nodes; i++)
+    {
+      final_order[i] = init_order[offset];
+      final_order[i+1] = init_order[offset];
+      offset += 2;
+      i++;
+    }
+    ind += nodes;
+    offset = 1;
+    for (int i = 0 ; i < nodes; i++)
+    {
+      final_order[i+ind] = -init_order[offset]-1;
+      final_order[i+1+ind] = -init_order[offset]-1;
+      offset += 2;
+      i++;
+    }
+    ind += nodes;
+    for (int i = 0; i < nodes; i++)
+      final_order[i+ind] = (i%2) ? 0 : 1;
+    ind += nodes;
+    for (int i = 0; i < nodes; i++)
+      final_order[i+ind] = 1;
+  }
+  else if (r == 1 && f)  // CASE 5
+  {
+    int ind = 0;
+    int offset = 0;
+    for (int i = 0; i < nodes; i++)
+    {
+      final_order[i] = -init_order[offset]-1;
+      final_order[i+1] = -init_order[offset]-1;
+      offset += 2;
+      i++;
+    }
+    ind += nodes;
+    offset = 1;
+    for (int i = 0 ; i < nodes; i++)
+    {
+      final_order[i+ind] = init_order[offset];
+      final_order[i+1+ind] = init_order[offset];
+      offset += 2;
+      i++;
+    }
+    ind += nodes;
+    for (int i = 0; i < nodes; i++)
+      final_order[i+ind] = 1;
+    ind += nodes;
+    for (int i = 0; i < nodes; i++)
+      final_order[i+ind] = (i%2) ? 1 : 0;
+  }
+  else if (r == 2 && f)  // CASE 6
+  {
+    int ind = 0;
+    for (int i = 0; i < nodes; i++)
+      final_order[i] = init_order[nodes-1-i];
+    ind += nodes;
+    for (int i = 0 ; i < nodes; i++)
+      final_order[i+ind] = init_order[nodes-1-i];
+    ind += nodes;
+    for (int i = 0; i < nodes; i++)
+      final_order[i+ind] = 0;
+    ind += nodes;
+    for (int i = 0; i < nodes; i++)
+      final_order[i+ind] = 1;
+  }
+  return final_order;
+}
+
 static unsigned const MAX_ND_ORDER = 10;
 enum {
   GAUSS_LEGENDRE,
@@ -781,12 +927,17 @@ class Nedelec: public FieldShape {
               for(int c = 0; c < size; c++)
                 Nodes(r,c) = temp(r,c);
           }
-          // get the ordered list
+          // get the init ordered list with all face nodes
+          int init_order[non];
           int i = 0;
           for ( int r = size-1; r >= 0; r--)
             for (int c = size-r-1 ; c < size; c++) {
-              order[i++] = Nodes(r,c)*2;    order[i++] = Nodes(r,c)*2 + 1;
+              init_order[i++] = Nodes(r,c)*2;    init_order[i++] = Nodes(r,c)*2 + 1;
             }
+
+          int* final_order = alignFaceNodes(init_order, non, rotate, flip);
+          for (int i = 0; i < 4*non; i++)
+            order[i] = final_order[i];
         }
       }
       void getVectorValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,

From 7e01aae39ce0fd9e14aeb3f57c794fb343026f3d Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Sun, 19 Apr 2020 01:05:27 -0400
Subject: [PATCH 101/555] Includes recent changes to faceAlignNodes

---
 apf/apfNedelec.cc | 115 ++++++++++++++++++++++++++++++++--------------
 1 file changed, 80 insertions(+), 35 deletions(-)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index 0713f29e8..339c5d58c 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -28,17 +28,29 @@ int* alignFaceNodes(int init_order[], int nodes, int r, bool f)
   if (r == 0 && !f)     // CASE 1
   {
     int ind = 0;
+    int offset = 0;
     for (int i = 0; i < nodes; i++)
-      final_order[i] = init_order[i];
+    {
+      final_order[i] = init_order[offset];
+      final_order[i+1] = init_order[offset];
+      offset += 2;
+      i++;
+    }
     ind += nodes;
+    offset = 1;
     for (int i = 0 ; i < nodes; i++)
-      final_order[i+ind] = init_order[i];
+    {
+      final_order[i+ind] = init_order[offset];
+      final_order[i+1+ind] = init_order[offset];
+      offset += 2;
+      i++;
+    }
     ind += nodes;
     for (int i = 0; i < nodes; i++)
-      final_order[i+ind] = 1;
+      final_order[i+ind] = (i%2) ? 0 : 1;
     ind += nodes;
     for (int i = 0; i < nodes; i++)
-      final_order[i+ind] = 0;
+      final_order[i+ind] = (i%2) ? 1 : 0;
   }
   else if (r == 1 && !f) // CASE 2
   {
@@ -46,8 +58,8 @@ int* alignFaceNodes(int init_order[], int nodes, int r, bool f)
     int offset = 0;
     for (int i = 0; i < nodes; i++)
     {
-      final_order[i] = -init_order[offset] - 1;
-      final_order[i+1] = -init_order[offset] - 1;
+      final_order[i] = init_order[offset];
+      final_order[i+1] = init_order[offset];
       offset += 2;
       i++;
     }
@@ -55,17 +67,17 @@ int* alignFaceNodes(int init_order[], int nodes, int r, bool f)
     offset = 1;
     for (int i = 0 ; i < nodes; i++)
     {
-      final_order[i+ind] = init_order[offset];
-      final_order[i+1+ind] = init_order[offset];
+      final_order[i+ind] = -init_order[offset] - 1;
+      final_order[i+1+ind] = -init_order[offset] - 1;
       offset += 2;
       i++;
     }
     ind += nodes;
     for (int i = 0; i < nodes; i++)
-      final_order[i+ind] = 1;
+      final_order[i+ind] = (i%2) ? 1 : 0;
     ind += nodes;
     for (int i = 0; i < nodes; i++)
-      final_order[i+ind] = (i%2) ? 0 : 1;
+      final_order[i+ind] = 1;
   }
   else if (r == 2 && !f) // CASE 3
   {
@@ -73,8 +85,8 @@ int* alignFaceNodes(int init_order[], int nodes, int r, bool f)
     int offset = 0;
     for (int i = 0; i < nodes; i++)
     {
-      final_order[i] = init_order[offset];
-      final_order[i+1] = init_order[offset];
+      final_order[i] = -init_order[offset] - 1;
+      final_order[i+1] = -init_order[offset] - 1;
       offset += 2;
       i++;
     }
@@ -82,17 +94,17 @@ int* alignFaceNodes(int init_order[], int nodes, int r, bool f)
     offset = 1;
     for (int i = 0 ; i < nodes; i++)
     {
-      final_order[i+ind] = -init_order[offset]-1;
-      final_order[i+1+ind] = -init_order[offset]-1;
+      final_order[i+ind] = init_order[offset];
+      final_order[i+1+ind] = init_order[offset];
       offset += 2;
       i++;
     }
     ind += nodes;
     for (int i = 0; i < nodes; i++)
-      final_order[i+ind] = (i%2) ? 1 : 0;
+      final_order[i+ind] = 1;
     ind += nodes;
     for (int i = 0; i < nodes; i++)
-      final_order[i+ind] = 1;
+      final_order[i+ind] = (i%2) ? 0 : 1;
   }
   else if (r == 0 && f)  // CASE 4
   {
@@ -109,17 +121,17 @@ int* alignFaceNodes(int init_order[], int nodes, int r, bool f)
     offset = 1;
     for (int i = 0 ; i < nodes; i++)
     {
-      final_order[i+ind] = -init_order[offset]-1;
-      final_order[i+1+ind] = -init_order[offset]-1;
+      final_order[i+ind] = init_order[offset];
+      final_order[i+1+ind] = init_order[offset];
       offset += 2;
       i++;
     }
     ind += nodes;
     for (int i = 0; i < nodes; i++)
-      final_order[i+ind] = (i%2) ? 0 : 1;
+      final_order[i+ind] = (i%2) ? 1 : 0;
     ind += nodes;
     for (int i = 0; i < nodes; i++)
-      final_order[i+ind] = 1;
+      final_order[i+ind] = (i%2) ? 0 : 1;
   }
   else if (r == 1 && f)  // CASE 5
   {
@@ -151,14 +163,26 @@ int* alignFaceNodes(int init_order[], int nodes, int r, bool f)
   else if (r == 2 && f)  // CASE 6
   {
     int ind = 0;
+    int offset = 0;
     for (int i = 0; i < nodes; i++)
-      final_order[i] = init_order[nodes-1-i];
+    {
+      final_order[i] = init_order[offset];
+      final_order[i+1] = init_order[offset];
+      offset += 2;
+      i++;
+    }
     ind += nodes;
+    offset = 1;
     for (int i = 0 ; i < nodes; i++)
-      final_order[i+ind] = init_order[nodes-1-i];
+    {
+      final_order[i+ind] = -init_order[offset]-1;
+      final_order[i+1+ind] = -init_order[offset]-1;
+      offset += 2;
+      i++;
+    }
     ind += nodes;
     for (int i = 0; i < nodes; i++)
-      final_order[i+ind] = 0;
+      final_order[i+ind] = (i%2) ? 0 : 1;
     ind += nodes;
     for (int i = 0; i < nodes; i++)
       final_order[i+ind] = 1;
@@ -893,40 +917,61 @@ class Nedelec: public FieldShape {
               Nodes(r,c) = (num++);
 
           // CASES
-          if(rotate == 1) {
+          if (rotate == 1  &&  !flip) // CASE 2
+          {
             for (int r = size-1; r >= 0; r--) {    // horizontal swaps
               int left = size-r-1; int right = size-1;
               for(int range = left; range <= left + (right-left)/2; range++) {
-                            std::swap( Nodes(r,range), Nodes(r,left+right-range) );
+                          std::swap( Nodes(r,range), Nodes(r,left+right-range) );
               }
             }
-            mth::Matrix<double> temp(size, size);
+            mth::Matrix<double> temp(size, size);  // transpose
             mth::transpose(Nodes,temp);
             for (int r = 0; r < size; r++)
               for(int c = 0; c < size; c++)
                 Nodes(r,c) = temp(r,c);
           }
-          if(rotate == 2) {
+          else if (rotate == 2  &&  !flip) // CASE 3
+          {
             for (int c = size-1; c >= 0; c--) {    // vertical swaps
-                    int left = size-c-1; int right = size-1;
-                    for(int range = left; range <= left + (right-left)/2; range++) {
-                            std::swap( Nodes(range,c), Nodes(left+right-range,c) );
-                    }
+              int left = size-c-1; int right = size-1;
+              for(int range = left; range <= left + (right-left)/2; range++) {
+                          std::swap( Nodes(range,c), Nodes(left+right-range,c) );
+              }
             }
-            mth::Matrix<double> temp(size, size);
+            mth::Matrix<double> temp(size, size);  // transpose
             mth::transpose(Nodes,temp);
             for (int r = 0; r < size; r++)
               for(int c = 0; c < size; c++)
                 Nodes(r,c) = temp(r,c);
           }
-          if(flip)
+          else if (!rotate  &&  flip) // CASE 4
+          {
+            for (int c = size-1; c >= 0; c--) {    // vertical swaps
+              int left = size-c-1; int right = size-1;
+              for(int range = left; range <= left + (right-left)/2; range++) {
+                          std::swap( Nodes(range,c), Nodes(left+right-range,c) );
+              }
+            }
+          }
+          else if (rotate == 1  &&  flip) // CASE 5
           {
-            mth::Matrix<double> temp(size, size);
+            for (int r = size-1; r >= 0; r--) {    // horizontal swaps
+              int left = size-r-1; int right = size-1;
+              for(int range = left; range <= left + (right-left)/2; range++) {
+                          std::swap( Nodes(r,range), Nodes(r,left+right-range) );
+              }
+            }
+          }
+          else if (rotate == 2  &&  flip) // CASE 6
+          {
+            mth::Matrix<double> temp(size, size);  // transpose
             mth::transpose(Nodes,temp);
             for (int r = 0; r < size; r++)
               for(int c = 0; c < size; c++)
                 Nodes(r,c) = temp(r,c);
           }
+
           // get the init ordered list with all face nodes
           int init_order[non];
           int i = 0;
@@ -934,8 +979,8 @@ class Nedelec: public FieldShape {
             for (int c = size-r-1 ; c < size; c++) {
               init_order[i++] = Nodes(r,c)*2;    init_order[i++] = Nodes(r,c)*2 + 1;
             }
-
           int* final_order = alignFaceNodes(init_order, non, rotate, flip);
+
           for (int i = 0; i < 4*non; i++)
             order[i] = final_order[i];
         }

From e156ab38445d3ff3713b3cca84d57b38e81bf03e Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Tue, 21 Apr 2020 17:59:58 -0400
Subject: [PATCH 102/555] Changes to alignSharedNodes for ND Tet.

Includes recent changes to alignSharedNodes for Tet reflecting the
appropriate interpretation of flip and rotate operations in getAlignment.
Works for up to 2nd order ND space.
---
 apf/apfNedelec.cc | 114 +++++++++++++++++++++++++++++++++++++---------
 1 file changed, 93 insertions(+), 21 deletions(-)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index 339c5d58c..dd7cfc1fc 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -58,8 +58,8 @@ int* alignFaceNodes(int init_order[], int nodes, int r, bool f)
     int offset = 0;
     for (int i = 0; i < nodes; i++)
     {
-      final_order[i] = init_order[offset];
-      final_order[i+1] = init_order[offset];
+      final_order[i] = -init_order[offset]-1;
+      final_order[i+1] = -init_order[offset]-1;
       offset += 2;
       i++;
     }
@@ -67,17 +67,17 @@ int* alignFaceNodes(int init_order[], int nodes, int r, bool f)
     offset = 1;
     for (int i = 0 ; i < nodes; i++)
     {
-      final_order[i+ind] = -init_order[offset] - 1;
-      final_order[i+1+ind] = -init_order[offset] - 1;
+      final_order[i+ind] = init_order[offset];
+      final_order[i+1+ind] = init_order[offset];
       offset += 2;
       i++;
     }
     ind += nodes;
     for (int i = 0; i < nodes; i++)
-      final_order[i+ind] = (i%2) ? 1 : 0;
+      final_order[i+ind] = 1;
     ind += nodes;
     for (int i = 0; i < nodes; i++)
-      final_order[i+ind] = 1;
+      final_order[i+ind] = (i%2) ? 0 : 1;
   }
   else if (r == 2 && !f) // CASE 3
   {
@@ -85,8 +85,8 @@ int* alignFaceNodes(int init_order[], int nodes, int r, bool f)
     int offset = 0;
     for (int i = 0; i < nodes; i++)
     {
-      final_order[i] = -init_order[offset] - 1;
-      final_order[i+1] = -init_order[offset] - 1;
+      final_order[i] = init_order[offset];
+      final_order[i+1] = init_order[offset];
       offset += 2;
       i++;
     }
@@ -94,17 +94,17 @@ int* alignFaceNodes(int init_order[], int nodes, int r, bool f)
     offset = 1;
     for (int i = 0 ; i < nodes; i++)
     {
-      final_order[i+ind] = init_order[offset];
-      final_order[i+1+ind] = init_order[offset];
+      final_order[i+ind] = -init_order[offset]-1;
+      final_order[i+1+ind] = -init_order[offset]-1;
       offset += 2;
       i++;
     }
     ind += nodes;
     for (int i = 0; i < nodes; i++)
-      final_order[i+ind] = 1;
+      final_order[i+ind] = (i%2) ? 1 : 0;
     ind += nodes;
     for (int i = 0; i < nodes; i++)
-      final_order[i+ind] = (i%2) ? 0 : 1;
+      final_order[i+ind] = 1;
   }
   else if (r == 0 && f)  // CASE 4
   {
@@ -121,17 +121,17 @@ int* alignFaceNodes(int init_order[], int nodes, int r, bool f)
     offset = 1;
     for (int i = 0 ; i < nodes; i++)
     {
-      final_order[i+ind] = init_order[offset];
-      final_order[i+1+ind] = init_order[offset];
+      final_order[i+ind] = -init_order[offset]-1;
+      final_order[i+1+ind] = -init_order[offset]-1;
       offset += 2;
       i++;
     }
     ind += nodes;
     for (int i = 0; i < nodes; i++)
-      final_order[i+ind] = (i%2) ? 1 : 0;
+      final_order[i+ind] = (i%2) ? 0 : 1;
     ind += nodes;
     for (int i = 0; i < nodes; i++)
-      final_order[i+ind] = (i%2) ? 0 : 1;
+      final_order[i+ind] = 1;
   }
   else if (r == 1 && f)  // CASE 5
   {
@@ -175,17 +175,17 @@ int* alignFaceNodes(int init_order[], int nodes, int r, bool f)
     offset = 1;
     for (int i = 0 ; i < nodes; i++)
     {
-      final_order[i+ind] = -init_order[offset]-1;
-      final_order[i+1+ind] = -init_order[offset]-1;
+      final_order[i+ind] = init_order[offset];
+      final_order[i+1+ind] = init_order[offset];
       offset += 2;
       i++;
     }
     ind += nodes;
     for (int i = 0; i < nodes; i++)
-      final_order[i+ind] = (i%2) ? 0 : 1;
+      final_order[i+ind] = (i%2) ? 1 : 0;
     ind += nodes;
     for (int i = 0; i < nodes; i++)
-      final_order[i+ind] = 1;
+      final_order[i+ind] = (i%2) ? 0 : 1;
   }
   return final_order;
 }
@@ -918,6 +918,67 @@ class Nedelec: public FieldShape {
 
           // CASES
           if (rotate == 1  &&  !flip) // CASE 2
+          {
+            for (int c = size-1; c >= 0; c--) {    // vertical swaps
+              int left = size-c-1; int right = size-1;
+              for(int range = left; range <= left + (right-left)/2; range++) {
+                          std::swap( Nodes(range,c), Nodes(left+right-range,c) );
+              }
+            }
+            mth::Matrix<double> temp(size, size);  // transpose
+            mth::transpose(Nodes,temp);
+            for (int r = 0; r < size; r++)
+              for(int c = 0; c < size; c++)
+                Nodes(r,c) = temp(r,c);
+          }
+          if (rotate == 2  &&  !flip) // CASE 3
+          {
+            for (int r = size-1; r >= 0; r--) {    // horizontal swaps
+              int left = size-r-1; int right = size-1;
+              for(int range = left; range <= left + (right-left)/2; range++) {
+                          std::swap( Nodes(r,range), Nodes(r,left+right-range) );
+              }
+            }
+            mth::Matrix<double> temp(size, size);  // transpose
+            mth::transpose(Nodes,temp);
+            for (int r = 0; r < size; r++)
+              for(int c = 0; c < size; c++)
+                Nodes(r,c) = temp(r,c);
+          }
+          else if (!rotate  &&  flip) // CASE 4
+          {
+            mth::Matrix<double> temp(size, size);  // transpose
+            mth::transpose(Nodes,temp);
+            for (int r = 0; r < size; r++)
+              for(int c = 0; c < size; c++)
+                Nodes(r,c) = temp(r,c);
+          }
+          else if (rotate == 1  &&  flip) // CASE 5
+          {
+            for (int r = size-1; r >= 0; r--) {    // horizontal swaps
+              int left = size-r-1; int right = size-1;
+              for(int range = left; range <= left + (right-left)/2; range++) {
+                          std::swap( Nodes(r,range), Nodes(r,left+right-range) );
+              }
+            }
+          }
+          else if (rotate == 2  &&  flip) // CASE 6
+          {
+            for (int c = size-1; c >= 0; c--) {    // vertical swaps
+              int left = size-c-1; int right = size-1;
+              for(int range = left; range <= left + (right-left)/2; range++) {
+                          std::swap( Nodes(range,c), Nodes(left+right-range,c) );
+              }
+            }
+          }
+
+
+
+
+
+
+          /*========
+          if (rotate == 1  &&  !flip) // CASE 2
           {
             for (int r = size-1; r >= 0; r--) {    // horizontal swaps
               int left = size-r-1; int right = size-1;
@@ -970,7 +1031,7 @@ class Nedelec: public FieldShape {
             for (int r = 0; r < size; r++)
               for(int c = 0; c < size; c++)
                 Nodes(r,c) = temp(r,c);
-          }
+          }*/
 
           // get the init ordered list with all face nodes
           int init_order[non];
@@ -983,6 +1044,17 @@ class Nedelec: public FieldShape {
 
           for (int i = 0; i < 4*non; i++)
             order[i] = final_order[i];
+
+          /* TEST
+          cout << "Initial Order " << " r " << rotate << " f " << flip << endl;
+          for (int i = 0; i < non; i++)
+            cout << init_order[i] << " ";
+          cout << endl;
+          cout << "Final Order " << " r " << rotate << " f " << flip << endl;
+          for (int i = 0; i < 4*non; i++)
+            cout << order[i] << " ";
+          cout << endl;
+          cout << endl;*/
         }
       }
       void getVectorValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,

From 4943d5fb5ae6060c440ee63acbf5835d41bf61dd Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Wed, 22 Apr 2020 02:34:40 -0400
Subject: [PATCH 103/555] Fixes the issue with tet dofs

Where the dofs are reordered for faces/edges in a tet, we had forgotten
to account for the dofs associated with the tet itself.
---
 apf/apfFieldData.cc | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/apf/apfFieldData.cc b/apf/apfFieldData.cc
index 45e71ad79..3d914ba2c 100644
--- a/apf/apfFieldData.cc
+++ b/apf/apfFieldData.cc
@@ -271,11 +271,14 @@ int FieldDataOf<T>::getElementData(MeshEntity* entity, NewArray<T>& data)
 	    get(a[i], &adata[0]);
 	    // TODO: We would want to have a different reorder here to handle
 	    // the fact that order now includes some extra info
-      if (mesh->getType(a[i]) == apf::Mesh::TRIANGLE)
-        reorderFaceData<T>(&adata[0], &data[n], &order[0], nc, nan);
-      else
+            if (mesh->getType(a[i]) == apf::Mesh::TRIANGLE)
+              reorderFaceData<T>(&adata[0], &data[n], &order[0], nc, nan);
+	    else
 	      reorderData<T>(&adata[0], &data[n], &order[0], nc, nan);
 	  }
+	  // this else is required to add the dofs associated with the tet
+	  else
+	    get(a[i], &data[n]);
 	}
         // for non vector shapes direction matters for nan>1
 	else {

From bcd84dfbf0749e8d01fc136c84052034d03bc98a Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Wed, 22 Apr 2020 02:36:58 -0400
Subject: [PATCH 104/555] Cleans up the code a bit.

a) Removes TODOs
b) Renames reorderFaceData to reorderDataNedelec and both edges and
faces are taken care of inside this function
c) Reverts reorderData to its original state.
---
 apf/apfFieldData.cc | 64 ++++++++++++++++++++++++++++-----------------
 1 file changed, 40 insertions(+), 24 deletions(-)

diff --git a/apf/apfFieldData.cc b/apf/apfFieldData.cc
index 3d914ba2c..cba4fa4d9 100644
--- a/apf/apfFieldData.cc
+++ b/apf/apfFieldData.cc
@@ -207,31 +207,49 @@ template <class T>
 void reorderData(T const dataIn[], T dataOut[], int const order[], int nc, int nn)
 {
   for (int i = 0; i < nn; ++i) {
-    int oi = order[i] >= 0 ? order[i] : -(order[i]+1);
+    int oi = order[i];
     for (int j = 0; j < nc; ++j)
-      dataOut[oi * nc + j] =
-      	order[i] >= 0 ? dataIn[i * nc + j] : -dataIn[i * nc + j];
+      dataOut[oi * nc + j] = dataIn[i * nc + j];
   }
 }
 
 // This is only used to reorder the data for interior face nodes on a face of
 // a Nedelec tet, where each node on the face contains 2 dof values.
 template <class T>
-void reorderFaceData(T const dataIn[], T dataOut[], int const order[], int nc, int nn)
+void reorderDataNedelec(
+    T const dataIn[],
+    T dataOut[],
+    int const order[],
+    int nc,
+    int nn,
+    int type)
 {
-  for (int i = 0; i < nn; ++i) {
-    if(order[2*nn+i])
-    {
-      dataOut[i*nc] = (order[i] >= 0) ? dataIn[ order[i] ] : -dataIn[ -(order[i]+1) ];
-    }
-    else
-      dataOut[i*nc] = 0.;
+  if (type == Mesh::TRIANGLE)
+    for (int i = 0; i < nn; ++i) {
+      if(order[2*nn+i])
+      {
+	dataOut[i*nc] = (order[i] >= 0) ?
+	  dataIn[ order[i] ] : -dataIn[ -(order[i]+1) ];
+      }
+      else
+	dataOut[i*nc] = 0.;
 
-    if(order[3*nn+i])
-    {
-      dataOut[i*nc] += (order[1*nn+i] >= 0) ? dataIn[ order[1*nn+i] ] : -dataIn[ -(order[1*nn+i]+1) ];
+      if(order[3*nn+i])
+      {
+	dataOut[i*nc] += (order[1*nn+i] >= 0) ?
+	  dataIn[ order[1*nn+i] ] : -dataIn[ -(order[1*nn+i]+1) ];
+      }
     }
-  }
+  else if (type == Mesh::EDGE)
+    for (int i = 0; i < nn; ++i) {
+      int oi = order[i] >= 0 ? order[i] : -(order[i]+1);
+      for (int j = 0; j < nc; ++j)
+	dataOut[oi * nc + j] =
+	  order[i] >= 0 ? dataIn[i * nc + j] : -dataIn[i * nc + j];
+    }
+  else
+    PCU_ALWAYS_ASSERT_VERBOSE(0,
+    	"type has to be Mesh::EDGE or Mesh::TRIANGLE!");
 }
 
 template <class T>
@@ -261,20 +279,18 @@ int FieldDataOf<T>::getElementData(MeshEntity* entity, NewArray<T>& data)
         // for vector shapes (i.e., nedelec) direction matters for nan>=1
         if (fs->isVectorShape()) {
 	  if (nan >= 1 && ed != d) {
-	    // The first  nen ints would tell you the first component
-	    // The second nen ints would tell you the second component
-	    // The last   nen ints would tell you the whether to add them
+	    // The 1st nen ints is the 1st contribution
+	    // The 2nd nen ints is the 2nd contribution
+	    // The 3rd nen ints tells whether to add 1st contribution
+	    // The 4th nen ints tells whether to add 2st contribution
 	    order.setSize(4*nen);
 	    adata.setSize(nen);
-	    // TODO: alignSharedNodes need to be updated for this extra info
 	    es->alignSharedNodes(mesh, entity, a[i], &order[0]);
 	    get(a[i], &adata[0]);
-	    // TODO: We would want to have a different reorder here to handle
+	    // We would want to have a different reorder here to handle
 	    // the fact that order now includes some extra info
-            if (mesh->getType(a[i]) == apf::Mesh::TRIANGLE)
-              reorderFaceData<T>(&adata[0], &data[n], &order[0], nc, nan);
-	    else
-	      reorderData<T>(&adata[0], &data[n], &order[0], nc, nan);
+	    int dtype = mesh->getType(a[i]);
+	    reorderDataNedelec<T>(&adata[0], &data[n], &order[0], nc, nan, dtype);
 	  }
 	  // this else is required to add the dofs associated with the tet
 	  else

From 4e14f100460a330e34214e2dc45f74e47ffe399d Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Mon, 27 Apr 2020 16:55:51 -0400
Subject: [PATCH 105/555] Implements rotateTetXi

---
 ma/maMesh.cc | 16 ++++++++++++++++
 ma/maMesh.h  |  1 +
 2 files changed, 17 insertions(+)

diff --git a/ma/maMesh.cc b/ma/maMesh.cc
index f5e5ffc7e..7d1b9246f 100644
--- a/ma/maMesh.cc
+++ b/ma/maMesh.cc
@@ -132,6 +132,22 @@ void unrotateTetXi(Vector& xi, int rotation)
   xi[0] = b[1]; xi[1] = b[2]; xi[2] = b[3];
 }
 
+/* given a set of element-local coordinates computed
+   based on the original set of vertices, this function
+   takes the rotation code and gives the coordinates
+   that match the coordinates computed based on a rotated
+   set of vertices */
+void rotateTetXi(Vector& xi, int rotation)
+{
+  double b[4];
+  b[0] = 1-xi[0]-xi[1]-xi[2]; b[1] = xi[0]; b[2] = xi[1]; b[3] = xi[2];
+  int const* originalIndexOf = tet_rotation[rotation];
+  double a[4];
+  for (int i = 0; i < 4; i++)
+    a[ originalIndexOf[i] ] = b[i];
+  xi[0] = a[1]; xi[1] = a[2]; xi[2] = a[3];
+}
+
 void rotateOct(Entity** iv, int n, Entity** ov)
 {
   for (int i=0; i < 6; ++i)
diff --git a/ma/maMesh.h b/ma/maMesh.h
index 724a9a92b..f5409d2b8 100644
--- a/ma/maMesh.h
+++ b/ma/maMesh.h
@@ -60,6 +60,7 @@ void rotateEntity(apf::Mesh* m, Entity* e, int n, Entity** v);
 
 int findTetRotation(Mesh* m, Entity* tet, Entity** v);
 void unrotateTetXi(Vector& xi, int rotation);
+void rotateTetXi(Vector& xi, int rotation);
 
 void rotateOct(Entity** iv, int n, Entity** ov);
 

From 121e27a3a3cf3080446256ff592a73cc5c93d57c Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Mon, 27 Apr 2020 16:59:14 -0400
Subject: [PATCH 106/555] Cleans up comments and todos

---
 apf/apfNedelec.cc | 115 ++++++++++------------------------------------
 1 file changed, 23 insertions(+), 92 deletions(-)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index dd7cfc1fc..9f4a09768 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -252,7 +252,7 @@ void getGaussLegendrePoints(int np, double* pts)
 }
 
 void getGaussLobattoPoints(int /*np*/, double* /*pts*/)
-{ /* TODO implement Gauss Lobatto points. Later when needed. */
+{ /* implement Gauss Lobatto points. Later when needed. */
 };
 
 const double* getPoints(int order, const int type)
@@ -428,7 +428,7 @@ static void computeTriangleTi(
   }
 
   // Populate T
-  mth::Matrix<double> T(non,non); // T(i,j)
+  mth::Matrix<double> T(non,non);
   for (int m = 0; m < non; m++)
   {
     const double *tm = tk + 2*dof2tk[m];
@@ -584,7 +584,7 @@ static void computeTetTi(
   }
 
   // Populate T
-  mth::Matrix<double> T(non,non); // T(i,j)
+  mth::Matrix<double> T(non,non);
   for (int m = 0; m < non; m++)
   {
     const double *tm = tk + 3*dof2tk[m];
@@ -674,8 +674,6 @@ class Nedelec: public FieldShape {
   public:
     const char* getName() const { return "Nedelec"; }
     bool isVectorShape() {return true;}
-    /* Nedelec(int order) : P(order) */
-    /* {} */
     Nedelec()
     {}
     class Vertex : public apf::EntityShape
@@ -685,11 +683,12 @@ class Nedelec: public FieldShape {
 	  apf::Vector3 const&, apf::NewArray<double>& values) const
       {
 	(void)values;
-	// TODO inform the user that this is not implemented and abort()
       }
       void getLocalGradients(apf::Mesh*, apf::MeshEntity*,
 	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
       {
+        PCU_ALWAYS_ASSERT_VERBOSE(0, "error: Nedelec shapes are not \
+            implemented for vertices. Aborting()!");
       }
       int countNodes() const {return 0;}
       void alignSharedNodes(apf::Mesh*,
@@ -708,12 +707,14 @@ class Nedelec: public FieldShape {
       void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<double>&) const
       {
-      	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getValues not implemented for Nedelec Edges. Aborting()!");
+      	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getValues not implemented for \
+      	    Nedelec Edges. Aborting()!");
       }
       void getLocalGradients(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
       {
-      	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getLocalGradients not implemented for Nedelec Edges. Aborting()!");
+      	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getLocalGradients not \
+      	    implemented for Nedelec Edges. Aborting()!");
       }
       int countNodes() const {return P;}
       void alignSharedNodes(apf::Mesh*,
@@ -724,12 +725,12 @@ class Nedelec: public FieldShape {
       void getVectorValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
       {
-      	// TODO: to be completed
+      	// : to be completed
       }
       void getLocalVectorCurls(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
       {
-      	// TODO: to be completed
+      	// : to be completed
       }
     };
     class Triangle : public apf::EntityShape
@@ -742,12 +743,14 @@ class Nedelec: public FieldShape {
       void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<double>&) const
       {
-      	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getValues not implemented for Nedelec Triangle. Try getVectorValues. Aborting()!");
+      	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getValues not implemented \
+      	    for Nedelec Triangle. Try getVectorValues. Aborting()!");
       }
       void getLocalGradients(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
       {
-      	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getLocalGradients not implemented for Nedelec Triangle. Aborting()!");
+      	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getLocalGradients not \
+      	    implemented for Nedelec Triangle. Aborting()!");
       }
       int countNodes() const {return countTriNodes(P);}
       void alignSharedNodes(apf::Mesh* m,
@@ -761,7 +764,7 @@ class Nedelec: public FieldShape {
             order[i] = i;
         else
           for(int i = 0; i < P; ++i)
-            order[i] = -(P-1-i)-1; //following MFEM ordering
+            order[i] = -(P-1-i)-1;
       }
       void getVectorValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const& xi, apf::NewArray<apf::Vector3>& shapes) const
@@ -878,12 +881,14 @@ class Nedelec: public FieldShape {
       void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<double>&) const
       {
-      	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getValues not implemented for Nedelec Tetrahedron. Try getVectorValues. Aborting()!");
+      	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getValues not implemented for \
+      	    Nedelec Tetrahedron. Try getVectorValues. Aborting()!");
       }
       void getLocalGradients(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
       {
-      	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getLocalGradients not implemented for Nedelec Tetrahedron. Aborting()!");
+      	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getLocalGradients not \
+      	    implemented for Nedelec Tetrahedron. Aborting()!");
       }
       int countNodes() const {return countTetNodes(P);}
       void alignSharedNodes(apf::Mesh* m,
@@ -899,7 +904,7 @@ class Nedelec: public FieldShape {
               order[i] = i;
           else
             for(int i = 0; i < P; ++i)
-              order[i] = -(P-1-i)-1; //following MFEM ordering
+              order[i] = -(P-1-i)-1;
           return;
         }
         //must be a triangle
@@ -972,89 +977,18 @@ class Nedelec: public FieldShape {
             }
           }
 
-
-
-
-
-
-          /*========
-          if (rotate == 1  &&  !flip) // CASE 2
-          {
-            for (int r = size-1; r >= 0; r--) {    // horizontal swaps
-              int left = size-r-1; int right = size-1;
-              for(int range = left; range <= left + (right-left)/2; range++) {
-                          std::swap( Nodes(r,range), Nodes(r,left+right-range) );
-              }
-            }
-            mth::Matrix<double> temp(size, size);  // transpose
-            mth::transpose(Nodes,temp);
-            for (int r = 0; r < size; r++)
-              for(int c = 0; c < size; c++)
-                Nodes(r,c) = temp(r,c);
-          }
-          else if (rotate == 2  &&  !flip) // CASE 3
-          {
-            for (int c = size-1; c >= 0; c--) {    // vertical swaps
-              int left = size-c-1; int right = size-1;
-              for(int range = left; range <= left + (right-left)/2; range++) {
-                          std::swap( Nodes(range,c), Nodes(left+right-range,c) );
-              }
-            }
-            mth::Matrix<double> temp(size, size);  // transpose
-            mth::transpose(Nodes,temp);
-            for (int r = 0; r < size; r++)
-              for(int c = 0; c < size; c++)
-                Nodes(r,c) = temp(r,c);
-          }
-          else if (!rotate  &&  flip) // CASE 4
-          {
-            for (int c = size-1; c >= 0; c--) {    // vertical swaps
-              int left = size-c-1; int right = size-1;
-              for(int range = left; range <= left + (right-left)/2; range++) {
-                          std::swap( Nodes(range,c), Nodes(left+right-range,c) );
-              }
-            }
-          }
-          else if (rotate == 1  &&  flip) // CASE 5
-          {
-            for (int r = size-1; r >= 0; r--) {    // horizontal swaps
-              int left = size-r-1; int right = size-1;
-              for(int range = left; range <= left + (right-left)/2; range++) {
-                          std::swap( Nodes(r,range), Nodes(r,left+right-range) );
-              }
-            }
-          }
-          else if (rotate == 2  &&  flip) // CASE 6
-          {
-            mth::Matrix<double> temp(size, size);  // transpose
-            mth::transpose(Nodes,temp);
-            for (int r = 0; r < size; r++)
-              for(int c = 0; c < size; c++)
-                Nodes(r,c) = temp(r,c);
-          }*/
-
           // get the init ordered list with all face nodes
           int init_order[non];
           int i = 0;
           for ( int r = size-1; r >= 0; r--)
             for (int c = size-r-1 ; c < size; c++) {
-              init_order[i++] = Nodes(r,c)*2;    init_order[i++] = Nodes(r,c)*2 + 1;
+              init_order[i++] = Nodes(r,c)*2;
+              init_order[i++] = Nodes(r,c)*2 + 1;
             }
           int* final_order = alignFaceNodes(init_order, non, rotate, flip);
 
           for (int i = 0; i < 4*non; i++)
             order[i] = final_order[i];
-
-          /* TEST
-          cout << "Initial Order " << " r " << rotate << " f " << flip << endl;
-          for (int i = 0; i < non; i++)
-            cout << init_order[i] << " ";
-          cout << endl;
-          cout << "Final Order " << " r " << rotate << " f " << flip << endl;
-          for (int i = 0; i < 4*non; i++)
-            cout << order[i] << " ";
-          cout << endl;
-          cout << endl;*/
         }
       }
       void getVectorValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
@@ -1234,9 +1168,6 @@ class Nedelec: public FieldShape {
     // consider the interior nodes, i.e.,
     // Faces: no need to count the nodes associated with bounding edges
     // Tets: no need to count the nodes associated with bounding edges/faces
-    // TODO: The above description is consistent with how things are done
-    // for other fields in pumi. We need to make sure this will not cause
-    // any problems for Nedelec fields
     bool hasNodesIn(int dimension)
     {
       if (dimension == 1) return true;

From c437bb8625e95c5360456ea6d55517cc33728d79 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Mon, 27 Apr 2020 23:16:32 -0400
Subject: [PATCH 107/555] Includes mth_def.h in apfNedlecShape

The code builds fine without optimization. But with optimization it has
trouble finding the appropriate mth::transpose(). Hence the reason for
including the definitions.
---
 apf/apfNedelec.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index 9f4a09768..782ff3456 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -10,9 +10,10 @@
 #include "apfFieldOf.h"
 #include "apfElement.h"
 #include "apfVectorElement.h"
-#include "mth.h"
-#include <pcu_util.h>
+#include <mth.h>
+#include <mth_def.h>
 #include <mthQR.h>
+#include <pcu_util.h>
 #include <PCU.h>
 
 #include <iostream>

From b81c9e65fd096e64fbf544440a5661f0017d4891 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Mon, 27 Apr 2020 23:18:27 -0400
Subject: [PATCH 108/555] Adds a regression test for nedelec shapes

---
 test/CMakeLists.txt   |   1 +
 test/nedelecShapes.cc | 183 ++++++++++++++++++++++++++++++++++++++++++
 test/testing.cmake    |   5 ++
 3 files changed, 189 insertions(+)
 create mode 100644 test/nedelecShapes.cc

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index a16d5382c..7e2f45ae7 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -175,6 +175,7 @@ test_exe_func(test_scaling test_scaling.cc)
 test_exe_func(mixedNumbering mixedNumbering.cc)
 test_exe_func(test_verify test_verify.cc)
 test_exe_func(hierarchic hierarchic.cc)
+test_exe_func(nedelecShapes nedelecShapes.cc)
 test_exe_func(poisson poisson.cc)
 test_exe_func(ph_adapt ph_adapt.cc)
 test_exe_func(assert_timing assert_timing.cc)
diff --git a/test/nedelecShapes.cc b/test/nedelecShapes.cc
new file mode 100644
index 000000000..2c8caf857
--- /dev/null
+++ b/test/nedelecShapes.cc
@@ -0,0 +1,183 @@
+#include <PCU.h>
+#include <gmi_mesh.h>
+#include <gmi_null.h>
+#include <apf.h>
+#include <apfMDS.h>
+#include <apfMesh2.h>
+#include <apfShape.h>
+#include <apfField.h>
+#include <lionPrint.h>
+#include <pcu_util.h>
+
+#include <cassert>
+#include <cstdlib>
+#include <iostream>
+#include <string>
+#include <math.h>
+
+using namespace std;
+
+// User defined vector functions E(x,y,z) of order up to 6
+void E_exact(const apf::Vector3& x, apf::Vector3& value, int p);
+
+void testNedelec(
+    apf::Mesh2* m,
+    const apf::Vector3& testXi,
+    int ndOrder, int exactOrder);
+
+int main(int argc, char** argv)
+{
+  MPI_Init(&argc,&argv);
+  PCU_Comm_Init();
+
+  lion_set_verbosity(0);
+
+  if (argc != 3) {
+    if(0==PCU_Comm_Self())
+      std::cerr << "usage: " << argv[0]
+        << " <model.dmg or .null> <mesh.smb>\n";
+    return EXIT_FAILURE;
+  }
+
+  gmi_register_mesh();
+  gmi_register_null();
+
+  gmi_model* g = gmi_load(argv[1]);
+  apf::Mesh2* m = apf::loadMdsMesh(g,argv[2]);
+  m->verify();
+
+
+  // constant fields have to be interpolated exactly by any order Nedelec shape
+  for (int order = 1; order <= 6; order++) {
+    testNedelec(
+	m, /* mesh */
+	apf::Vector3(1./4., 1./5., 1./6.), /* test point */
+	order, /* order of Nedelec field */
+	0); /* constant field */
+  }
+
+  // Fields of order p are  interpolated exactly by Nedelec shapes of order p+1
+  for (int i = 1; i <= 6; i++) {
+    testNedelec(
+	m, /* mesh */
+	apf::Vector3(1./4., 1./5., 1./6.), /* test point */
+	i, /* order of Nedelec field */
+	i-1); /* order of test field */
+  }
+
+  apf::destroyMesh(m);
+  PCU_Comm_Free();
+  MPI_Finalize();
+}
+
+void E_exact(const apf::Vector3& x, apf::Vector3& value, int p)
+{
+  // Polynomial coefficients for each component of exact vector field
+  double a[6] = { 1.0, -1.0,  2., -2., -1.0,  1.0};
+  double b[6] = {-2.0,  1.0, -2.,  2., -1.0, -1.0};
+  double c[6] = { 3.0,  0.0, -1.,  0., -1.0,  1.0};
+
+  value[0] = 0.0;
+  value[1] = 0.0;
+  value[2] = 0.0;
+  for (int i = p;  i >= 0; i--) {
+    value[0] += pow(x[0],p)*a[p];
+    value[1] += pow(x[1],p)*b[p];
+    value[2] += pow(x[2],p)*c[p];
+  }
+}
+
+
+
+void testNedelec(
+    apf::Mesh2* m,
+    const apf::Vector3& testXi,
+    int ndOrder, int exactOrder)
+{
+  apf::Field* ndField = apf::createField(
+      m, "nedelec_test", apf::SCALAR, apf::getNedelec(ndOrder));
+
+  // Loop over all nodes and set scalar dofs.
+  int dim = m->getDimension();
+  apf::MeshEntity* ent;
+  apf::MeshIterator* it;
+
+  for (int d = 0; d <= dim; d++) {
+    if (!ndField->getShape()->countNodesOn(apf::Mesh::simplexTypes[d])) {
+      lion_oprint(1, "no nodes in dimension %d\n", d);
+      continue;
+    }
+    else
+      lion_oprint(1, "computing dofs for dimension %d\n", d);
+    it = m->begin(d);
+    int count = 0;
+    while( (ent = m->iterate(it)) ) {
+      int type = m->getType(ent);
+      int non = ndField->getShape()->countNodesOn(type);
+      for (int i = 0; i < non; i++)
+      {
+        apf::Vector3 xi, p, value;
+        ndField->getShape()->getNodeXi(type, i, xi);
+        apf::MeshElement* me = apf::createMeshElement(m, ent);
+        apf::mapLocalToGlobal(me, xi, p);
+        E_exact(p, value, exactOrder);
+
+        apf::Matrix3x3 J;
+        apf::getJacobian( me, xi, J);
+
+        apf::Vector3 t;
+        ndField->getShape()->getNodeTangent(type, i, t);
+
+
+	// dof is t . J^T value
+	// note getJacobian returns the transpose of J, so no need to transpose
+	// J again.
+        apf::Vector3 temp = J * value;
+        double dof = temp * t;
+      	apf::setScalar(ndField, ent, i, dof);
+      }
+      count++;
+    }
+    m->end(it);
+  }
+
+
+  // Verify that interpolated solution field agrees with exact field.
+  double L2ErrorE = 0.;
+  double L2ErrorCurlE = 0.;
+  it = m->begin(3);
+  int count = 0;
+  while( (ent = m->iterate(it)) ) {
+    apf::MeshElement* me = apf::createMeshElement(m, ent);
+    apf::Vector3 x;
+    apf::mapLocalToGlobal(me, testXi, x);
+    apf::Vector3 eFieldExact;
+    E_exact(x, eFieldExact, exactOrder);
+
+    // obtain interpolated value
+    apf::Element* el = apf::createElement(ndField, me);
+    apf::Vector3 eFieldValue;
+    apf::getVector(el, testXi, eFieldValue);
+    // obtain curl field values
+    apf::Vector3 eCurlValue;
+    apf::getCurl(el, testXi, eCurlValue);
+
+    L2ErrorE += ((eFieldValue - eFieldExact) * (eFieldValue - eFieldExact))
+      / (eFieldExact * eFieldExact); // normalization factor
+    L2ErrorCurlE += eCurlValue * eCurlValue;
+    apf::destroyMeshElement(me);
+    apf::destroyElement(el);
+    count++;
+  }
+  m->end(it);
+
+  // check for field interpolation
+  PCU_ALWAYS_ASSERT_VERBOSE(L2ErrorE < 1.e-16,
+      "Fields were not interpolated correctly!");
+  // check for curl (only applies for constant field)
+  if (exactOrder == 0)
+    PCU_ALWAYS_ASSERT_VERBOSE(L2ErrorCurlE < 1.e-16,
+	"Curls are expected to be zero!");
+
+  apf::destroyField(ndField);
+}
diff --git a/test/testing.cmake b/test/testing.cmake
index 56d4f9770..6f7519f94 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -509,6 +509,11 @@ mpi_test(hierarchic_2p_3D 1
   "${MDIR}/cube.dmg"
   "${MDIR}/cube.smb"
   2)
+set(MDIR ${MESHES}/cube/pumi24)
+mpi_test(nedelec 1
+  ./nedelecShapes
+  "${MDIR}/cube.dmg"
+  "${MDIR}/cube.smb")
 set(MDIR ${MESHES}/cube)
 mpi_test(test_verify 4
   ./test_verify

From e04115c16888098b6fc8a03fa4162e930547616b Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Mon, 27 Apr 2020 23:26:02 -0400
Subject: [PATCH 109/555] Adds a todo for getElementNodeXis

There seems to be two use cases for this and they have different
requirements. So this functions needs to be generalized or a different
one is required.
---
 apf/apfShape.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/apf/apfShape.cc b/apf/apfShape.cc
index c49b00e31..e93133d08 100644
--- a/apf/apfShape.cc
+++ b/apf/apfShape.cc
@@ -1085,6 +1085,10 @@ int countElementNodes(FieldShape* s, int type)
   return s->getEntityShape(type)->countNodes();
 }
 
+// TODO this need to be revisited.
+// Currently it gives the nodes as they appear in the type.
+// Sometimes what we need is to give them in the order they
+// appear in the downward adjacent entities.
 void getElementNodeXis(FieldShape* s, int type,
     apf::NewArray<apf::Vector3>& xis)
 {

From 33306d6c0aef65d5a7ee3319b735b30df4fea23f Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 28 Apr 2020 15:17:38 -0400
Subject: [PATCH 110/555] Removes print statement

---
 apf/apfShape.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/apf/apfShape.cc b/apf/apfShape.cc
index e93133d08..d572bc960 100644
--- a/apf/apfShape.cc
+++ b/apf/apfShape.cc
@@ -1149,7 +1149,6 @@ void getElementNodeXis(FieldShape* s, int type,
         parentXi.zero();
         evi = j;
         for (int i = 0; i < apf::Mesh::adjacentCount[bt][0]; ++i) {
-          printf("i,x,j,bt,d    %d,%d,%d,%d,%d\n", i, x, j, bt, d);
           if(bt == apf::Mesh::EDGE && type == apf::Mesh::TRIANGLE)
             evi = apf::tri_edge_verts[j][i];
           else if(bt == apf::Mesh::EDGE && type == apf::Mesh::TET)

From e1a264e29a80d50846cdd7f48e5211e37d7c537d Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 28 Apr 2020 15:50:17 -0400
Subject: [PATCH 111/555] Adds another getElementNodeXis

This one returns the xi in the order the appear in the downward adjacent
entities. This a more useful one for the purposes of field transfer from
solvers and should eliminated the need for calls to aligneShareNodes
(which is a very low level functionality in apf Shapes and its better
that the clients of the code would not use them).
---
 apf/apfShape.cc | 35 +++++++++++++++++++++++++++++++----
 apf/apfShape.h  | 12 +++++++++++-
 2 files changed, 42 insertions(+), 5 deletions(-)

diff --git a/apf/apfShape.cc b/apf/apfShape.cc
index d572bc960..7ebaa52f0 100644
--- a/apf/apfShape.cc
+++ b/apf/apfShape.cc
@@ -1085,10 +1085,6 @@ int countElementNodes(FieldShape* s, int type)
   return s->getEntityShape(type)->countNodes();
 }
 
-// TODO this need to be revisited.
-// Currently it gives the nodes as they appear in the type.
-// Sometimes what we need is to give them in the order they
-// appear in the downward adjacent entities.
 void getElementNodeXis(FieldShape* s, int type,
     apf::NewArray<apf::Vector3>& xis)
 {
@@ -1167,4 +1163,35 @@ void getElementNodeXis(FieldShape* s, int type,
   PCU_ALWAYS_ASSERT(row == s->getEntityShape(type)->countNodes());
 }
 
+void getElementNodeXis(FieldShape* s, Mesh* m, MeshEntity* e,
+    apf::NewArray<apf::Vector3>& xis)
+{
+  int type = m->getType(e);
+  if (!xis.allocated())
+    xis.allocate(countElementNodes(s, type));
+  else
+    xis.resize(countElementNodes(s, type));
+
+  int td = apf::Mesh::typeDimension[type];
+  apf::Vector3 childXi, parentXi;
+
+  int row = 0;
+  for(int d = 0; d <= td; ++d){
+    MeshEntity* down[12];
+    int nDown = m->getDownward(e, d, down);
+    int bt = apf::Mesh::simplexTypes[d];
+    int non = s->countNodesOn(bt);
+    if (!non) continue;
+    for(int j = 0; j < nDown; ++j){
+      for(int x = 0; x < non; ++x){
+        s->getNodeXi(bt, x, childXi);
+        xis[row] = (bt != type) ?
+          boundaryToElementXi(m, down[j], e, childXi) : childXi;
+        ++row;
+      }
+    }
+  }
+  PCU_ALWAYS_ASSERT(row == s->getEntityShape(type)->countNodes());
+}
+
 }//namespace apf
diff --git a/apf/apfShape.h b/apf/apfShape.h
index 1b02b110f..4042e814d 100644
--- a/apf/apfShape.h
+++ b/apf/apfShape.h
@@ -177,11 +177,21 @@ FieldShape* getShapeByName(const char* name);
 int countElementNodes(FieldShape* s, int type);
 
 /** \brief gets the xi coordinates for all the nodes
-  \details order follows downward adjacency
+  \details order follows canonical notation. See tables
+  apf::Mesh::tri_edge_verts, apf::Mesh::tet_edge_verts, and
+  apf::Mesh::tet_tri_verts
   \param type select from apf::Mesh::Type */
 void getElementNodeXis(FieldShape* s, int type,
     apf::NewArray<apf::Vector3>& xis);
 
+/** \brief gets the xi coordinates for all the nodes
+  \details order follows downward adjacency and global
+  directions for the bounding entities. xi coordinates
+  will be with respect to the entity e
+  \param type select from apf::Mesh::Type */
+void getElementNodeXis(FieldShape* s, MeshEntity*e,
+    apf::NewArray<apf::Vector3>& xis);
+
 /** \brief Reparameterize from boundary entity to element
   \details This function converts a point in the local
   parametric space of a boundary mesh entity into the

From 14e062ad5601bcbc1fbabfb68990df7271fcb61d Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 28 Apr 2020 16:14:33 -0400
Subject: [PATCH 112/555] Fixes the issue with loop variable increments

---
 apf/apfNedelec.cc | 36 ++++++++++++------------------------
 1 file changed, 12 insertions(+), 24 deletions(-)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index 782ff3456..3ca0e76e6 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -30,21 +30,19 @@ int* alignFaceNodes(int init_order[], int nodes, int r, bool f)
   {
     int ind = 0;
     int offset = 0;
-    for (int i = 0; i < nodes; i++)
+    for (int i = 0; i < nodes; i+=2)
     {
       final_order[i] = init_order[offset];
       final_order[i+1] = init_order[offset];
       offset += 2;
-      i++;
     }
     ind += nodes;
     offset = 1;
-    for (int i = 0 ; i < nodes; i++)
+    for (int i = 0 ; i < nodes; i+=2)
     {
       final_order[i+ind] = init_order[offset];
       final_order[i+1+ind] = init_order[offset];
       offset += 2;
-      i++;
     }
     ind += nodes;
     for (int i = 0; i < nodes; i++)
@@ -57,21 +55,19 @@ int* alignFaceNodes(int init_order[], int nodes, int r, bool f)
   {
     int ind = 0;
     int offset = 0;
-    for (int i = 0; i < nodes; i++)
+    for (int i = 0; i < nodes; i+=2)
     {
       final_order[i] = -init_order[offset]-1;
       final_order[i+1] = -init_order[offset]-1;
       offset += 2;
-      i++;
     }
     ind += nodes;
     offset = 1;
-    for (int i = 0 ; i < nodes; i++)
+    for (int i = 0 ; i < nodes; i+=2)
     {
       final_order[i+ind] = init_order[offset];
       final_order[i+1+ind] = init_order[offset];
       offset += 2;
-      i++;
     }
     ind += nodes;
     for (int i = 0; i < nodes; i++)
@@ -84,21 +80,19 @@ int* alignFaceNodes(int init_order[], int nodes, int r, bool f)
   {
     int ind = 0;
     int offset = 0;
-    for (int i = 0; i < nodes; i++)
+    for (int i = 0; i < nodes; i+=2)
     {
       final_order[i] = init_order[offset];
       final_order[i+1] = init_order[offset];
       offset += 2;
-      i++;
     }
     ind += nodes;
     offset = 1;
-    for (int i = 0 ; i < nodes; i++)
+    for (int i = 0 ; i < nodes; i+=2)
     {
       final_order[i+ind] = -init_order[offset]-1;
       final_order[i+1+ind] = -init_order[offset]-1;
       offset += 2;
-      i++;
     }
     ind += nodes;
     for (int i = 0; i < nodes; i++)
@@ -111,21 +105,19 @@ int* alignFaceNodes(int init_order[], int nodes, int r, bool f)
   {
     int ind = 0;
     int offset = 0;
-    for (int i = 0; i < nodes; i++)
+    for (int i = 0; i < nodes; i+=2)
     {
       final_order[i] = init_order[offset];
       final_order[i+1] = init_order[offset];
       offset += 2;
-      i++;
     }
     ind += nodes;
     offset = 1;
-    for (int i = 0 ; i < nodes; i++)
+    for (int i = 0 ; i < nodes; i+=2)
     {
       final_order[i+ind] = -init_order[offset]-1;
       final_order[i+1+ind] = -init_order[offset]-1;
       offset += 2;
-      i++;
     }
     ind += nodes;
     for (int i = 0; i < nodes; i++)
@@ -138,21 +130,19 @@ int* alignFaceNodes(int init_order[], int nodes, int r, bool f)
   {
     int ind = 0;
     int offset = 0;
-    for (int i = 0; i < nodes; i++)
+    for (int i = 0; i < nodes; i+=2)
     {
       final_order[i] = -init_order[offset]-1;
       final_order[i+1] = -init_order[offset]-1;
       offset += 2;
-      i++;
     }
     ind += nodes;
     offset = 1;
-    for (int i = 0 ; i < nodes; i++)
+    for (int i = 0 ; i < nodes; i+=2)
     {
       final_order[i+ind] = init_order[offset];
       final_order[i+1+ind] = init_order[offset];
       offset += 2;
-      i++;
     }
     ind += nodes;
     for (int i = 0; i < nodes; i++)
@@ -165,21 +155,19 @@ int* alignFaceNodes(int init_order[], int nodes, int r, bool f)
   {
     int ind = 0;
     int offset = 0;
-    for (int i = 0; i < nodes; i++)
+    for (int i = 0; i < nodes; i+=2)
     {
       final_order[i] = init_order[offset];
       final_order[i+1] = init_order[offset];
       offset += 2;
-      i++;
     }
     ind += nodes;
     offset = 1;
-    for (int i = 0 ; i < nodes; i++)
+    for (int i = 0 ; i < nodes; i+=2)
     {
       final_order[i+ind] = init_order[offset];
       final_order[i+1+ind] = init_order[offset];
       offset += 2;
-      i++;
     }
     ind += nodes;
     for (int i = 0; i < nodes; i++)

From bac58bf42cd160ab4e0137c51b4c6313d7496cda Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 28 Apr 2020 21:48:41 -0400
Subject: [PATCH 113/555] Adds asserts to (un)rotatateTetXi

This assert checks the range of the input "rotation" to make sure it
does not fall out of bound.
---
 ma/maMesh.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ma/maMesh.cc b/ma/maMesh.cc
index 7d1b9246f..348fbe220 100644
--- a/ma/maMesh.cc
+++ b/ma/maMesh.cc
@@ -123,6 +123,7 @@ int findTetRotation(Mesh* m, Entity* tet, Entity** v)
    that match the original set of vertices */
 void unrotateTetXi(Vector& xi, int rotation)
 {
+  PCU_ALWAYS_ASSERT(rotation >= 0 && rotation < 12);
   double a[4];
   a[0] = 1-xi[0]-xi[1]-xi[2]; a[1] = xi[0]; a[2] = xi[1]; a[3] = xi[2];
   int const* originalIndexOf = tet_rotation[rotation];
@@ -139,6 +140,7 @@ void unrotateTetXi(Vector& xi, int rotation)
    set of vertices */
 void rotateTetXi(Vector& xi, int rotation)
 {
+  PCU_ALWAYS_ASSERT(rotation >= 0 && rotation < 12);
   double b[4];
   b[0] = 1-xi[0]-xi[1]-xi[2]; b[1] = xi[0]; b[2] = xi[1]; b[3] = xi[2];
   int const* originalIndexOf = tet_rotation[rotation];

From eb1b2b8794e496b5e6895fd5377c549de0e261a9 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 28 Apr 2020 21:53:06 -0400
Subject: [PATCH 114/555] Adds explanation for MixedVectorElement class

Also removes extra spaces in one of the functions definitions inside
apfHierarchic.cc
---
 apf/apfHierarchic.cc        | 2 +-
 apf/apfMixedVectorElement.h | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/apf/apfHierarchic.cc b/apf/apfHierarchic.cc
index c864a8224..b7ff960f4 100644
--- a/apf/apfHierarchic.cc
+++ b/apf/apfHierarchic.cc
@@ -160,7 +160,7 @@ class HTriangle3 : public EntityShape {
       N[9] = l0*l1*l2;
     }
     void getLocalGradients(
-        Mesh* m, MeshEntity* e, Vector3 const& xi, NewArray<Vector3>& dN) const    {
+        Mesh* m, MeshEntity* e, Vector3 const& xi, NewArray<Vector3>& dN) const {
       dN.allocate(10);
 
       /* edge orientations */
diff --git a/apf/apfMixedVectorElement.h b/apf/apfMixedVectorElement.h
index 35d3f6107..a6fba7f24 100644
--- a/apf/apfMixedVectorElement.h
+++ b/apf/apfMixedVectorElement.h
@@ -15,6 +15,11 @@ namespace apf {
 
 class MixedVectorField;
 
+/* Fields with vector shapes are a bit peculiar, in that
+ * the shapes functions are vectors but the dof holders are
+ * scalars. Hence the need for this Mixed class. An example of
+ * such fields are Nedelec fields.
+ */
 class MixedVectorElement : public ElementOf<Vector3, double>
 {
   public:

From e5130de7bc2de099535da681ee0860e1b7bf88b1 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 28 Apr 2020 21:54:29 -0400
Subject: [PATCH 115/555] Fixes  coding/style issues in apfFieldData

* removed namespace std
* fixed indentation problems
* removed an "if (cond) {}" and replaced it with "if (!cond) continue;" in the for loop
---
 apf/apfFieldData.cc | 77 ++++++++++++++++++++++-----------------------
 1 file changed, 38 insertions(+), 39 deletions(-)

diff --git a/apf/apfFieldData.cc b/apf/apfFieldData.cc
index cba4fa4d9..5d46a5f59 100644
--- a/apf/apfFieldData.cc
+++ b/apf/apfFieldData.cc
@@ -3,8 +3,8 @@
 #include "apfShape.h"
 #include <pcu_util.h>
 #include <cstdlib>
-using namespace std;
 #include <iostream>
+
 namespace apf {
 
 FieldData::~FieldData()
@@ -271,46 +271,45 @@ int FieldDataOf<T>::getElementData(MeshEntity* entity, NewArray<T>& data)
   apf::DynamicArray<T> adata;
   int n = 0;
   for (int d = 0; d <= ed; ++d) {
-    if (fs->hasNodesIn(d)) {
-      Downward a;
-      int na = mesh->getDownward(entity,d,a);
-      for (int i = 0; i < na; ++i) {
-        int nan = fs->countNodesOn(mesh->getType(a[i]));
-        // for vector shapes (i.e., nedelec) direction matters for nan>=1
-        if (fs->isVectorShape()) {
-	  if (nan >= 1 && ed != d) {
-	    // The 1st nen ints is the 1st contribution
-	    // The 2nd nen ints is the 2nd contribution
-	    // The 3rd nen ints tells whether to add 1st contribution
-	    // The 4th nen ints tells whether to add 2st contribution
-	    order.setSize(4*nen);
-	    adata.setSize(nen);
-	    es->alignSharedNodes(mesh, entity, a[i], &order[0]);
-	    get(a[i], &adata[0]);
-	    // We would want to have a different reorder here to handle
-	    // the fact that order now includes some extra info
-	    int dtype = mesh->getType(a[i]);
-	    reorderDataNedelec<T>(&adata[0], &data[n], &order[0], nc, nan, dtype);
-	  }
-	  // this else is required to add the dofs associated with the tet
-	  else
-	    get(a[i], &data[n]);
-	}
-        // for non vector shapes direction matters for nan>1
-	else {
-	  if (nan > 1 && ed != d) { /* multiple shared nodes, check alignment */
-	    order.setSize(nen); /* nen >= nan */
-	    adata.setSize(nen); /* setSize is no-op for the same size */
-	    es->alignSharedNodes(mesh, entity, a[i], &order[0]);
-	    get(a[i], &adata[0]);
-	    reorderData<T>(&adata[0], &data[n], &order[0], nc, nan);
-	  } else if (nan) { /* non-zero set of nodes, either one
-			      or not shared */
-	    get(a[i], &data[n]);
-	  }
+    if (!fs->hasNodesIn(d)) continue;
+    Downward a;
+    int na = mesh->getDownward(entity,d,a);
+    for (int i = 0; i < na; ++i) {
+      int nan = fs->countNodesOn(mesh->getType(a[i]));
+      // for vector shapes (i.e., nedelec) direction matters for nan>=1
+      if (fs->isVectorShape()) {
+        if (nan >= 1 && ed != d) {
+          // The 1st nen ints is the 1st contribution
+          // The 2nd nen ints is the 2nd contribution
+          // The 3rd nen ints tells whether to add 1st contribution
+          // The 4th nen ints tells whether to add 2st contribution
+          order.setSize(4*nen);
+          adata.setSize(nen);
+          es->alignSharedNodes(mesh, entity, a[i], &order[0]);
+          get(a[i], &adata[0]);
+          // We would want to have a different reorder here to handle
+          // the fact that order now includes some extra info
+          int dtype = mesh->getType(a[i]);
+          reorderDataNedelec<T>(&adata[0], &data[n], &order[0], nc, nan, dtype);
+        }
+        // this else is required to add the dofs associated with the tet
+        else
+          get(a[i], &data[n]);
+      }
+      // for non vector shapes direction matters for nan>1
+      else {
+	if (nan > 1 && ed != d) { /* multiple shared nodes, check alignment */
+	  order.setSize(nen); /* nen >= nan */
+	  adata.setSize(nen); /* setSize is no-op for the same size */
+	  es->alignSharedNodes(mesh, entity, a[i], &order[0]);
+	  get(a[i], &adata[0]);
+	  reorderData<T>(&adata[0], &data[n], &order[0], nc, nan);
+	} else if (nan) { /* non-zero set of nodes, either one
+			    or not shared */
+	  get(a[i], &data[n]);
 	}
-        n += nc * nan;
       }
+      n += nc * nan;
     }
   }
   PCU_ALWAYS_ASSERT(n == nc * nen);

From 3771c77bb905a8234aaad1988c8ed5e621fbe85f Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 28 Apr 2020 21:57:01 -0400
Subject: [PATCH 116/555] Adds PCU_ASSERT in empty else blocks

Informing users that 3D surface meshes (manifolds) are not yet handled.
---
 apf/apf.cc | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/apf/apf.cc b/apf/apf.cc
index 271b9436d..5ebfb231a 100644
--- a/apf/apf.cc
+++ b/apf/apf.cc
@@ -483,7 +483,9 @@ void getVectorShapeValues(Element* e, Vector3 const& local,
   }
   else
   {
-    // TODO when reference dimension != mesh space dimension. Psedoinverse needed.
+    // TODO when ref dim != mesh space dim. Pseudo-inverse needed.
+    PCU_ALWAYS_ASSERT_VERBOSE(false,
+    	"not yet implemented for 3D surface meshes (i.e., manifolds)!");
   }
 }
 
@@ -521,7 +523,9 @@ void getCurlShapeValues(Element* e, Vector3 const& local,
   }
   else
   {
-    // TODO for 2d
+    // TODO when ref dim != mesh space dim. Pseudo-inverse needed.
+    PCU_ALWAYS_ASSERT_VERBOSE(false,
+    	"not yet implemented for 3D surface meshes (i.e., manifolds)!");
   }
 
 

From e7eb6bc02b2598b0850b8f1681771dd9db51da5a Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 28 Apr 2020 21:58:58 -0400
Subject: [PATCH 117/555] Fixes memory leaks

In test/nedelecShapes.cc apf::MeshElements created but not destroyed.
In apf/apfNedelec.cc array created by new were not deleted properly. All of those are replaced by apf::NewArray.
---
 apf/apfNedelec.cc     | 83 ++++++++++++++++++++++++++++---------------
 test/nedelecShapes.cc |  3 +-
 2 files changed, 57 insertions(+), 29 deletions(-)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index 3ca0e76e6..574c800d5 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -21,11 +21,11 @@ using namespace std;
 
 namespace apf {
 
-int* alignFaceNodes(int init_order[], int nodes, int r, bool f)
+static void alignFaceNodes(
+    int init_order[],
+    int final_order[],
+    int nodes, int r, bool f)
 {
-  int size = 4*nodes;
-  int* final_order = new int[size];
-
   if (r == 0 && !f)     // CASE 1
   {
     int ind = 0;
@@ -176,10 +176,10 @@ int* alignFaceNodes(int init_order[], int nodes, int r, bool f)
     for (int i = 0; i < nodes; i++)
       final_order[i+ind] = (i%2) ? 0 : 1;
   }
-  return final_order;
 }
 
 static unsigned const MAX_ND_ORDER = 10;
+
 enum {
   GAUSS_LEGENDRE,
   GAUSS_LOBATTO
@@ -244,34 +244,45 @@ void getGaussLobattoPoints(int /*np*/, double* /*pts*/)
 { /* implement Gauss Lobatto points. Later when needed. */
 };
 
-const double* getPoints(int order, const int type)
+static void getPoints(
+    int order,
+    apf::NewArray<double>& points,
+    int type)
 {
   int np = order + 1;
-  double* points = new double[np];
+  points.allocated() ? points.resize(np) : points.allocate(np);
   switch (type)
   {
     case GAUSS_LEGENDRE:
     {
-      getGaussLegendrePoints(np, points);
+      getGaussLegendrePoints(np, &points[0]);
       break;
     }
     case GAUSS_LOBATTO:
     {
-        getGaussLobattoPoints(np, points);
+      getGaussLobattoPoints(np, &points[0]);
       break;
     }
+    default:
+      PCU_ALWAYS_ASSERT_VERBOSE(false,
+      	  "type should be either GAUSS_LEGENDRE or GAUSS_LOBATTO!");
   }
-  return points;
 }
 
-const double* getOpenPoints(int order, const int type = GAUSS_LEGENDRE)
+void getOpenPoints(
+    int order,
+    apf::NewArray<double>& op,
+    int type = GAUSS_LEGENDRE)
 {
-  return getPoints(order, type);
+  getPoints(order, op, type);
 }
 
-const double* getClosedPoints(int order, const int type = GAUSS_LOBATTO)
+void getClosedPoints(
+    int order,
+    apf::NewArray<double>& cp,
+    int type = GAUSS_LOBATTO)
 {
-    return getPoints(order, type);
+    getPoints(order, cp, type);
 }
 
 void getChebyshevT(int order, double xi, double* u)
@@ -375,8 +386,12 @@ static void computeTriangleTi(
   const double c = 1./3.;
   int non = countTriNodes(P);
 
-  const double *eop = getOpenPoints(P - 1);
-  const double *iop = (P > 1) ? getOpenPoints(P - 2) : NULL;
+  apf::NewArray<double> eop;
+  apf::NewArray<double> iop;
+  getOpenPoints(P - 1, eop);
+  if (P > 1)
+    getOpenPoints(P - 2, iop);
+
 
   const int p = P, pm1 = P - 1, pm2 = P - 2;
   apf::NewArray<double> shape_x(p);
@@ -462,9 +477,16 @@ static void computeTetTi(
      // this last one is the negative of the third one
      // and it is used for face and tet tangents only
      0.,   1.,  0.};
-  const double *eop = getOpenPoints(P - 1);
-  const double *fop = (P > 1) ? getOpenPoints(P - 2) : NULL;
-  const double *iop = (P > 2) ? getOpenPoints(P - 3) : NULL;
+
+
+  apf::NewArray<double> eop;
+  apf::NewArray<double> fop;
+  apf::NewArray<double> iop;
+  getOpenPoints(P - 1, eop);
+  if (P > 1)
+    getOpenPoints(P - 2, fop);
+  if (P > 2)
+    getOpenPoints(P - 3, iop);
 
   const int p = P, pm1 = P - 1, pm2 = P - 2, pm3 = P - 3;
 
@@ -968,13 +990,14 @@ class Nedelec: public FieldShape {
 
           // get the init ordered list with all face nodes
           int init_order[non];
+          int final_order[4*non];
           int i = 0;
           for ( int r = size-1; r >= 0; r--)
             for (int c = size-r-1 ; c < size; c++) {
               init_order[i++] = Nodes(r,c)*2;
               init_order[i++] = Nodes(r,c)*2 + 1;
             }
-          int* final_order = alignFaceNodes(init_order, non, rotate, flip);
+          alignFaceNodes(init_order, final_order, non, rotate, flip);
 
           for (int i = 0; i < 4*non; i++)
             order[i] = final_order[i];
@@ -1176,22 +1199,25 @@ class Nedelec: public FieldShape {
     int getOrder() {return P;}
     void getNodeXi(int type, int node, Vector3& xi)
     {
+      apf::NewArray<double> op;
       if(type == Mesh::EDGE)
       {
-        const double *eop = (P > 0) ? getOpenPoints(P-1) : NULL;
-        xi = Vector3( -1 + 2*eop[node], 0., 0. ); // map from [0,1] to [-1,1]
+      	if (P > 0)
+      	  getOpenPoints(P-1, op);
+        xi = Vector3( -1 + 2*op[node], 0., 0. ); // map from [0,1] to [-1,1]
         return;
       }
       else if (type == Mesh::TRIANGLE)
       {
-        const double *iop = (P > 1) ? getOpenPoints(P - 2) : NULL;
+      	if (P > 1)
+      	  getOpenPoints(P-2, op);
         int pm2 = P - 2; int c = 0;
 
         for (int j = 0; j <= pm2; j++) {
           for (int i = 0; i + j <= pm2; i++) {
             if (node/2 == c) {  // since 2 dofs per node on the face
-              double w = iop[i] + iop[j] + iop[pm2-i-j];
-              xi = Vector3( iop[i]/w, iop[j]/w, 0. );
+              double w = op[i] + op[j] + op[pm2-i-j];
+              xi = Vector3( op[i]/w, op[j]/w, 0. );
               return;
             }
             else
@@ -1201,15 +1227,16 @@ class Nedelec: public FieldShape {
       }
       else if (type == Mesh::TET)
       {
-        const double *iop = (P > 2) ? getOpenPoints(P - 3) : NULL;
+      	if (P>2)
+      	  getOpenPoints(P-3, op);
         int pm3 = P - 3; int c = 0;
 
         for (int k = 0; k <= pm3; k++) {
           for (int j = 0; j + k <= pm3; j++) {
             for (int i = 0; i + j + k <= pm3; i++) {
               if( node/3 == c) {  // since 3 dofs per node on interior tet
-                double w = iop[i] + iop[j] + iop[k] + iop[pm3-i-j-k];
-                xi = Vector3( iop[i]/w, iop[j]/w,  iop[k]/w );
+                double w = op[i] + op[j] + op[k] + op[pm3-i-j-k];
+                xi = Vector3( op[i]/w, op[j]/w,  op[k]/w );
                 return;
               }
               else
diff --git a/test/nedelecShapes.cc b/test/nedelecShapes.cc
index 2c8caf857..cb404f2c1 100644
--- a/test/nedelecShapes.cc
+++ b/test/nedelecShapes.cc
@@ -114,11 +114,11 @@ void testNedelec(
     while( (ent = m->iterate(it)) ) {
       int type = m->getType(ent);
       int non = ndField->getShape()->countNodesOn(type);
+      apf::MeshElement* me = apf::createMeshElement(m, ent);
       for (int i = 0; i < non; i++)
       {
         apf::Vector3 xi, p, value;
         ndField->getShape()->getNodeXi(type, i, xi);
-        apf::MeshElement* me = apf::createMeshElement(m, ent);
         apf::mapLocalToGlobal(me, xi, p);
         E_exact(p, value, exactOrder);
 
@@ -136,6 +136,7 @@ void testNedelec(
         double dof = temp * t;
       	apf::setScalar(ndField, ent, i, dof);
       }
+      apf::destroyMeshElement(me);
       count++;
     }
     m->end(it);

From 409b85ee268a3cb83322626743ed99544643c4d3 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 28 Apr 2020 22:15:41 -0400
Subject: [PATCH 118/555] Removess unneeded `using namespace std` in apf.cc

---
 apf/apf.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/apf/apf.cc b/apf/apf.cc
index 5ebfb231a..4d7de9390 100644
--- a/apf/apf.cc
+++ b/apf/apf.cc
@@ -30,7 +30,6 @@
 #include "mth.h"
 #include "mth_def.h"
 
-using namespace std;
 namespace apf {
 
 void destroyMesh(Mesh* m)

From 0c27efe09670884300627eedb777df3d5c982ae0 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 29 Apr 2020 08:13:58 -0400
Subject: [PATCH 119/555] update pumi-meshes

should fix travis CI errors
---
 pumi-meshes | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pumi-meshes b/pumi-meshes
index 6d1601a29..e3980df41 160000
--- a/pumi-meshes
+++ b/pumi-meshes
@@ -1 +1 @@
-Subproject commit 6d1601a29152e2043b16b2522c1a346b877920a0
+Subproject commit e3980df41b1c9446c60190a1e3870f8fddb0e589

From aa27011e68709d16b37295d2411009e6b3311738 Mon Sep 17 00:00:00 2001
From: Cameron Smith <cwsmith@users.noreply.github.com>
Date: Wed, 29 Apr 2020 10:23:39 -0400
Subject: [PATCH 120/555] ctest: add --output-on-failure flag

---
 .travis.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 2075c0f1b..3a874a611 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -43,8 +43,8 @@ script:
   - cd build
   - cmake .. -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_INSTALL_PREFIX=${DEVROOT}/install/core -DIS_TESTING=ON -DBUILD_EXES=ON -DCMAKE_BUILD_TYPE=Release -DMESHES=${DEVROOT}/SCOREC/core/pumi-meshes
   - make -j ${NP}
-  - ctest
+  - ctest --output-on-failure
   - cmake .. -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_INSTALL_PREFIX=${DEVROOT}/install/core -DIS_TESTING=ON -DBUILD_EXES=ON -DCMAKE_BUILD_TYPE=Debug -DMESHES=${DEVROOT}/SCOREC/core/pumi-meshes
   - make -j ${NP}
-  - ctest
+  - ctest --output-on-failure
 

From 36a84243701068988a071b0a9c56462e17b5aa94 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 29 Apr 2020 10:48:42 -0400
Subject: [PATCH 121/555] print_pumipic_partion requires zoltan

---
 test/testing.cmake | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/testing.cmake b/test/testing.cmake
index 6f7519f94..210f3f155 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -28,6 +28,7 @@ mpi_test(bezierSubdivision 1 ./bezierSubdivision)
 mpi_test(bezierValidity 1 ./bezierValidity)
 mpi_test(ma_analytic 1 ./ma_test_analytic_model)
 
+if(ENABLE_ZOLTAN)
 mpi_test(print_pumipic_partion 1
          ./print_pumipic_partition
          ${MESHES}/cube/cube.dmg
@@ -35,6 +36,7 @@ mpi_test(print_pumipic_partion 1
          4
          pumipic_cube
          )
+endif()
 
 mpi_test(align 1 ./align)
 mpi_test(eigen_test 1 ./eigen_test)

From 33f75fae8b0e5d270a7038f81fb0ac1c28b6ea7b Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Wed, 29 Apr 2020 23:50:10 -0400
Subject: [PATCH 122/555] Adds missing argument in for getElementNodeXis

---
 apf/apfShape.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apf/apfShape.h b/apf/apfShape.h
index 4042e814d..37e259408 100644
--- a/apf/apfShape.h
+++ b/apf/apfShape.h
@@ -189,7 +189,7 @@ void getElementNodeXis(FieldShape* s, int type,
   directions for the bounding entities. xi coordinates
   will be with respect to the entity e
   \param type select from apf::Mesh::Type */
-void getElementNodeXis(FieldShape* s, MeshEntity*e,
+void getElementNodeXis(FieldShape* s, Mesh* m, MeshEntity* e,
     apf::NewArray<apf::Vector3>& xis);
 
 /** \brief Reparameterize from boundary entity to element

From 663ee7303eef612a91316a64d4df0e878c0c6e8d Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@portal2.scorec.rpi.edu>
Date: Fri, 1 May 2020 01:36:55 -0400
Subject: [PATCH 123/555] Initial commit for Error estimator.

Builds edge based patches and assembles LHS topology matrices on
both the interior and the boundary edge patches.
---
 em/CMakeLists.txt           | 28 ++++++++++++++++++++++++++++
 em/cmake/Dependencies.cmake |  3 +++
 em/em.h                     | 29 +++++++++++++++++++++++++++++
 em/emResidualFunctionals.cc | 31 +++++++++++++++++++++++++++++++
 em/pkg_tribits.cmake        | 19 +++++++++++++++++++
 5 files changed, 110 insertions(+)
 create mode 100644 em/CMakeLists.txt
 create mode 100644 em/cmake/Dependencies.cmake
 create mode 100644 em/em.h
 create mode 100644 em/emResidualFunctionals.cc
 create mode 100644 em/pkg_tribits.cmake

diff --git a/em/CMakeLists.txt b/em/CMakeLists.txt
new file mode 100644
index 000000000..4790e15e3
--- /dev/null
+++ b/em/CMakeLists.txt
@@ -0,0 +1,28 @@
+if(DEFINED TRIBITS_PACKAGE)
+  include(pkg_tribits.cmake)
+  return()
+endif()
+
+# Package sources
+set(SOURCES
+  emResidualFunctionals.cc)
+
+# Package headers
+set(HEADERS
+  em.h)
+
+# Add the em library
+add_library(em ${SOURCES})
+
+# Include directories
+target_include_directories(em INTERFACE
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+    $<INSTALL_INTERFACE:include>
+    )
+
+# Link this package to these libraries
+target_link_libraries(em PUBLIC apf pcu)
+
+scorec_export_library(em)
+
+bob_end_subdir()
diff --git a/em/cmake/Dependencies.cmake b/em/cmake/Dependencies.cmake
new file mode 100644
index 000000000..9d9456a31
--- /dev/null
+++ b/em/cmake/Dependencies.cmake
@@ -0,0 +1,3 @@
+TRIBITS_PACKAGE_DEFINE_DEPENDENCIES(
+  LIB_REQUIRED_PACKAGES SCORECapf
+  )
diff --git a/em/em.h b/em/em.h
new file mode 100644
index 000000000..11cfcc8c9
--- /dev/null
+++ b/em/em.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright (C) 2011 Scientific Computation Research Center
+ *
+ * This work is open source software, licensed under the terms of the
+ * BSD license as described in the LICENSE file in the top-level directory.
+ */
+
+#ifndef EM_H
+#define EM_H
+
+
+/** \file em.h
+ *  \brief The Elegtromagnetics Equilibrated Residual error estimator inteface
+ */
+#include "apf.h"
+
+
+
+
+
+
+
+
+
+
+
+
+
+#endif
diff --git a/em/emResidualFunctionals.cc b/em/emResidualFunctionals.cc
new file mode 100644
index 000000000..46d079000
--- /dev/null
+++ b/em/emResidualFunctionals.cc
@@ -0,0 +1,31 @@
+/*
+ * Copyright (C) 2011 Scientific Computation Research Center
+ *
+ * This work is open source software, licensed under the terms of the
+ * BSD license as described in the LICENSE file in the top-level directory.
+ */
+
+#include <PCU.h>
+#include <lionPrint.h>
+
+#include "spr.h"
+
+#include <apfMesh.h>
+#include <apfShape.h>
+#include <apfCavityOp.h>
+
+#include <limits>
+
+namespace em {
+
+
+
+
+
+
+
+
+
+
+
+}
diff --git a/em/pkg_tribits.cmake b/em/pkg_tribits.cmake
new file mode 100644
index 000000000..fd64fa9e4
--- /dev/null
+++ b/em/pkg_tribits.cmake
@@ -0,0 +1,19 @@
+tribits_package(SCORECem)
+
+# THIS IS WHERE TRIBITS GETS HEADERS
+include_directories(${CMAKE_CURRENT_SOURCE_DIR})
+
+#Sources & Headers
+set(SOURCES
+  emResidualFunctionals.cc)
+
+set(HEADERS
+  em.h)
+
+#Library
+tribits_add_library(
+  em
+  HEADERS ${HEADERS}
+  SOURCES ${SOURCES})
+
+tribits_package_postprocess()

From ba84108043941ccca7fb690f1abba10aa9a1017e Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@portal2.scorec.rpi.edu>
Date: Fri, 1 May 2020 01:43:52 -0400
Subject: [PATCH 124/555] Initial commit for Error estimator.

Builds edge based patches and assembles LHS topology matrices on
both the interior and the boundary edge patches.
---
 em/emResidualFunctionals.cc | 173 +++++++++++++++++++++++++++++++++++-
 1 file changed, 172 insertions(+), 1 deletion(-)

diff --git a/em/emResidualFunctionals.cc b/em/emResidualFunctionals.cc
index 46d079000..1b707edd7 100644
--- a/em/emResidualFunctionals.cc
+++ b/em/emResidualFunctionals.cc
@@ -4,23 +4,194 @@
  * This work is open source software, licensed under the terms of the
  * BSD license as described in the LICENSE file in the top-level directory.
  */
+#include <iostream>
 
 #include <PCU.h>
 #include <lionPrint.h>
 
-#include "spr.h"
+#include "em.h"
 
 #include <apfMesh.h>
 #include <apfShape.h>
+#include <apfField.h>
 #include <apfCavityOp.h>
+#include "crv.h"
+#include "crvShape.h"
+
+#include <mthQR.h>
+#include <mth.h>
+#include <mth_def.h>
 
 #include <limits>
 
+#include <set>
+#include <pcu_util.h>
+
 namespace em {
 
+/* overall information useful during equilibration */
+struct Equilibration {
+  apf::Mesh* mesh;
+  /* mesh dimension, so far handling 3 only */
+  int dim;
+  /* polynomial order of Nedelec space */
+  int order;
+  /* input scalar field containing Nedelec dofs for electric field */
+  apf::Field* ef;
+};
+
+static void setupEquilibration(Equilibration* eq, apf::Field* f)
+{
+  eq->mesh = apf::getMesh(f);
+  eq->dim = eq->mesh->getDimension();
+  eq->ef = f;
+  eq->order = f->getShape()->getOrder();
+}
+
+struct QRDecomp {
+  mth::Matrix<double> Q;
+  mth::Matrix<double> R;
+};
+
+typedef std::set<apf::MeshEntity*> EntitySet;
+
+struct EdgePatch {
+  apf::Mesh* mesh;
+  Equilibration* equilibration;
+  /* the entity around which the patch
+     is centered. a patch collects entities
+     (faces & tets) around this edge entity */
+  apf::MeshEntity* entity;
+  EntitySet tets;
+  EntitySet faces;
+  mth::Matrix<double> A;
+  QRDecomp qr;
+};
+
+static void setupEdgePatch(EdgePatch* p, Equilibration* eq)
+{
+  p->mesh = eq->mesh;
+  p->equilibration = eq;
+  p->entity = 0;
+}
+
+static void startEdgePatch(EdgePatch* p, apf::MeshEntity* e)
+{
+  p->tets.clear();
+  p->faces.clear();
+  p->entity = e;
+}
 
+static void addEntityToPatch(EdgePatch* p, apf::MeshEntity* e)
+{
+  if(p->mesh->getType(e) == apf::Mesh::TRIANGLE)
+    p->faces.insert(e);
+  if(p->mesh->getType(e) == apf::Mesh::TET)
+    p->tets.insert(e);
+}
 
+static void addEntitiesToPatch(EdgePatch* p, apf::DynamicArray<apf::MeshEntity*>& es)
+{
+  for (std::size_t i=0; i < es.getSize(); ++i)
+    addEntityToPatch(p, es[i]);
+}
 
+static bool getInitialEdgePatch(EdgePatch* p, apf::CavityOp* o)
+{
+  if ( ! o->requestLocality(&p->entity,1))
+    return false;
+  apf::DynamicArray<apf::MeshEntity*> adjacent;
+  p->mesh->getAdjacent(p->entity, 3, adjacent);
+  addEntitiesToPatch(p, adjacent);
+  p->mesh->getAdjacent(p->entity, 2, adjacent);
+  addEntitiesToPatch(p, adjacent);
+  return true;
+}
+
+static bool buildEdgePatch(EdgePatch* p, apf::CavityOp* o)
+{
+  if (!getInitialEdgePatch(p, o)) return false;
+  return true;
+}
+
+static void assembleLHS(EdgePatch* p)
+{
+  int ne = p->tets.size();
+  int nf = p->faces.size();
+  printf("ne %d nf %d \n", ne, nf); // TODO remove
+  if( crv::isBoundaryEntity(p->mesh, p->entity) ) {
+    p->A.resize(ne+nf, ne+nf);
+    p->A.zero();
+    for (int i = 0; i < nf; i++)
+      p->A(i,i) = 2.;
+    for (int i = 0; i < ne-1; i++) {
+      p->A(i+nf,i) = 1.; p->A(i+nf,i+1) = -1.;
+      p->A(i,i+nf) = 1.; p->A(i+1,i+nf) = -1.;
+    }
+    p->A(ne+nf-1, ne-1) = 1.; p->A(ne+nf-1, ne) = 1.;
+    p->A(ne-1, ne+nf-1) = 1.; p->A(ne, ne+nf-1) = 1.;
+
+    std::cout << "boundary" << std::endl; // TODO remove
+    std::cout << p->A << std::endl; // TODO remove
+  }
+  else if( ! crv::isBoundaryEntity(p->mesh, p->entity) ) {
+    mth::Matrix<double> m(ne, nf);
+    m.zero();
+    for (int i = 0; i < ne-1; i++) {
+      m(i,i) = 1.;  m(i,i+1) = -1.;
+    }
+    m(ne-1,0) = -1.; m(ne-1,ne-1) = 1.;
+
+    // m is singular so do (m*mt) + 1.0
+    // to pick a particular solution
+    mth::Matrix<double> mt(nf,ne);
+    mth::transpose(m, mt);
+    mth::multiply(m, mt, p->A);
+
+    for (int i = 0; i < ne; i++)
+      for (int j = 0; j < ne; j++)
+        p->A(i,j) += 1.;
+    std::cout << "interior" << std::endl; // TODO remove
+    std::cout << p->A << std::endl; // TODO remove
+  }
+}
+
+static void runErm(EdgePatch* p)
+{
+  assembleLHS(p);
+}
+
+
+class EdgePatchOp : public apf::CavityOp
+{
+public:
+  EdgePatchOp(Equilibration* eq):
+    apf::CavityOp(eq->mesh)
+  {
+    setupEdgePatch(&edgePatch, eq);
+  }
+  virtual Outcome setEntity(apf::MeshEntity* e)
+  {
+    startEdgePatch(&edgePatch, e);
+    if ( ! buildEdgePatch(&edgePatch, this))
+      return REQUEST;
+    return OK;
+    return SKIP;
+  }
+  virtual void apply()
+  {
+    runErm(&edgePatch);
+  }
+  EdgePatch edgePatch;
+};
+
+void equilibrateResiduals(apf::Field* f)
+{
+  Equilibration equilibration;
+  setupEquilibration(&equilibration, f);
+  EdgePatchOp op(&equilibration);
+  op.applyToDimension(1); // edges
+}
 
 
 

From 21865e4af688cc7197d95c7f66c82f5594b358fb Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@portal2.scorec.rpi.edu>
Date: Sat, 2 May 2020 13:44:59 -0400
Subject: [PATCH 125/555] Adds subdirectory em (electromagnetics) in CMake

---
 CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 207ca93f2..794c9bc93 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -133,6 +133,7 @@ add_subdirectory(pumi)
 add_subdirectory(ma)
 add_subdirectory(crv)
 add_subdirectory(spr)
+add_subdirectory(em)
 add_subdirectory(sam)
 add_subdirectory(phasta)
 add_subdirectory(stk)

From 5783d8fb1dc96d5227a88ab9fc4f67bcfc6cb0ad Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@portal2.scorec.rpi.edu>
Date: Sat, 2 May 2020 13:46:33 -0400
Subject: [PATCH 126/555] adds em/CMakeLists.txt

---
 em/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/em/CMakeLists.txt b/em/CMakeLists.txt
index 4790e15e3..c9886e8fd 100644
--- a/em/CMakeLists.txt
+++ b/em/CMakeLists.txt
@@ -21,7 +21,7 @@ target_include_directories(em INTERFACE
     )
 
 # Link this package to these libraries
-target_link_libraries(em PUBLIC apf pcu)
+target_link_libraries(em PUBLIC apf pcu crv)
 
 scorec_export_library(em)
 

From 791d10398e5c2501f402887e61d9393b22667305 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@portal2.scorec.rpi.edu>
Date: Sat, 2 May 2020 14:56:29 -0400
Subject: [PATCH 127/555] Adds getOrderedTetsandFaces.

Required to arrange tets and faces in a cw or ccw order in an edge patch.
---
 em/emResidualFunctionals.cc | 84 +++++++++++++++++++++++++++++++++++++
 1 file changed, 84 insertions(+)

diff --git a/em/emResidualFunctionals.cc b/em/emResidualFunctionals.cc
index 1b707edd7..54e78c6e3 100644
--- a/em/emResidualFunctionals.cc
+++ b/em/emResidualFunctionals.cc
@@ -156,9 +156,93 @@ static void assembleLHS(EdgePatch* p)
   }
 }
 
+// The following two functions help order tets and faces in a cavity in a
+// clockwise (or ccw) direction.
+static apf::MeshEntity* getTetOppFaceSharingEdge(
+    apf::Mesh* m, apf::MeshEntity* t, apf::MeshEntity* f, apf::MeshEntity* e)
+{
+  apf::MeshEntity* fs[4];
+  m->getDownward(t, 2, fs);
+  for (int i = 0; i < 4; i++) {
+    if (fs[i] == f) continue;
+    apf::MeshEntity* es[3];
+    m->getDownward(fs[i], 1, es);
+    if (apf::findIn(es, 3, e) > -1)
+      return fs[i];
+  }
+  return 0;
+}
+static void getOrderedTetsandFaces(apf::Mesh* mesh, apf::MeshEntity* edge,
+     std::vector<apf::MeshEntity*>& tets, std::vector<apf::MeshEntity*>& faces)
+{
+  tets.clear();
+  faces.clear();
+  if( ! crv::isBoundaryEntity(mesh, edge) ) { // interior edge patches
+    apf::MeshEntity* currentFace = mesh->getUpward(edge, 0);
+    apf::Up up;
+    mesh->getUp(currentFace, up);
+    PCU_ALWAYS_ASSERT(up.n == 2);
+    apf::MeshEntity* firstTet = up.e[0];
+    apf::MeshEntity* nextTet  = up.e[1];
+    tets.push_back(firstTet);
+    apf::MeshEntity* firstFace = getTetOppFaceSharingEdge(mesh, firstTet,
+        currentFace, edge);
+    faces.push_back(firstFace);
+
+    while (nextTet != firstTet) {
+      tets.push_back(nextTet);
+      faces.push_back(currentFace);
+      currentFace = getTetOppFaceSharingEdge(mesh, nextTet, currentFace, edge);
+      PCU_ALWAYS_ASSERT(currentFace);
+      apf::Up up;
+      mesh->getUp(currentFace, up);
+      PCU_ALWAYS_ASSERT(up.n == 2);
+      if (nextTet != up.e[0])
+        nextTet = up.e[0];
+      else
+        nextTet = up.e[1];
+    }
+  }
+  else { // boundary edge patches
+    apf::Up up;
+    mesh->getUp(edge, up);
+    apf::MeshEntity* firstFace;
+    for (int i = 0; i < up.n; i++) {
+      if ( crv::isBoundaryEntity(mesh, up.e[i]) ) {
+        firstFace = up.e[i]; break;
+      }
+    }
+    faces.push_back(firstFace);
+    mesh->getUp(firstFace, up);
+    PCU_ALWAYS_ASSERT(up.n == 1);
+    apf::MeshEntity* firstTet = up.e[0];
+    tets.push_back(firstTet);
+
+    apf::MeshEntity* nextFace = getTetOppFaceSharingEdge(mesh, firstTet,
+        firstFace, edge);
+    apf::MeshEntity* nextTet = firstTet;
+    mesh->getUp(nextFace, up);
+    while( up.n == 2) {
+      faces.push_back(nextFace);
+      if (nextTet != up.e[0])
+        nextTet = up.e[0];
+      else
+        nextTet = up.e[1];
+      tets.push_back(nextTet);
+
+      nextFace = getTetOppFaceSharingEdge(mesh, nextTet, nextFace, edge);
+      mesh->getUp(nextFace, up);
+    }
+    faces.push_back(nextFace);
+  }
+}
+
 static void runErm(EdgePatch* p)
 {
   assembleLHS(p);
+  std::vector<apf::MeshEntity*> otets; // ordered tets
+  std::vector<apf::MeshEntity*> ofaces; // ordered faces
+  getOrderedTetsandFaces(p->mesh, p->entity, otets, ofaces);
 }
 
 

From 68455cbad5172257e6d7175f0fa4c86436756592 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@portal2.scorec.rpi.edu>
Date: Sat, 2 May 2020 16:24:09 -0400
Subject: [PATCH 128/555] updates em.h

---
 em/em.h | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/em/em.h b/em/em.h
index 11cfcc8c9..c8159fd8c 100644
--- a/em/em.h
+++ b/em/em.h
@@ -14,7 +14,17 @@
  */
 #include "apf.h"
 
+namespace em {
 
+void equilibrateResiduals(apf::Field* f);
+
+
+
+
+
+
+
+}
 
 
 

From 4238fc821e5f57d86637457e99b647a75fd80ca9 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@portal2.scorec.rpi.edu>
Date: Sat, 2 May 2020 16:27:03 -0400
Subject: [PATCH 129/555] implementation of assembleRHS in progress.

---
 em/emResidualFunctionals.cc | 76 ++++++++++++++++++++++++++++++-------
 1 file changed, 63 insertions(+), 13 deletions(-)

diff --git a/em/emResidualFunctionals.cc b/em/emResidualFunctionals.cc
index 54e78c6e3..513f2fb2a 100644
--- a/em/emResidualFunctionals.cc
+++ b/em/emResidualFunctionals.cc
@@ -62,10 +62,13 @@ struct EdgePatch {
      is centered. a patch collects entities
      (faces & tets) around this edge entity */
   apf::MeshEntity* entity;
+  bool isOnBdry;
   EntitySet tets;
   EntitySet faces;
   mth::Matrix<double> A;
+  mth::Vector<double> b;
   QRDecomp qr;
+
 };
 
 static void setupEdgePatch(EdgePatch* p, Equilibration* eq)
@@ -80,6 +83,7 @@ static void startEdgePatch(EdgePatch* p, apf::MeshEntity* e)
   p->tets.clear();
   p->faces.clear();
   p->entity = e;
+  p->isOnBdry = crv::isBoundaryEntity(p->mesh, p->entity);
 }
 
 static void addEntityToPatch(EdgePatch* p, apf::MeshEntity* e)
@@ -156,6 +160,48 @@ static void assembleLHS(EdgePatch* p)
   }
 }
 
+// computes local bilinear form integral restricted to
+// an edge of an element
+static int getLocalBLFIntegral(EdgePatch* p, apf::MeshEntity* tet)
+{
+  // find position of edge in downward edges of element
+  apf::Downward e;
+  int ne = p->mesh->getDownward(tet, 1, e);
+  int ei = apf::findIn(e, ne, p->entity);
+  // get Element Dofs
+  //
+  // assemble curl curl element matrix
+  // assemble vector mass element matrix
+  // add element matrices
+  // multiply element matrix with element dofs
+  // pick edge index from the resulting vector
+}
+
+
+static void assembleRHS(EdgePatch* p)
+{
+  if (p->isOnBdry) {
+    p->b.resize(p->tets.size() + p->faces.size());
+    p->b.zero();
+  }
+  else {
+    p->b.resize(p->tets.size());
+    p->b.zero();
+  }
+
+  apf::MeshEntity* edge = p->entity[i];
+  int ne = p->tets.size();
+  for (int i = 0; i < ne; i++) {
+    apf::MeshEntity* tet = p->tets[i];
+    int blfIntegral = getLocalBLFIntegral(p, tet);
+  }
+  // TODO computeBilinearFormIntegral
+  // TODO computeLinearFormIntegral
+  // TODO computeFluxTermIntegral
+
+
+}
+
 // The following two functions help order tets and faces in a cavity in a
 // clockwise (or ccw) direction.
 static apf::MeshEntity* getTetOppFaceSharingEdge(
@@ -173,7 +219,7 @@ static apf::MeshEntity* getTetOppFaceSharingEdge(
   return 0;
 }
 static void getOrderedTetsandFaces(apf::Mesh* mesh, apf::MeshEntity* edge,
-     std::vector<apf::MeshEntity*>& tets, std::vector<apf::MeshEntity*>& faces)
+     EntitySet& tets, EntitySet& faces)
 {
   tets.clear();
   faces.clear();
@@ -184,14 +230,14 @@ static void getOrderedTetsandFaces(apf::Mesh* mesh, apf::MeshEntity* edge,
     PCU_ALWAYS_ASSERT(up.n == 2);
     apf::MeshEntity* firstTet = up.e[0];
     apf::MeshEntity* nextTet  = up.e[1];
-    tets.push_back(firstTet);
+    tets.insert(firstTet);
     apf::MeshEntity* firstFace = getTetOppFaceSharingEdge(mesh, firstTet,
         currentFace, edge);
-    faces.push_back(firstFace);
+    faces.insert(firstFace);
 
     while (nextTet != firstTet) {
-      tets.push_back(nextTet);
-      faces.push_back(currentFace);
+      tets.insert(nextTet);
+      faces.insert(currentFace);
       currentFace = getTetOppFaceSharingEdge(mesh, nextTet, currentFace, edge);
       PCU_ALWAYS_ASSERT(currentFace);
       apf::Up up;
@@ -212,37 +258,41 @@ static void getOrderedTetsandFaces(apf::Mesh* mesh, apf::MeshEntity* edge,
         firstFace = up.e[i]; break;
       }
     }
-    faces.push_back(firstFace);
+    faces.insert(firstFace);
     mesh->getUp(firstFace, up);
     PCU_ALWAYS_ASSERT(up.n == 1);
     apf::MeshEntity* firstTet = up.e[0];
-    tets.push_back(firstTet);
+    tets.insert(firstTet);
 
     apf::MeshEntity* nextFace = getTetOppFaceSharingEdge(mesh, firstTet,
         firstFace, edge);
     apf::MeshEntity* nextTet = firstTet;
     mesh->getUp(nextFace, up);
     while( up.n == 2) {
-      faces.push_back(nextFace);
+      faces.insert(nextFace);
       if (nextTet != up.e[0])
         nextTet = up.e[0];
       else
         nextTet = up.e[1];
-      tets.push_back(nextTet);
+      tets.insert(nextTet);
 
       nextFace = getTetOppFaceSharingEdge(mesh, nextTet, nextFace, edge);
       mesh->getUp(nextFace, up);
     }
-    faces.push_back(nextFace);
+    faces.insert(nextFace);
   }
 }
 
 static void runErm(EdgePatch* p)
 {
   assembleLHS(p);
-  std::vector<apf::MeshEntity*> otets; // ordered tets
-  std::vector<apf::MeshEntity*> ofaces; // ordered faces
-  getOrderedTetsandFaces(p->mesh, p->entity, otets, ofaces);
+  // TODO decompose A into Q and R
+  //std::vector<apf::MeshEntity*> otets; // ordered tets // TODO remove these two
+  //std::vector<apf::MeshEntity*> ofaces; // ordered faces
+  getOrderedTetsandFaces(p->mesh, p->entity, p->tets, p->faces); // TODO maybe use p->tets, p->faces
+
+
+  // TODO assemble RHS
 }
 
 

From 9d5af8ea2c485f32f2c9ae2bdbcaf05f0a463d1b Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@portal2.scorec.rpi.edu>
Date: Mon, 4 May 2020 23:31:13 -0400
Subject: [PATCH 130/555] Adds getElementDofs.

Adds getElementDofs to get all the dofs of an element (including
dofs on bounding entities) in a downward adjacent order.
Vertices first, then edges, then faces, and then volume dofs.
---
 apf/apfElement.cc | 10 ++++++++++
 apf/apfElement.h  |  4 ++++
 2 files changed, 14 insertions(+)

diff --git a/apf/apfElement.cc b/apf/apfElement.cc
index 0b9df9436..3c25b146b 100644
--- a/apf/apfElement.cc
+++ b/apf/apfElement.cc
@@ -12,6 +12,9 @@
 #include "apfMesh.h"
 #include "apfVectorElement.h"
 
+#include <mth.h>
+#include <mth_def.h>
+
 namespace apf {
 
 void Element::init(Field* f, MeshEntity* e, VectorElement* p)
@@ -119,4 +122,11 @@ void Element::getNodeData()
   field->getData()->getElementData(entity,nodeData);
 }
 
+void Element::getElementDofs(NewArray<double>& d)
+{
+  d.allocated() ? d.resize(nen) : d.allocate(nen);
+  for (int i = 0; i < nen; i++)
+    d[i] = nodeData[i];
+}
+
 }//namespace apf
diff --git a/apf/apfElement.h b/apf/apfElement.h
index 54318d32b..72db8185e 100644
--- a/apf/apfElement.h
+++ b/apf/apfElement.h
@@ -12,6 +12,9 @@
 #include "apfField.h"
 #include "apfShape.h"
 
+#include <mth.h>
+#include <mth_def.h>
+
 namespace apf {
 
 class EntityShape;
@@ -35,6 +38,7 @@ class Element
     EntityShape* getShape() {return shape;}
     FieldShape* getFieldShape() {return field->getShape();}
     void getComponents(Vector3 const& xi, double* c);
+    void getElementDofs(NewArray<double>& d);
   protected:
     void init(Field* f, MeshEntity* e, VectorElement* p);
     void getNodeData();

From 221377ef7649d91cfb0c6fd6093e46646ee1dd63 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@portal2.scorec.rpi.edu>
Date: Mon, 4 May 2020 23:35:32 -0400
Subject: [PATCH 131/555] removes extra headers from apfElement.h and cc

---
 apf/apfElement.cc | 3 ---
 apf/apfElement.h  | 3 ---
 2 files changed, 6 deletions(-)

diff --git a/apf/apfElement.cc b/apf/apfElement.cc
index 3c25b146b..5b666fa57 100644
--- a/apf/apfElement.cc
+++ b/apf/apfElement.cc
@@ -12,9 +12,6 @@
 #include "apfMesh.h"
 #include "apfVectorElement.h"
 
-#include <mth.h>
-#include <mth_def.h>
-
 namespace apf {
 
 void Element::init(Field* f, MeshEntity* e, VectorElement* p)
diff --git a/apf/apfElement.h b/apf/apfElement.h
index 72db8185e..96b3677ce 100644
--- a/apf/apfElement.h
+++ b/apf/apfElement.h
@@ -12,9 +12,6 @@
 #include "apfField.h"
 #include "apfShape.h"
 
-#include <mth.h>
-#include <mth_def.h>
-
 namespace apf {
 
 class EntityShape;

From a26f9a31fdf5fec945e50ac127b99d2fd4b0ac53 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@portal2.scorec.rpi.edu>
Date: Mon, 4 May 2020 23:38:00 -0400
Subject: [PATCH 132/555] Adds functions to compute integrals.

Adds functions to compute integrals required to set up the RHS
of the edge patch system of equations.
TODO add flux term integral.

The functions added compute
1) local bilinear form integral
  a) assemble curl-curl element matrix
  b) assemble vector mass element matrix
2) local linear form intgral
  a) evaluate user defined vector function
  a) assemble domain linear form element vector
---
 em/emResidualFunctionals.cc | 230 ++++++++++++++++++++++++++++++++++--
 1 file changed, 218 insertions(+), 12 deletions(-)

diff --git a/em/emResidualFunctionals.cc b/em/emResidualFunctionals.cc
index 513f2fb2a..36b865927 100644
--- a/em/emResidualFunctionals.cc
+++ b/em/emResidualFunctionals.cc
@@ -6,6 +6,7 @@
  */
 #include <iostream>
 
+#include <apf.h>
 #include <PCU.h>
 #include <lionPrint.h>
 
@@ -14,6 +15,7 @@
 #include <apfMesh.h>
 #include <apfShape.h>
 #include <apfField.h>
+#include <apfElement.h>
 #include <apfCavityOp.h>
 #include "crv.h"
 #include "crvShape.h"
@@ -160,46 +162,250 @@ static void assembleLHS(EdgePatch* p)
   }
 }
 
+static void assembleCurlCurlElementMatrix(apf::Mesh* mesh,apf::MeshEntity* e,
+    apf::Field* f, mth::Matrix<double>& elmat)
+{
+  apf::FieldShape* fs = f->getShape();
+  int type = mesh->getType(e);
+  int nd = apf::countElementNodes(fs, type);
+  int dim = apf::getDimension(mesh, e);
+  int dimc = (dim == 3) ? 3 : 1;
+  double w;
+
+  apf::NewArray<apf::Vector3> curlshape(nd);
+  mth::Matrix<double> phys_curlshape(nd, dimc);
+  elmat.resize(nd,nd);
+
+  apf::MeshElement* me = apf::createMeshElement(mesh, e);
+  apf::Element* el = apf::createElement(f, me);
+  int order = 2 * fs->getOrder() - 2;
+  int np = apf::countIntPoints(me, order); // int points required
+
+  elmat.zero();
+  apf::Vector3 p;
+  for (int i = 0; i < np; i++) {
+    apf::getIntPoint(me, order, i, p);
+    double weight = apf::getIntWeight(me, order, i);
+    apf::Matrix3x3 J;
+    apf::getJacobian(me, p, J);
+    double jdet = apf::getJacobianDeterminant(J, dim);
+    w = weight / jdet;
+
+    if (dim == 3) {
+      el->getShape()->getLocalVectorCurls(mesh, e, p, curlshape);
+      phys_curlshape.zero();
+      for (int i = 0; i < nd; i++)
+        for (int j = 0; j < dim; j++)
+          for (int k = 0; k < dim; k++)
+            phys_curlshape(i,j) += curlshape[i][k] * J[k][j];
+    }
+    else {
+      /*
+      TODO 2D TODO in apfNedelec.cc
+      el->getShape()->getLocalVectorCurls(mesh, e, p, curlshape);
+      phys_curlshape.zero();
+      for (int i = 0; i < nd; i++)
+        for (int j = 0; j < dimc; j++)
+        */
+    }
+    mth::Matrix<double> phys_curlshapeT;
+    mth::transpose(phys_curlshape, phys_curlshapeT);
+    mth::Matrix<double> M (nd, nd);
+    M.zero();
+    mth::multiply(phys_curlshape, phys_curlshapeT, M);
+    M *= w;
+    elmat += M;
+  }
+}
+
+static void assembleVectorMassElementMatrix(apf::Mesh* mesh,apf::MeshEntity* e,
+    apf::Field* f, mth::Matrix<double>& elmat)
+{
+  apf::FieldShape* fs = f->getShape();
+  int type = mesh->getType(e);
+  int nd = apf::countElementNodes(fs, type);
+  int dim = apf::getDimension(mesh, e);
+  double w;
+
+  apf::NewArray<apf::Vector3> vectorshape(nd);
+  elmat.resize(nd,nd);
+
+  apf::MeshElement* me = apf::createMeshElement(mesh, e);
+  apf::Element* el = apf::createElement(f, me);
+  int order = 2 * fs->getOrder();
+  int np = apf::countIntPoints(me, order); // int points required
+
+  elmat.zero();
+  apf::Vector3 p;
+  for (int i = 0; i < np; i++) {
+    apf::getIntPoint(me, order, i, p);
+    double weight = apf::getIntWeight(me, order, i);
+    apf::Matrix3x3 J;
+    apf::getJacobian(me, p, J);
+    double jdet = apf::getJacobianDeterminant(J, dim);
+    w = weight * jdet;
+
+    apf::getVectorShapeValues(el, p, vectorshape);
+    mth::Matrix<double> vectorShape (nd, dim);
+    for (int i = 0; i < nd; i++)
+      for (int j = 0; j < dim; j++)
+        vectorShape(i,j) = vectorshape[i][j];
+
+    mth::Matrix<double> vectorShapeT (dim, nd);
+    mth::transpose(vectorShape, vectorShapeT);
+    mth::Matrix<double> M (nd,nd);
+    M.zero();
+    mth::multiply(vectorShape, vectorShapeT, M);
+    M *= w;
+    elmat += M;
+  }
+}
+
 // computes local bilinear form integral restricted to
 // an edge of an element
-static int getLocalBLFIntegral(EdgePatch* p, apf::MeshEntity* tet)
+static double getLocalBLFIntegral(EdgePatch* p, apf::MeshEntity* tet)
 {
   // find position of edge in downward edges of element
   apf::Downward e;
   int ne = p->mesh->getDownward(tet, 1, e);
   int ei = apf::findIn(e, ne, p->entity);
   // get Element Dofs
-  //
+  apf::MeshElement* me = apf::createMeshElement(p->mesh, tet);
+  apf::Element* el = apf::createElement(p->equilibration->ef, me);
+  int type = p->mesh->getType(tet);
+  int nd = apf::countElementNodes(el->getFieldShape(), type);
+  apf::NewArray<double> d (nd);
+  el->getElementDofs(d);
+  mth::Vector<double> dofs (nd);
+  for (int i = 0; i < nd; i++) // TODO cleanup
+    dofs(i) = d[i];
   // assemble curl curl element matrix
+  mth::Matrix<double> curl_elmat;
+  assembleCurlCurlElementMatrix(p->mesh, tet,
+      p->equilibration->ef, curl_elmat);
   // assemble vector mass element matrix
+  mth::Matrix<double> mass_elmat;
+  assembleVectorMassElementMatrix(p->mesh, tet,
+      p->equilibration->ef, mass_elmat);
   // add element matrices
+  mth::Matrix<double> elmat(nd, nd);
+  elmat.zero();
+  elmat += curl_elmat;
+  elmat += mass_elmat;
   // multiply element matrix with element dofs
+  mth::Vector<double> integrals (nd);
+  mth::multiply(elmat, dofs, integrals);
   // pick edge index from the resulting vector
+  return integrals(ei);
+}
+
+// TODO redo this to allow user access from outside
+static void pumiUserFunction(const apf::Vector3& x, mth::Vector<double>& f,
+    apf::MeshEntity* tet, apf::Mesh* mesh)
+{
+  double freq = 1.;
+  double kappa = freq * M_PI;
+  int dim = apf::getDimension(mesh, tet);
+  if (dim == 3) {
+      f(0) = (1. + kappa * kappa) * sin(kappa * x[1]);
+      f(1) = (1. + kappa * kappa) * sin(kappa * x[2]);
+      f(2) = (1. + kappa * kappa) * sin(kappa * x[0]);
+  }
+  else {
+     f(0) = (1. + kappa * kappa) * sin(kappa * x[1]);
+     f(1) = (1. + kappa * kappa) * sin(kappa * x[0]);
+     f(2) = 0.0;
+  }
+}
+
+void assembleDomainLFElementVector(apf::Mesh* mesh, apf::MeshEntity* e,
+    apf::Field* f, mth::Vector<double>& elvect)
+{
+  apf::MeshElement* me = apf::createMeshElement(mesh, e);
+  apf::Element* el = apf::createElement(f, me);
+  int type = mesh->getType(e);
+  int nd = apf::countElementNodes(el->getFieldShape(), type);
+  int dim = apf::getDimension(mesh, e);
+
+  apf::NewArray<apf::Vector3> vectorshape(nd);
+  elvect.resize(nd);
+  mth::Vector<double> val (3);
+  val.zero();
+  apf::Vector3 p;
+  double w;
+
+  apf::FieldShape* fs = f->getShape();
+  int order = 2 * fs->getOrder();
+  int np = apf::countIntPoints(me, order); // int points required
+
+  elvect.zero();
+  for (int i = 0; i < np; i++) {
+    apf::getIntPoint(me, order, i, p);
+    double weight = apf::getIntWeight(me, order, i);
+    apf::Matrix3x3 J;
+    apf::getJacobian(me, p, J);
+    double jdet = apf::getJacobianDeterminant(J, dim);
+    w = weight * jdet;
+
+    apf::getVectorShapeValues(el, p, vectorshape);
+    pumiUserFunction(p, val, e, apf::getMesh(f)); // eval vector function
+    val *= w;
+
+    mth::Matrix<double> vectorShape (nd, dim);
+    for (int i = 0; i < nd; i++)
+      for (int j = 0; j < dim; j++)
+        vectorShape(i,j) = vectorshape[i][j];
+    mth::Vector<double> temp (nd);
+    temp.zero();
+    mth::multiply(vectorShape, val, temp);
+    elvect += temp;
+  }
+}
+
+static double getLocalLFIntegral(EdgePatch* p, apf::MeshEntity* tet)
+{
+  // find position of edge in downward edges of element
+  apf::Downward e;
+  int ne = p->mesh->getDownward(tet, 1, e);
+  int ei = apf::findIn(e, ne, p->entity);
+  // assemble Domain LF Vector
+  mth::Vector<double> elvect;
+  assembleDomainLFElementVector(p->mesh, tet,
+      p->equilibration->ef, elvect);
+  return elvect(ei);
 }
 
 
 static void assembleRHS(EdgePatch* p)
 {
   if (p->isOnBdry) {
+    std::cout << "boundary" << std::endl; // TODO remove
     p->b.resize(p->tets.size() + p->faces.size());
     p->b.zero();
   }
   else {
+    std::cout << "interior" << std::endl; // TODO remove
     p->b.resize(p->tets.size());
     p->b.zero();
   }
-
-  apf::MeshEntity* edge = p->entity[i];
+  printf("ASSEMBLE RHS\n");
+  //apf::MeshEntity* edge = p->entity;
   int ne = p->tets.size();
+  int nf = p->faces.size();
   for (int i = 0; i < ne; i++) {
-    apf::MeshEntity* tet = p->tets[i];
-    int blfIntegral = getLocalBLFIntegral(p, tet);
+    // TODO computeBilinearFormIntegral
+    EntitySet::iterator it = std::next(p->tets.begin(), i);
+    apf::MeshEntity* tet = *it;
+    double blfIntegral = getLocalBLFIntegral(p, tet);
+    double lfIntegral = getLocalLFIntegral(p, tet);
+    // TODO computeLinearFormIntegral
+    // TODO computeFluxTermIntegral
+    if(p->isOnBdry)
+      p->b(nf+i) = blfIntegral - lfIntegral; // TODO add -flux term integral
+    else
+      p->b(i) = blfIntegral - lfIntegral;    // TODO add -flux term integral
   }
-  // TODO computeBilinearFormIntegral
-  // TODO computeLinearFormIntegral
-  // TODO computeFluxTermIntegral
-
-
+  std::cout << p->b << std::endl; // TODO remove
 }
 
 // The following two functions help order tets and faces in a cavity in a
@@ -290,7 +496,7 @@ static void runErm(EdgePatch* p)
   //std::vector<apf::MeshEntity*> otets; // ordered tets // TODO remove these two
   //std::vector<apf::MeshEntity*> ofaces; // ordered faces
   getOrderedTetsandFaces(p->mesh, p->entity, p->tets, p->faces); // TODO maybe use p->tets, p->faces
-
+  assembleRHS(p);
 
   // TODO assemble RHS
 }

From 5540707b9ec6ab4faf06c8694fd860b8234f12dc Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Tue, 5 May 2020 23:58:12 -0400
Subject: [PATCH 133/555] Adds getFluxIntegrals. Solves system of eqns.

---
 em/emResidualFunctionals.cc | 154 +++++++++++++++++++++++++++++++-----
 1 file changed, 133 insertions(+), 21 deletions(-)

diff --git a/em/emResidualFunctionals.cc b/em/emResidualFunctionals.cc
index 36b865927..b4d51dea9 100644
--- a/em/emResidualFunctionals.cc
+++ b/em/emResidualFunctionals.cc
@@ -69,6 +69,7 @@ struct EdgePatch {
   EntitySet faces;
   mth::Matrix<double> A;
   mth::Vector<double> b;
+  mth::Vector<double> x;
   QRDecomp qr;
 
 };
@@ -124,7 +125,7 @@ static void assembleLHS(EdgePatch* p)
 {
   int ne = p->tets.size();
   int nf = p->faces.size();
-  printf("ne %d nf %d \n", ne, nf); // TODO remove
+  printf("ne %d nf %d \n", ne, nf); // REMOVE
   if( crv::isBoundaryEntity(p->mesh, p->entity) ) {
     p->A.resize(ne+nf, ne+nf);
     p->A.zero();
@@ -137,8 +138,8 @@ static void assembleLHS(EdgePatch* p)
     p->A(ne+nf-1, ne-1) = 1.; p->A(ne+nf-1, ne) = 1.;
     p->A(ne-1, ne+nf-1) = 1.; p->A(ne, ne+nf-1) = 1.;
 
-    std::cout << "boundary" << std::endl; // TODO remove
-    std::cout << p->A << std::endl; // TODO remove
+    std::cout << "boundary" << std::endl; // REMOVE
+    std::cout << p->A << std::endl; // REMOVE
   }
   else if( ! crv::isBoundaryEntity(p->mesh, p->entity) ) {
     mth::Matrix<double> m(ne, nf);
@@ -157,9 +158,14 @@ static void assembleLHS(EdgePatch* p)
     for (int i = 0; i < ne; i++)
       for (int j = 0; j < ne; j++)
         p->A(i,j) += 1.;
-    std::cout << "interior" << std::endl; // TODO remove
-    std::cout << p->A << std::endl; // TODO remove
+    std::cout << "interior" << std::endl; // REMOVE
+    std::cout << p->A << std::endl; // REMOVE
   }
+  mth::decomposeQR(p->A, p->qr.Q, p->qr.R);
+  std::cout << "Q" << std::endl;
+  std::cout << p->qr.Q << std::endl;
+  std::cout << "R" << std::endl;
+  std::cout << p->qr.R << std::endl;
 }
 
 static void assembleCurlCurlElementMatrix(apf::Mesh* mesh,apf::MeshEntity* e,
@@ -216,6 +222,8 @@ static void assembleCurlCurlElementMatrix(apf::Mesh* mesh,apf::MeshEntity* e,
     M *= w;
     elmat += M;
   }
+  apf::destroyElement(el);
+  apf::destroyMeshElement(me);
 }
 
 static void assembleVectorMassElementMatrix(apf::Mesh* mesh,apf::MeshEntity* e,
@@ -259,6 +267,8 @@ static void assembleVectorMassElementMatrix(apf::Mesh* mesh,apf::MeshEntity* e,
     M *= w;
     elmat += M;
   }
+  apf::destroyElement(el);
+  apf::destroyMeshElement(me);
 }
 
 // computes local bilinear form integral restricted to
@@ -295,11 +305,15 @@ static double getLocalBLFIntegral(EdgePatch* p, apf::MeshEntity* tet)
   // multiply element matrix with element dofs
   mth::Vector<double> integrals (nd);
   mth::multiply(elmat, dofs, integrals);
+
+  apf::destroyElement(el);
+  apf::destroyMeshElement(me);
+
   // pick edge index from the resulting vector
   return integrals(ei);
 }
 
-// TODO redo this to allow user access from outside
+// TODO QUESTION redo this to allow user access from outside
 static void pumiUserFunction(const apf::Vector3& x, mth::Vector<double>& f,
     apf::MeshEntity* tet, apf::Mesh* mesh)
 {
@@ -360,6 +374,8 @@ void assembleDomainLFElementVector(apf::Mesh* mesh, apf::MeshEntity* e,
     mth::multiply(vectorShape, val, temp);
     elvect += temp;
   }
+  apf::destroyElement(el);
+  apf::destroyMeshElement(me);
 }
 
 static double getLocalLFIntegral(EdgePatch* p, apf::MeshEntity* tet)
@@ -375,37 +391,134 @@ static double getLocalLFIntegral(EdgePatch* p, apf::MeshEntity* tet)
   return elvect(ei);
 }
 
+static double getFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
+{
+  double fluxIntegral = 0.0;
+  // 1. find position of edge in downward edges of element
+  apf::Downward e;
+  int ne = ep->mesh->getDownward(tet, 1, e);
+  int ei = apf::findIn(e, ne, ep->entity);
+  // 2. get faces of the tet in the patch
+  apf::Downward f;
+  int nf = ep->mesh->getDownward(tet, 2, f);
+  PCU_ALWAYS_ASSERT(nf == 4);
+  std::vector<apf::MeshEntity*> patchFaces;
+  for (int i = 0; i < nf; i++) {
+    if(ep->faces.count(f[i]))
+      patchFaces.push_back(f[i]);
+  }
+  PCU_ALWAYS_ASSERT(patchFaces.size() == 2);
+  //  3. loop over the patch faces
+  for (unsigned int i = 0; i < patchFaces.size(); i++) {
+    // 4. get upward tets of the current face
+    apf::Up up;
+    apf::MeshEntity* currentFace = patchFaces[i];
+    ep->mesh->getUp(currentFace, up);
+    if (crv::isBoundaryEntity(ep->mesh, currentFace))
+      PCU_ALWAYS_ASSERT( up.n == 1);
+    else
+      PCU_ALWAYS_ASSERT( up.n == 2);
+
+    apf::MeshEntity* firstTet = up.e[0];
+    apf::MeshEntity* secondTet;
+    if (up.n == 2)
+      secondTet  = up.e[1];
+
+    // 5. count integration points for flux face integral
+    apf::FieldShape* fs = ep->equilibration->ef->getShape();
+    apf::MeshElement* fme = apf::createMeshElement(ep->mesh, currentFace);
+    apf::Element* fel = apf::createElement(ep->equilibration->ef, fme);
+    int order = 2 * fs->getOrder();
+    int np = apf::countIntPoints(fme, order); // int points required
+
+    // loop over integration points
+    apf::Vector3 p, tet1xi, tet2xi, curl1, curl2, curl, fnormal, tk, vshape;
+    for (int n = 0; n < np; n++) {
+      apf::getIntPoint(fme, order, n, p);
+      double weight = apf::getIntWeight(fme, order, n);
+
+      // compute face normal
+      apf::Matrix3x3 fJ;
+      apf::getJacobian(fme, p, fJ);
+      double jdet = apf::getJacobianDeterminant(
+          fJ, apf::getDimension(ep->mesh, currentFace));
+      apf::Vector3 r1 = fJ[0];
+      apf::Vector3 r2 = fJ[1];
+      fnormal = apf::cross( r1, r2 );
+      fnormal = fnormal.normalize();
+
+      curl.zero();
+      // compute curl1
+      tet1xi = apf::boundaryToElementXi(ep->mesh, currentFace, firstTet, p);
+      apf::MeshElement* me1 = apf::createMeshElement(ep->mesh, firstTet);
+      apf::Element* el1 = apf::createElement(ep->equilibration->ef, me1);
+      apf::getCurl(el1, tet1xi, curl1);
+      curl += curl1;
+      apf::destroyElement(el1);
+      apf::destroyMeshElement(me1);
+
+      // compute curl2
+      if (up.n == 2) {
+        tet2xi = apf::boundaryToElementXi(ep->mesh, currentFace, secondTet, p);
+        apf::MeshElement* me2 = apf::createMeshElement(ep->mesh, secondTet);
+        apf::Element* el2 = apf::createElement(ep->equilibration->ef, me2);
+        apf::getCurl(el2, tet2xi, curl2);
+        curl += curl2;
+        apf::destroyElement(el2);
+        apf::destroyMeshElement(me2);
+      }
+
+      // compute tk (inter-element averaged flux)
+      tk = apf::cross(fnormal, curl);
+      tk = tk * 1./2.;
+
+      // compute vector shape
+      int type = ep->mesh->getType(tet);
+      int nd = apf::countElementNodes(fs, type);
+      apf::NewArray<apf::Vector3> vectorshapes (nd);
+      apf::MeshElement* me = apf::createMeshElement(ep->mesh, tet);
+      apf::Element* el = apf::createElement(ep->equilibration->ef, me);
+      apf::getVectorShapeValues(el, p, vectorshapes);
+      vshape = vectorshapes[ei];
+      apf::destroyElement(el);
+      apf::destroyMeshElement(me);
+
+      // compute integral
+      fluxIntegral += tk * vshape * weight * jdet;
+    }
+    apf::destroyElement(fel);
+    apf::destroyMeshElement(fme);
+  }
+  return fluxIntegral;
+}
 
 static void assembleRHS(EdgePatch* p)
 {
   if (p->isOnBdry) {
-    std::cout << "boundary" << std::endl; // TODO remove
+    std::cout << "boundary" << std::endl; // REMOVE
     p->b.resize(p->tets.size() + p->faces.size());
     p->b.zero();
   }
   else {
-    std::cout << "interior" << std::endl; // TODO remove
+    std::cout << "interior" << std::endl; // REMOVE
     p->b.resize(p->tets.size());
     p->b.zero();
   }
-  printf("ASSEMBLE RHS\n");
-  //apf::MeshEntity* edge = p->entity;
+  printf("RHS \n");
   int ne = p->tets.size();
   int nf = p->faces.size();
   for (int i = 0; i < ne; i++) {
-    // TODO computeBilinearFormIntegral
     EntitySet::iterator it = std::next(p->tets.begin(), i);
     apf::MeshEntity* tet = *it;
     double blfIntegral = getLocalBLFIntegral(p, tet);
     double lfIntegral = getLocalLFIntegral(p, tet);
-    // TODO computeLinearFormIntegral
-    // TODO computeFluxTermIntegral
+    double fluxIntegral = getFluxIntegral(p, tet);
     if(p->isOnBdry)
-      p->b(nf+i) = blfIntegral - lfIntegral; // TODO add -flux term integral
+      p->b(nf+i) = blfIntegral - lfIntegral - fluxIntegral;
     else
-      p->b(i) = blfIntegral - lfIntegral;    // TODO add -flux term integral
+      p->b(i) = blfIntegral - lfIntegral - fluxIntegral;
   }
-  std::cout << p->b << std::endl; // TODO remove
+  std::cout << p->b << std::endl; // REMOVE
 }
 
 // The following two functions help order tets and faces in a cavity in a
@@ -491,14 +604,13 @@ static void getOrderedTetsandFaces(apf::Mesh* mesh, apf::MeshEntity* edge,
 
 static void runErm(EdgePatch* p)
 {
+  getOrderedTetsandFaces(p->mesh, p->entity, p->tets, p->faces);
   assembleLHS(p);
-  // TODO decompose A into Q and R
-  //std::vector<apf::MeshEntity*> otets; // ordered tets // TODO remove these two
-  //std::vector<apf::MeshEntity*> ofaces; // ordered faces
-  getOrderedTetsandFaces(p->mesh, p->entity, p->tets, p->faces); // TODO maybe use p->tets, p->faces
   assembleRHS(p);
+  mth::solveFromQR(p->qr.Q, p->qr.R, p->b, p->x);
 
-  // TODO assemble RHS
+  std::cout << "x" << std::endl; // REMOVE
+  std::cout << p->x << std::endl; // REMOVE
 }
 
 

From 8e49bc264ea86204dc0f653ae143f2e7695457d4 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Wed, 6 May 2020 11:04:21 -0400
Subject: [PATCH 134/555] Implements getNodeXi for constant shapes

Only for simplex types.
---
 apf/apfShape.cc | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/apf/apfShape.cc b/apf/apfShape.cc
index 7ebaa52f0..fe4f38b7d 100644
--- a/apf/apfShape.cc
+++ b/apf/apfShape.cc
@@ -1065,6 +1065,20 @@ class Constant : public FieldShape
         return 0;
    }
     int getOrder() {return 0;}
+    void getNodeXi(int type, int node, Vector3& xi)
+    {
+      PCU_ALWAYS_ASSERT(node == 0);
+      if (type == Mesh::VERTEX)
+        xi = Vector3(0., 0., 0.);
+      else if (type == Mesh::EDGE)
+        xi = Vector3(0., 0., 0.);
+      else if (type == Mesh::TRIANGLE)
+        xi = Vector3(1./3., 1./3., 0);
+      else if (type == Mesh::TET)
+        xi = Vector3(1./4., 1./4., 1./4.);
+      else
+        PCU_ALWAYS_ASSERT_VERBOSE(0, "non implemented for non simplex types!");
+    }
   private:
     std::string name;
 };

From b15c30af301204c322641431a5e39b242b0b42b4 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 6 May 2020 15:22:25 -0400
Subject: [PATCH 135/555] add test that loads and deletes a mesh, fix leaks

issue #107
---
 pumi/pumi.h          |  7 ++++---
 pumi/pumi_geom.cc    | 16 +++++++++-------
 pumi/pumi_mesh.cc    | 10 +---------
 test/CMakeLists.txt  |  1 +
 test/pumi.cc         |  3 ++-
 test/pumiLoadMesh.cc | 15 +++++++++++++++
 test/testing.cmake   |  5 +++++
 7 files changed, 37 insertions(+), 20 deletions(-)
 create mode 100644 test/pumiLoadMesh.cc

diff --git a/pumi/pumi.h b/pumi/pumi.h
index af252137e..529d528ba 100644
--- a/pumi/pumi.h
+++ b/pumi/pumi.h
@@ -86,7 +86,10 @@ class pumi
 public:
   pumi();
   ~pumi();
-  static pumi* instance();
+  static pumi* instance() {
+    static pumi _instance;
+    return &_instance;
+  };
 
   pMesh mesh;
   pGeom model;  
@@ -97,8 +100,6 @@ class pumi
   pMeshTag ghost_tag;
   std::vector<pMeshEnt> ghost_vec[4];
   std::vector<pMeshEnt> ghosted_vec[4];
-private:
-  static pumi* _instance;
 };
 
 //************************************
diff --git a/pumi/pumi_geom.cc b/pumi/pumi_geom.cc
index 6e5bc5273..6fe0edb5b 100644
--- a/pumi/pumi_geom.cc
+++ b/pumi/pumi_geom.cc
@@ -24,10 +24,7 @@ gModel::gModel(gmi_model* model) : TagHolder()
   g = model;
 }
 
-gModel::~gModel()
-{
-  delete g;
-}
+gModel::~gModel() {}
 
 gEntity* gModel::getGeomEnt(int d, gmi_ent* ge)
 {
@@ -90,13 +87,18 @@ void pumi_geom_delete(pGeom g)
 {
   pTag id_tag=pumi_geom_findTag(g, "ID");
 
-  for (int i=0; i<4; ++i)
+  for (int i=0; i<4; ++i) {
+    std::vector<pGeomEnt> vge(g->size(i));
     for (pGeomIter gent_it = g->begin(i); gent_it!=g->end(i);++gent_it)
     {
       if (id_tag) pumi_gent_deleteTag(*gent_it, id_tag);
-      delete *gent_it;
+      vge.push_back(*gent_it);
     }
-   pumi_geom_deleteTag(g, id_tag);
+    for(size_t j=0; j<vge.size(); j++)
+      delete vge[j];
+  }
+  pumi_geom_deleteTag(g, id_tag);
+  delete g;
 }
 
 void pumi_geom_freeze(pGeom g)
diff --git a/pumi/pumi_mesh.cc b/pumi/pumi_mesh.cc
index 422315647..b48ed473f 100644
--- a/pumi/pumi_mesh.cc
+++ b/pumi/pumi_mesh.cc
@@ -162,17 +162,9 @@ pumi::~pumi()
     delete [] num_own_ent;
     delete [] num_global_ent;
   }
-  delete _instance;
-  _instance = NULL;
 }
 
-pumi* pumi::_instance=NULL;
-pumi* pumi::instance()
-{
-  if (_instance==NULL)
-    _instance = new pumi();
-  return _instance;
-}
+
 
 apf::Migration* getPlan(apf::Mesh* m, int num_target_part)
 {
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 7e2f45ae7..af2bc7936 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -158,6 +158,7 @@ test_exe_func(fusion3 fusion3.cc)
 test_exe_func(1d 1d.cc)
 test_exe_func(base64 base64.cc)
 test_exe_func(test_pumi pumi.cc)
+test_exe_func(pumiLoadMesh pumiLoadMesh.cc)
 test_exe_func(xgc_split xgc_split.cc)
 test_exe_func(ma_insphere ma_insphere.cc)
 test_exe_func(ma_test ma_test.cc)
diff --git a/test/pumi.cc b/test/pumi.cc
index 249cadce8..c46696b70 100644
--- a/test/pumi.cc
+++ b/test/pumi.cc
@@ -191,6 +191,7 @@ int main(int argc, char** argv)
   TEST_MESH(m);
  
  // re-load partitioned mesh via file i/o
+  pumi_geom_delete(g);
   pumi_mesh_delete(m);
 
   g = pumi_geom_load(modelFile);
@@ -341,7 +342,7 @@ void TEST_GENT_SETGET_TAG (pGeom g, pGeomEnt ent)
 
   // pumi_gent_set/getPtrTag 
   pumi_gent_setPtrTag (ent, pointer_tag, (void*)(data));
-  void* void_data = (void*)calloc(strlen(data), sizeof(char));
+  void* void_data = (void*)calloc(strlen(data), sizeof(char)); //this memory is leaked
   pumi_gent_getPtrTag (ent, pointer_tag, &void_data);
   PCU_ALWAYS_ASSERT(!strcmp((char*)void_data, data));
 
diff --git a/test/pumiLoadMesh.cc b/test/pumiLoadMesh.cc
new file mode 100644
index 000000000..c5b5d0d01
--- /dev/null
+++ b/test/pumiLoadMesh.cc
@@ -0,0 +1,15 @@
+#include "mpi.h"
+#include "pumi.h"
+
+int main(int argc, char** argv)
+{
+  MPI_Init(&argc,&argv);
+  pumi_start();
+  pGeom g = pumi_geom_load(argv[1], "mesh");
+  pMesh m = pumi_mesh_load(g, argv[2], 1);
+  pumi_mesh_delete(m);
+  pumi_geom_delete(g);
+  pumi_finalize();
+  MPI_Finalize();
+}
+
diff --git a/test/testing.cmake b/test/testing.cmake
index 210f3f155..02754de6f 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -141,6 +141,7 @@ if(ENABLE_SIMMETRIX AND PCU_COMPRESS AND SIM_PARASOLID
     WORKING_DIRECTORY ${MDIR})
 endif()
 
+
 if(ENABLE_ZOLTAN)
   mpi_test(pumi3d-1p 4
     ./test_pumi
@@ -516,6 +517,10 @@ mpi_test(nedelec 1
   ./nedelecShapes
   "${MDIR}/cube.dmg"
   "${MDIR}/cube.smb")
+mpi_test(pumiLoadMesh-1p 1
+  ./pumiLoadMesh
+  ${MDIR}/cube.dmg
+  ${MDIR}/cube.smb)
 set(MDIR ${MESHES}/cube)
 mpi_test(test_verify 4
   ./test_verify

From a0660a8c8b16614d3e377ac7a3f235f81d5a5d16 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@portal2.scorec.rpi.edu>
Date: Wed, 6 May 2020 17:44:57 -0400
Subject: [PATCH 136/555] Fixes errors to verify edge equilibration.

Fixes errors in the computation of local bilinear and linear form
integrals by negating the value of the integrals when the edge
is flipped.
Also, maps the local xi to global point in the user defined
funtion.
---
 em/emResidualFunctionals.cc | 81 ++++++++++++++++++++++++++++++++-----
 1 file changed, 71 insertions(+), 10 deletions(-)

diff --git a/em/emResidualFunctionals.cc b/em/emResidualFunctionals.cc
index b4d51dea9..72c484363 100644
--- a/em/emResidualFunctionals.cc
+++ b/em/emResidualFunctionals.cc
@@ -5,7 +5,7 @@
  * BSD license as described in the LICENSE file in the top-level directory.
  */
 #include <iostream>
-
+#include <cstdlib>
 #include <apf.h>
 #include <PCU.h>
 #include <lionPrint.h>
@@ -110,8 +110,32 @@ static bool getInitialEdgePatch(EdgePatch* p, apf::CavityOp* o)
   apf::DynamicArray<apf::MeshEntity*> adjacent;
   p->mesh->getAdjacent(p->entity, 3, adjacent);
   addEntitiesToPatch(p, adjacent);
+  // REMOVE ALL
+  int ne = adjacent.size();
+  std::cout << " Unordered Tets, ne: " << ne << std::endl; // REMOVE
+  std::cout << " Center of Unordered tets" << std::endl;
+  for (int i = 0; i < ne; i++) {
+    apf::MeshEntity* tet = adjacent[i];;
+    apf::Vector3 center = apf::getLinearCentroid(p->mesh, tet);
+    std::cout << i << ":  " << center << std::endl;
+  } //////////
+
+
+
   p->mesh->getAdjacent(p->entity, 2, adjacent);
   addEntitiesToPatch(p, adjacent);
+
+  // REMOVE ALL
+  int nf = adjacent.size();
+  std::cout << " Unordered Faces, nf: " << nf << std::endl; // REMOVE
+  std::cout << " Center of Unordered Faces" << std::endl;
+  for (int i = 0; i < nf; i++) {
+    apf::MeshEntity* face = adjacent[i];
+    apf::Vector3 center = apf::getLinearCentroid(p->mesh, face);
+    std::cout << i << ":  " << center << std::endl;
+  }
+  std::cout << "----------------" << std::endl;
+
   return true;
 }
 
@@ -125,7 +149,7 @@ static void assembleLHS(EdgePatch* p)
 {
   int ne = p->tets.size();
   int nf = p->faces.size();
-  printf("ne %d nf %d \n", ne, nf); // REMOVE
+  //printf("ne %d nf %d \n", ne, nf); // REMOVE
   if( crv::isBoundaryEntity(p->mesh, p->entity) ) {
     p->A.resize(ne+nf, ne+nf);
     p->A.zero();
@@ -138,8 +162,8 @@ static void assembleLHS(EdgePatch* p)
     p->A(ne+nf-1, ne-1) = 1.; p->A(ne+nf-1, ne) = 1.;
     p->A(ne-1, ne+nf-1) = 1.; p->A(ne, ne+nf-1) = 1.;
 
-    std::cout << "boundary" << std::endl; // REMOVE
-    std::cout << p->A << std::endl; // REMOVE
+    //std::cout << "boundary" << std::endl; // REMOVE
+    //std::cout << p->A << std::endl; // REMOVE
   }
   else if( ! crv::isBoundaryEntity(p->mesh, p->entity) ) {
     mth::Matrix<double> m(ne, nf);
@@ -158,14 +182,14 @@ static void assembleLHS(EdgePatch* p)
     for (int i = 0; i < ne; i++)
       for (int j = 0; j < ne; j++)
         p->A(i,j) += 1.;
-    std::cout << "interior" << std::endl; // REMOVE
-    std::cout << p->A << std::endl; // REMOVE
+    //std::cout << "interior" << std::endl; // REMOVE
+    //std::cout << p->A << std::endl; // REMOVE
   }
   mth::decomposeQR(p->A, p->qr.Q, p->qr.R);
-  std::cout << "Q" << std::endl;
+  /*std::cout << "Q" << std::endl;
   std::cout << p->qr.Q << std::endl;
   std::cout << "R" << std::endl;
-  std::cout << p->qr.R << std::endl;
+  std::cout << p->qr.R << std::endl;*/
 }
 
 static void assembleCurlCurlElementMatrix(apf::Mesh* mesh,apf::MeshEntity* e,
@@ -310,6 +334,10 @@ static double getLocalBLFIntegral(EdgePatch* p, apf::MeshEntity* tet)
   apf::destroyMeshElement(me);
 
   // pick edge index from the resulting vector
+  int which, rotate; bool flip;
+  apf::getAlignment(p->mesh, tet, p->entity, which, flip, rotate);
+  if (flip)
+    return -1*integrals(ei);
   return integrals(ei);
 }
 
@@ -362,7 +390,9 @@ void assembleDomainLFElementVector(apf::Mesh* mesh, apf::MeshEntity* e,
     w = weight * jdet;
 
     apf::getVectorShapeValues(el, p, vectorshape);
-    pumiUserFunction(p, val, e, apf::getMesh(f)); // eval vector function
+    apf::Vector3 global;
+    apf::mapLocalToGlobal(me, p, global);
+    pumiUserFunction(global, val, e, apf::getMesh(f)); // eval vector function
     val *= w;
 
     mth::Matrix<double> vectorShape (nd, dim);
@@ -388,6 +418,10 @@ static double getLocalLFIntegral(EdgePatch* p, apf::MeshEntity* tet)
   mth::Vector<double> elvect;
   assembleDomainLFElementVector(p->mesh, tet,
       p->equilibration->ef, elvect);
+  int which, rotate; bool flip;
+  apf::getAlignment(p->mesh, tet, p->entity, which, flip, rotate);
+  if (flip)
+    return -1*elvect(ei);
   return elvect(ei);
 }
 
@@ -504,7 +538,9 @@ static void assembleRHS(EdgePatch* p)
     p->b.resize(p->tets.size());
     p->b.zero();
   }
-  printf("RHS \n");
+  printf("RHS B\n");
+  double testlfblf = 0.0; // REMOVE
+  double testflux = 0.0; // REMOVE
   int ne = p->tets.size();
   int nf = p->faces.size();
   for (int i = 0; i < ne; i++) {
@@ -513,12 +549,17 @@ static void assembleRHS(EdgePatch* p)
     double blfIntegral = getLocalBLFIntegral(p, tet);
     double lfIntegral = getLocalLFIntegral(p, tet);
     double fluxIntegral = getFluxIntegral(p, tet);
+    testlfblf += blfIntegral - lfIntegral;
+    testflux += fluxIntegral;
     if(p->isOnBdry)
       p->b(nf+i) = blfIntegral - lfIntegral - fluxIntegral;
     else
       p->b(i) = blfIntegral - lfIntegral - fluxIntegral;
   }
   std::cout << p->b << std::endl; // REMOVE
+  std::cout << "Sum of bilinear - linear integrals " << testlfblf << std::endl; // want this to be zero
+  if (abs(testlfblf) > 0.0001 && (!p->isOnBdry)) std::cout << "nonzero" << std::endl;
+  std::cout << "Sum of flux integrals over the edge patch " << testflux << std::endl; // want this to be zero
 }
 
 // The following two functions help order tets and faces in a cavity in a
@@ -609,6 +650,26 @@ static void runErm(EdgePatch* p)
   assembleRHS(p);
   mth::solveFromQR(p->qr.Q, p->qr.R, p->b, p->x);
 
+  // REMOVE ALL BELOW
+  int ne = p->tets.size();
+  int nf = p->faces.size();
+  std::cout << " Reordered Tets, ne: " << ne << " nf " << nf << std::endl; // REMOVE
+  std::cout << " Center of Reordered tets" << std::endl;
+  for (int i = 0; i < ne; i++) {
+    EntitySet::iterator it = std::next(p->tets.begin(), i);
+    apf::MeshEntity* tet = *it;
+    apf::Vector3 center = apf::getLinearCentroid(p->mesh, tet);
+    std::cout << i << ":  " << center << std::endl;
+  }
+  std::cout << " Center of Reordered Faces" << std::endl;
+  for (int i = 0; i < nf; i++) {
+    EntitySet::iterator it = std::next(p->faces.begin(), i);
+    apf::MeshEntity* face = *it;
+    apf::Vector3 center = apf::getLinearCentroid(p->mesh, face);
+    std::cout << i << ":  " << center << std::endl;
+  }
+  std::cout << "----------------" << std::endl;
+
   std::cout << "x" << std::endl; // REMOVE
   std::cout << p->x << std::endl; // REMOVE
 }

From 3bc3e5380ea9c89b07de212dbb58f1c68ee1b4f6 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Wed, 6 May 2020 22:40:56 -0400
Subject: [PATCH 137/555] Instantiates apf::getCofactor for 2,3,4

Without any instantiation this function will not be added to the libraries when the code is compiled optimized.
---
 apf/apfMatrix.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/apf/apfMatrix.cc b/apf/apfMatrix.cc
index 75f668bc8..92d15f402 100644
--- a/apf/apfMatrix.cc
+++ b/apf/apfMatrix.cc
@@ -141,6 +141,10 @@ template Matrix<1,1> getMinor(Matrix<2,2> const& A, std::size_t i, std::size_t j
 template Matrix<2,2> getMinor(Matrix<3,3> const& A, std::size_t i, std::size_t j);
 template Matrix<3,3> getMinor(Matrix<4,4> const& A, std::size_t i, std::size_t j);
 
+template double getCofactor(Matrix<2,2> const& A, std::size_t i, std::size_t);
+template double getCofactor(Matrix<3,3> const& A, std::size_t i, std::size_t);
+template double getCofactor(Matrix<4,4> const& A, std::size_t i, std::size_t);
+
 template double getDeterminant(Matrix<2,2> const& A);
 template double getDeterminant(Matrix<3,3> const& A);
 template double getDeterminant(Matrix<4,4> const& A);

From d6e887311e7a8f96f27948f531fa618ceb4b1334 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@portal2.scorec.rpi.edu>
Date: Thu, 7 May 2020 22:30:49 -0400
Subject: [PATCH 138/555] Fixes bugs. Successfully passes edge equilib. test

Code needs a clean up.
---
 em/emResidualFunctionals.cc | 129 +++++++++++++++++++++++++++++++-----
 1 file changed, 111 insertions(+), 18 deletions(-)

diff --git a/em/emResidualFunctionals.cc b/em/emResidualFunctionals.cc
index 72c484363..c79244524 100644
--- a/em/emResidualFunctionals.cc
+++ b/em/emResidualFunctionals.cc
@@ -425,6 +425,66 @@ static double getLocalLFIntegral(EdgePatch* p, apf::MeshEntity* tet)
   return elvect(ei);
 }
 
+// Given a tet and one of its faces, the vertex of the tet
+// opposite to the face is returned.
+static apf::MeshEntity* getTetOppVert(
+    apf::Mesh* m, apf::MeshEntity* t, apf::MeshEntity* f)
+{
+  apf::Downward fvs;
+  int fnv = m->getDownward(f, 0, fvs);
+  apf::Downward tvs;
+  int tnv = m->getDownward(t, 0, tvs);
+  PCU_ALWAYS_ASSERT(tnv == 4 && fnv == 3);
+  for (int i = 0; i < tnv; i++) {
+    if (apf::findIn(fvs, fnv, tvs[i]) == -1)
+      return tvs[i];
+  }
+  return 0;
+}
+
+static apf::Vector3 computeFaceNormal(apf::Mesh* m,
+    apf::MeshEntity* f, apf::Vector3 const& p)
+{
+  // Compute face normal using face Jacobian
+  apf::MeshElement* me = apf::createMeshElement(m, f);
+  apf::Matrix3x3 J;
+  apf::getJacobian(me, p, J);
+  apf::destroyMeshElement(me);
+
+  apf::Vector3 g1 = J[0];
+  apf::Vector3 g2 = J[1];
+  apf::Vector3 n = apf::cross( g1, g2 );
+  return n.normalize();
+}
+
+static apf::Vector3 computeFaceOutwardNormal(apf::Mesh* m,
+    apf::MeshEntity* t, apf::MeshEntity* f, apf::Vector3 const& p)
+{
+  apf::Vector3 n = computeFaceNormal(m, f, p);
+
+  // Orient the normal outwards from the tet
+  apf::MeshEntity* oppVert = getTetOppVert(m, t, f);
+  apf::Vector3 vxi = apf::Vector3(0.,0.,0.);
+
+  // get global coordinates of the vertex
+  apf::Vector3 txi;
+  m->getPoint(oppVert, 0, txi);
+
+  // get global coordinates of the point on the face
+  apf::MeshElement* fme = apf::createMeshElement(m, f);
+  apf::Vector3 pxi;
+  apf::mapLocalToGlobal(fme, p, pxi);
+  apf::destroyMeshElement(fme);
+
+  apf::Vector3 pxiTotxi = txi - pxi;
+  //std::cout << "dot product " << pxiTotxi*n << std::endl;
+  if (pxiTotxi*n > 0) {
+    n = n*-1.;
+  }
+  return n;
+}
+
+
 static double getFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
 {
   double fluxIntegral = 0.0;
@@ -444,6 +504,7 @@ static double getFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
   PCU_ALWAYS_ASSERT(patchFaces.size() == 2);
   //  3. loop over the patch faces
   for (unsigned int i = 0; i < patchFaces.size(); i++) {
+    double fluxFaceIntegral = 0.0;
     // 4. get upward tets of the current face
     apf::Up up;
     apf::MeshEntity* currentFace = patchFaces[i];
@@ -460,26 +521,36 @@ static double getFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
 
     // 5. count integration points for flux face integral
     apf::FieldShape* fs = ep->equilibration->ef->getShape();
+    int order = 2 * fs->getOrder() - 2;
     apf::MeshElement* fme = apf::createMeshElement(ep->mesh, currentFace);
-    apf::Element* fel = apf::createElement(ep->equilibration->ef, fme);
-    int order = 2 * fs->getOrder();
-    int np = apf::countIntPoints(fme, order); // int points required
+    int np = apf::countIntPoints(fme, order);
+    std::cout << "np " << np << std::endl;
 
     // loop over integration points
-    apf::Vector3 p, tet1xi, tet2xi, curl1, curl2, curl, fnormal, tk, vshape;
+    apf::Vector3 p, tet1xi, tet2xi, curl1, curl2, curl,
+      fnormal1, fnormal2, tk, vshape;
     for (int n = 0; n < np; n++) {
       apf::getIntPoint(fme, order, n, p);
       double weight = apf::getIntWeight(fme, order, n);
-
-      // compute face normal
       apf::Matrix3x3 fJ;
       apf::getJacobian(fme, p, fJ);
       double jdet = apf::getJacobianDeterminant(
           fJ, apf::getDimension(ep->mesh, currentFace));
-      apf::Vector3 r1 = fJ[0];
-      apf::Vector3 r2 = fJ[1];
-      fnormal = apf::cross( r1, r2 );
-      fnormal = fnormal.normalize();
+
+      // compute face outward normals wrt tets
+      if (tet == firstTet)
+        fnormal1 = computeFaceOutwardNormal(ep->mesh, firstTet, currentFace, p);
+      else
+        fnormal1 = computeFaceOutwardNormal(ep->mesh, secondTet, currentFace, p);
+      if (up.n == 2) {
+        if (tet == firstTet)
+          fnormal2 = computeFaceOutwardNormal(ep->mesh, secondTet, currentFace, p);
+        else
+          fnormal2 = computeFaceOutwardNormal(ep->mesh, firstTet, currentFace, p);
+        std::cout << "normal1 " << fnormal1 << std::endl;
+        std::cout << "normal2 " << fnormal2 << std::endl;
+      }
+
 
       curl.zero();
       // compute curl1
@@ -487,7 +558,9 @@ static double getFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
       apf::MeshElement* me1 = apf::createMeshElement(ep->mesh, firstTet);
       apf::Element* el1 = apf::createElement(ep->equilibration->ef, me1);
       apf::getCurl(el1, tet1xi, curl1);
-      curl += curl1;
+      //curl += curl1;
+      apf::Vector3 temp1 = apf::cross(fnormal1, curl1); //
+      curl += temp1; //
       apf::destroyElement(el1);
       apf::destroyMeshElement(me1);
 
@@ -497,14 +570,17 @@ static double getFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
         apf::MeshElement* me2 = apf::createMeshElement(ep->mesh, secondTet);
         apf::Element* el2 = apf::createElement(ep->equilibration->ef, me2);
         apf::getCurl(el2, tet2xi, curl2);
-        curl += curl2;
+        //curl += curl2;
+        apf::Vector3 temp2 = apf::cross(fnormal2, curl2); //
+        curl += (temp2 * -1.); //
         apf::destroyElement(el2);
         apf::destroyMeshElement(me2);
       }
 
       // compute tk (inter-element averaged flux)
-      tk = apf::cross(fnormal, curl);
-      tk = tk * 1./2.;
+      //tk = apf::cross(fnormal, curl);
+      tk = curl * 1./2.;
+      std::cout << "tk " << tk << std::endl;
 
       // compute vector shape
       int type = ep->mesh->getType(tet);
@@ -512,16 +588,32 @@ static double getFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
       apf::NewArray<apf::Vector3> vectorshapes (nd);
       apf::MeshElement* me = apf::createMeshElement(ep->mesh, tet);
       apf::Element* el = apf::createElement(ep->equilibration->ef, me);
-      apf::getVectorShapeValues(el, p, vectorshapes);
+      apf::Vector3 tetxi = apf::boundaryToElementXi(ep->mesh, currentFace, tet, p);
+      apf::getVectorShapeValues(el, tetxi, vectorshapes);
+      std::cout << "p " << p << std::endl;
+      std::cout << "tetxi " << tetxi << std::endl;
       vshape = vectorshapes[ei];
       apf::destroyElement(el);
       apf::destroyMeshElement(me);
 
+      // negate if edge is flipped
+      int which, rotate; bool flip;
+      apf::getAlignment(ep->mesh, tet, ep->entity, which, flip, rotate);
+      if (flip) {
+        vshape = vshape * -1.;
+        std::cout << "flip " << flip << std::endl;
+        std::cout << "vshape " << vshape << std::endl;
+      }
+
+
+
+
       // compute integral
-      fluxIntegral += tk * vshape * weight * jdet;
+      fluxFaceIntegral += (tk * vshape) * weight * jdet;
     }
-    apf::destroyElement(fel);
     apf::destroyMeshElement(fme);
+    fluxIntegral += fluxFaceIntegral;
+    std::cout << "flux Face integral " << fluxFaceIntegral << std::endl; // REMOVE
   }
   return fluxIntegral;
 }
@@ -558,8 +650,9 @@ static void assembleRHS(EdgePatch* p)
   }
   std::cout << p->b << std::endl; // REMOVE
   std::cout << "Sum of bilinear - linear integrals " << testlfblf << std::endl; // want this to be zero
-  if (abs(testlfblf) > 0.0001 && (!p->isOnBdry)) std::cout << "nonzero" << std::endl;
+  if (abs(testlfblf) > 1e-08 && (!p->isOnBdry)) std::cout << "nonzero" << std::endl;
   std::cout << "Sum of flux integrals over the edge patch " << testflux << std::endl; // want this to be zero
+  if (abs(testflux) > 1e-08 && (!p->isOnBdry)) std::cout << "nonzero" << std::endl;
 }
 
 // The following two functions help order tets and faces in a cavity in a

From eda6788930977a53563faa786d7b5ce24be1399f Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Sun, 10 May 2020 11:57:42 -0400
Subject: [PATCH 139/555] Adds the flux correction functions in a file.

---
 em/CMakeLists.txt |  3 ++-
 em/em.h           | 14 +++++++++++++-
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/em/CMakeLists.txt b/em/CMakeLists.txt
index c9886e8fd..03b1a14c2 100644
--- a/em/CMakeLists.txt
+++ b/em/CMakeLists.txt
@@ -5,7 +5,8 @@ endif()
 
 # Package sources
 set(SOURCES
-  emResidualFunctionals.cc)
+  emResidualFunctionals.cc
+  emFluxCorrection.cc)
 
 # Package headers
 set(HEADERS
diff --git a/em/em.h b/em/em.h
index c8159fd8c..df3deb129 100644
--- a/em/em.h
+++ b/em/em.h
@@ -14,9 +14,21 @@
  */
 #include "apf.h"
 
+#include <mthQR.h>
+#include <mth.h>
+#include <mth_def.h>
+
 namespace em {
 
-void equilibrateResiduals(apf::Field* f);
+void assembleVectorMassElementMatrix(apf::Mesh* mesh,apf::MeshEntity* e,
+    apf::Field* f, mth::Matrix<double>& elmat);
+
+apf::Field* computeFluxCorrection(apf::Field* ef, apf::Field* g);
+
+apf::Field* equilibrateResiduals(apf::Field* f);
+
+
+
 
 
 

From e06dc7b53a3e1d7cbaf206e9ec6e31d821632b6b Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@portal2.scorec.rpi.edu>
Date: Sun, 10 May 2020 12:15:52 -0400
Subject: [PATCH 140/555] Adds Piola transformation for 2D ND vector shapes.

TODO Clean up required.
---
 apf/apf.cc | 39 +++++++++++++++++++++++++++++++++++----
 1 file changed, 35 insertions(+), 4 deletions(-)

diff --git a/apf/apf.cc b/apf/apf.cc
index 4d7de9390..043d3a0ff 100644
--- a/apf/apf.cc
+++ b/apf/apf.cc
@@ -4,7 +4,7 @@
  * This work is open source software, licensed under the terms of the
  * BSD license as described in the LICENSE file in the top-level directory.
  */
-
+#include <iostream>
 #include "apf.h"
 #include "apfScalarField.h"
 #include "apfScalarElement.h"
@@ -482,9 +482,40 @@ void getVectorShapeValues(Element* e, Vector3 const& local,
   }
   else
   {
-    // TODO when ref dim != mesh space dim. Pseudo-inverse needed.
-    PCU_ALWAYS_ASSERT_VERBOSE(false,
-    	"not yet implemented for 3D surface meshes (i.e., manifolds)!");
+    // TODO clean up
+    apf::Matrix3x3 J;
+    apf::getJacobian(e->getParent(), local, J);
+    std::cout << "J (3x3) last row zero expected" << std::endl;
+    std::cout << J << std::endl;
+    apf::Matrix3x3 JT = apf::transpose(J);
+    apf::Matrix3x3 JJT = J * JT;
+    // take inverse of JTJ and transpose it
+    apf::Matrix<2,2> jjt;
+    for (int i = 0; i < 2; i++)
+      for (int j = 0; j < 2; j++)
+        jjt[i][j] = JJT[i][j];
+    apf::Matrix<2,2> JJTinv = apf::invert(jjt);
+    apf::Matrix<2,2> JJTinvT = apf::transpose(JJTinv);
+    std::cout << "JJTinvT (2x2)" << std::endl;
+    std::cout << JJTinvT << std::endl;
+    apf::Matrix<2,3> JJTinvTJ; // JJTinvT * J
+    for( int i = 0; i < 2; i++ ) {
+      for ( int j = 0; j < 3; j++ ) {
+        JJTinvTJ[i][j] = 0.;
+        for ( int k = 0; k < 2; k++ )
+          JJTinvTJ[i][j] += JJTinvT[i][k] * J[k][j];
+      }
+    }
+    // u(x_hat) * J(x_hat)^{-1}
+    for( size_t i = 0; i < values.size(); i++ ) {
+      for ( int j = 0; j < 3; j++ ) {
+        values[i][j] = 0.;
+        for ( int k = 0; k < 2; k++ )
+          values[i][j] += vvals[i][k] * JJTinvTJ[k][j];
+      }
+    }
+    /*PCU_ALWAYS_ASSERT_VERBOSE(false,
+    	"not yet implemented for 3D surface meshes (i.e., manifolds)!");*/
   }
 }
 

From 0fba9b0e214c3ed647d8c4fd507ab7c3bd304eaf Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@portal2.scorec.rpi.edu>
Date: Sun, 10 May 2020 12:18:17 -0400
Subject: [PATCH 141/555] Adds functions to compute flux 'correction' field.

TODOs:
Negate 2D shape functions if edge is flipped.
Clean up.
---
 em/emFluxCorrection.cc      | 100 ++++++++++++++++++++++++++++++++++++
 em/emResidualFunctionals.cc |  43 ++++++++++++++--
 2 files changed, 138 insertions(+), 5 deletions(-)
 create mode 100644 em/emFluxCorrection.cc

diff --git a/em/emFluxCorrection.cc b/em/emFluxCorrection.cc
new file mode 100644
index 000000000..d83332c9f
--- /dev/null
+++ b/em/emFluxCorrection.cc
@@ -0,0 +1,100 @@
+/*
+ * Copyright (C) 2011 Scientific Computation Research Center
+ *
+ * This work is open source software, licensed under the terms of the
+ * BSD license as described in the LICENSE file in the top-level directory.
+ */
+#include <iostream>
+#include <cstdlib>
+#include <apf.h>
+#include <PCU.h>
+#include <lionPrint.h>
+
+#include "em.h"
+
+#include <apfMesh.h>
+#include <apfShape.h>
+#include <apfField.h>
+#include <apfElement.h>
+#include <apfCavityOp.h>
+#include "crv.h"
+#include "crvShape.h"
+
+#include <mthQR.h>
+#include <mth.h>
+#include <mth_def.h>
+#include <limits>
+
+#include <set>
+#include <pcu_util.h>
+
+#include "em.h"
+
+namespace em {
+
+struct QRDecomp {
+  mth::Matrix<double> Q;
+  mth::Matrix<double> R;
+};
+
+// This function takes the NedelecField and computes
+// correction vectors on faces and stores them in a field
+apf::Field* computeFluxCorrection(apf::Field* ef, apf::Field* g)
+{
+  int order = ef->getShape()->getOrder();
+  apf::Field* faceNedelecField = apf::createField(
+      apf::getMesh(g), "face_nedelec_field", apf::SCALAR, apf::getNedelec(order));
+  apf::zeroField(faceNedelecField);
+
+
+  apf::Field* correctedFluxField =  createPackedField(
+      apf::getMesh(g), "corrected_flux_field", 3, apf::getConstant(2));
+
+  // iterate over all faces of the mesh
+  apf::MeshEntity* face;
+  apf::MeshIterator* it = apf::getMesh(g)->begin(2);
+  while ((face = apf::getMesh(g)->iterate(it))) {
+    // assemble RHS vector
+    double components[3];
+    apf::getComponents(g, face, 0, components);
+    mth::Vector<double> rhs(3);
+    rhs(0) = components[0]; rhs(1) = components[1]; rhs(2) = components[2];
+
+    // assemlbe face mass matrix
+    mth::Matrix<double> M;
+    assembleVectorMassElementMatrix(
+        apf::getMesh(g), face, faceNedelecField, M);
+
+    // solve the system
+    QRDecomp qr;
+    mth::decomposeQR(M, qr.Q, qr.R);
+    std::cout << "M" << std::endl;
+    std::cout << M << std::endl;
+    std::cout << "Q" << std::endl;
+    std::cout << qr.Q << std::endl;
+    std::cout << "R" << std::endl;
+    std::cout << qr.R << std::endl;
+    std::cout << "RHS" << std::endl;
+    std::cout << rhs << std::endl;
+    
+    mth::Vector<double> theta;
+    mth::solveFromQR(qr.Q, qr.R, rhs, theta);
+    std::cout << "theta" << std::endl;
+    std::cout << theta << std::endl;
+
+    // set solution vector on face field
+    //theta.toArray(components);
+    components[0] = theta(0);
+    components[1] = theta(1);
+    components[2] = theta(2);
+    apf::setComponents(
+        correctedFluxField, face, 0, components);
+  }
+  apf::getMesh(g)->end(it);
+
+  return correctedFluxField;
+}
+
+}
+
+
diff --git a/em/emResidualFunctionals.cc b/em/emResidualFunctionals.cc
index c79244524..55acea078 100644
--- a/em/emResidualFunctionals.cc
+++ b/em/emResidualFunctionals.cc
@@ -40,14 +40,17 @@ struct Equilibration {
   int order;
   /* input scalar field containing Nedelec dofs for electric field */
   apf::Field* ef;
+  /* output field containing correction values */
+  apf::Field* g; 
 };
 
-static void setupEquilibration(Equilibration* eq, apf::Field* f)
+static void setupEquilibration(Equilibration* eq, apf::Field* f, apf::Field* g)
 {
   eq->mesh = apf::getMesh(f);
   eq->dim = eq->mesh->getDimension();
   eq->ef = f;
   eq->order = f->getShape()->getOrder();
+  eq->g = g;
 }
 
 struct QRDecomp {
@@ -250,7 +253,7 @@ static void assembleCurlCurlElementMatrix(apf::Mesh* mesh,apf::MeshEntity* e,
   apf::destroyMeshElement(me);
 }
 
-static void assembleVectorMassElementMatrix(apf::Mesh* mesh,apf::MeshEntity* e,
+void assembleVectorMassElementMatrix(apf::Mesh* mesh,apf::MeshEntity* e,
     apf::Field* f, mth::Matrix<double>& elmat)
 {
   apf::FieldShape* fs = f->getShape();
@@ -743,9 +746,27 @@ static void runErm(EdgePatch* p)
   assembleRHS(p);
   mth::solveFromQR(p->qr.Q, p->qr.R, p->b, p->x);
 
+  int nf = p->faces.size();
+  for(int i = 0; i < nf; i++) {
+    // get face entitiy
+    EntitySet::iterator it = std::next(p->faces.begin(), i);
+    apf::MeshEntity* face = *it;
+    // get ei of edge in face downward edges
+    apf::Downward e;
+    int ned = p->mesh->getDownward(face, 1, e);
+    int ei = apf::findIn(e, ned, p->entity);
+    PCU_ALWAYS_ASSERT(ned == 3 && ei != -1);
+    // save the g value at that index on that face in the g field
+    double components[3];
+    apf::getComponents(p->equilibration->g, face, 0, components);
+    components[ei] = p->x(i);
+    apf::setComponents(p->equilibration->g, face, 0, components);
+  }
+
+
   // REMOVE ALL BELOW
   int ne = p->tets.size();
-  int nf = p->faces.size();
+  //int nf = p->faces.size();
   std::cout << " Reordered Tets, ne: " << ne << " nf " << nf << std::endl; // REMOVE
   std::cout << " Center of Reordered tets" << std::endl;
   for (int i = 0; i < ne; i++) {
@@ -791,12 +812,24 @@ class EdgePatchOp : public apf::CavityOp
   EdgePatch edgePatch;
 };
 
-void equilibrateResiduals(apf::Field* f)
+apf::Field* equilibrateResiduals(apf::Field* f)
 {
+  // initialize the field 'g' with zeros
+  apf::Field* g = createPackedField( apf::getMesh(f), "g", 3, apf::getConstant(2) );
+  double zeros[3] = {0., 0., 0.};
+  apf::MeshEntity* face;
+  apf::MeshIterator* it = apf::getMesh(f)->begin(2);
+  while ((face = apf::getMesh(f)->iterate(it))) {
+    apf::setComponents(g, face, 0, zeros);
+  }
+  apf::getMesh(f)->end(it);
+
+
   Equilibration equilibration;
-  setupEquilibration(&equilibration, f);
+  setupEquilibration(&equilibration, f, g);
   EdgePatchOp op(&equilibration);
   op.applyToDimension(1); // edges
+  return g;
 }
 
 

From b9de71a9bdcf600f6d5b5aa16a01b68e392c7204 Mon Sep 17 00:00:00 2001
From: Morteza HS <hakimi1364@gmail.com>
Date: Sun, 10 May 2020 12:39:21 -0400
Subject: [PATCH 142/555] Changes python from python 2 to general version

---
 cmake/FindMPI4PY.cmake         | 4 ++--
 python_wrappers/CMakeLists.txt | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/cmake/FindMPI4PY.cmake b/cmake/FindMPI4PY.cmake
index fcd16d689..f30da590d 100644
--- a/cmake/FindMPI4PY.cmake
+++ b/cmake/FindMPI4PY.cmake
@@ -9,7 +9,7 @@
 
 if(NOT MPI4PY_INCLUDE_DIR)
     execute_process(COMMAND
-      "${Python2_EXECUTABLE}" "-c" "import mpi4py; print(mpi4py.get_include())"
+      "${Python_EXECUTABLE}" "-c" "import mpi4py; print(mpi4py.get_include())"
       OUTPUT_VARIABLE MPI4PY_INCLUDE_DIR
       RESULT_VARIABLE MPI4PY_COMMAND_RESULT
       OUTPUT_STRIP_TRAILING_WHITESPACE)
@@ -33,7 +33,7 @@ endif(NOT MPI4PY_INCLUDE_DIR)
 
 if(MPI4PY_FOUND)
 	execute_process(COMMAND
-	        "${Python2_EXECUTABLE}" "-c" "from mpi4py import MPI; print(MPI.get_vendor())"
+	        "${Python_EXECUTABLE}" "-c" "from mpi4py import MPI; print(MPI.get_vendor())"
 		OUTPUT_VARIABLE MPI4PY_VENDOR
 		OUTPUT_STRIP_TRAILING_WHITESPACE)
 endif()
diff --git a/python_wrappers/CMakeLists.txt b/python_wrappers/CMakeLists.txt
index 0c80759a6..074b0f388 100644
--- a/python_wrappers/CMakeLists.txt
+++ b/python_wrappers/CMakeLists.txt
@@ -4,7 +4,7 @@
 # TODO: MPI4PY mpich build has to be the same as the one scorec is
 # built with. That is if OPENMPI is being used for building scorec,
 # an MPI4PY build with OPENMPI must be used.
-find_package(Python2 REQUIRED COMPONENTS Interpreter Development)
+find_package(Python REQUIRED COMPONENTS Interpreter Development)
 find_package(MPI4PY REQUIRED)
 message(STATUS "MPI4PY_VENDOR is ${MPI4PY_VENDOR}")
 include_directories(${MPI4PY_INCLUDE_DIR})
@@ -47,7 +47,7 @@ set_property(TARGET pyCore
 )
 
 set(pyCoreDepLibs
-  Python2::Python
+  Python::Python
   core)
 
 if(ENABLE_SIMMETRIX)

From 399d54ad6b2b95a5c070a346ff85a555846c9532 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@portal2.scorec.rpi.edu>
Date: Sun, 10 May 2020 12:40:48 -0400
Subject: [PATCH 143/555] Fixes bugs.

---
 em/em.h                     | 2 --
 em/emResidualFunctionals.cc | 8 ++++----
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/em/em.h b/em/em.h
index df3deb129..c119887ad 100644
--- a/em/em.h
+++ b/em/em.h
@@ -20,8 +20,6 @@
 
 namespace em {
 
-void assembleVectorMassElementMatrix(apf::Mesh* mesh,apf::MeshEntity* e,
-    apf::Field* f, mth::Matrix<double>& elmat);
 
 apf::Field* computeFluxCorrection(apf::Field* ef, apf::Field* g);
 
diff --git a/em/emResidualFunctionals.cc b/em/emResidualFunctionals.cc
index 55acea078..582fe05a5 100644
--- a/em/emResidualFunctionals.cc
+++ b/em/emResidualFunctionals.cc
@@ -253,7 +253,7 @@ static void assembleCurlCurlElementMatrix(apf::Mesh* mesh,apf::MeshEntity* e,
   apf::destroyMeshElement(me);
 }
 
-void assembleVectorMassElementMatrix(apf::Mesh* mesh,apf::MeshEntity* e,
+static void assembleVectorMassElementMatrix(apf::Mesh* mesh,apf::MeshEntity* e,
     apf::Field* f, mth::Matrix<double>& elmat)
 {
   apf::FieldShape* fs = f->getShape();
@@ -282,9 +282,9 @@ void assembleVectorMassElementMatrix(apf::Mesh* mesh,apf::MeshEntity* e,
 
     apf::getVectorShapeValues(el, p, vectorshape);
     mth::Matrix<double> vectorShape (nd, dim);
-    for (int i = 0; i < nd; i++)
-      for (int j = 0; j < dim; j++)
-        vectorShape(i,j) = vectorshape[i][j];
+    for (int j = 0; j < nd; j++)
+      for (int k = 0; k < dim; k++)
+        vectorShape(j,k) = vectorshape[j][k];
 
     mth::Matrix<double> vectorShapeT (dim, nd);
     mth::transpose(vectorShape, vectorShapeT);

From c90015ac54d5457e10e9de1cf13a748fc6366111 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@portal2.scorec.rpi.edu>
Date: Sun, 10 May 2020 12:43:12 -0400
Subject: [PATCH 144/555] Adds computeFaceMassMatrix and fixes bug.

---
 em/emFluxCorrection.cc | 62 +++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 61 insertions(+), 1 deletion(-)

diff --git a/em/emFluxCorrection.cc b/em/emFluxCorrection.cc
index d83332c9f..c1dfbff47 100644
--- a/em/emFluxCorrection.cc
+++ b/em/emFluxCorrection.cc
@@ -37,6 +37,66 @@ struct QRDecomp {
   mth::Matrix<double> R;
 };
 
+static void assembleFaceMassMatrix(apf::Mesh* mesh,apf::MeshEntity* e,
+    apf::Field* f, mth::Matrix<double>& elmat)
+{
+  apf::FieldShape* fs = f->getShape();
+  int type = mesh->getType(e);
+  int nd = apf::countElementNodes(fs, type);
+  int dim = apf::getDimension(mesh, e);
+  double w;
+
+  apf::NewArray<apf::Vector3> vectorshape(nd);
+  elmat.resize(nd,nd);
+
+  apf::MeshElement* me = apf::createMeshElement(mesh, e);
+  apf::Element* el = apf::createElement(f, me);
+  int order = 2 * fs->getOrder();
+  int np = apf::countIntPoints(me, order); // int points required
+
+  apf::Downward edges;
+  int ned =  mesh->getDownward(e, 1, edges);
+  PCU_ALWAYS_ASSERT(ned == 3);
+  int which, rotate; bool flip;
+
+  elmat.zero();
+  apf::Vector3 p;
+  for (int i = 0; i < np; i++) {
+    apf::getIntPoint(me, order, i, p);
+    double weight = apf::getIntWeight(me, order, i);
+    apf::Matrix3x3 J;
+    apf::getJacobian(me, p, J);
+    double jdet = apf::getJacobianDeterminant(J, dim);
+    w = weight * jdet;
+
+    apf::getVectorShapeValues(el, p, vectorshape);
+    mth::Matrix<double> vectorShape (nd, dim);
+    for (int j = 0; j < nd; j++)
+      for (int k = 0; k < dim; k++)
+        vectorShape(j,k) = vectorshape[j][k];
+
+    // negate negatve dof indices
+    for (int ei = 0; ei < ned; ei++) {
+      apf::getAlignment(mesh, e, edges[ei], which, flip, rotate);
+      if(flip) {
+        for (int j = 0; j < dim; j++)
+          vectorShape(ei, j) = -1*vectorShape(ei, j);
+      }
+    }
+
+    mth::Matrix<double> vectorShapeT (dim, nd);
+    mth::transpose(vectorShape, vectorShapeT);
+    mth::Matrix<double> M (nd,nd);
+    M.zero();
+    mth::multiply(vectorShape, vectorShapeT, M);
+    M *= w;
+    elmat += M;
+  }
+  apf::destroyElement(el);
+  apf::destroyMeshElement(me);
+}
+
+
 // This function takes the NedelecField and computes
 // correction vectors on faces and stores them in a field
 apf::Field* computeFluxCorrection(apf::Field* ef, apf::Field* g)
@@ -62,7 +122,7 @@ apf::Field* computeFluxCorrection(apf::Field* ef, apf::Field* g)
 
     // assemlbe face mass matrix
     mth::Matrix<double> M;
-    assembleVectorMassElementMatrix(
+    assembleFaceMassMatrix(
         apf::getMesh(g), face, faceNedelecField, M);
 
     // solve the system

From a6c120355008b775d34b6d8b33819511af04b2dc Mon Sep 17 00:00:00 2001
From: Morteza HS <hakimi1364@gmail.com>
Date: Sun, 10 May 2020 22:24:17 -0400
Subject: [PATCH 145/555] Fixes out of range issue issue with array

Newer compilers complain about this when using -DSCOREC_CXX_WARNINGS=ON
---
 ma/maSnap.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ma/maSnap.cc b/ma/maSnap.cc
index 1e92b4231..fd9b1f47b 100644
--- a/ma/maSnap.cc
+++ b/ma/maSnap.cc
@@ -66,7 +66,7 @@ static size_t isSurfUnderlyingFaceDegenerate(
     param[periodicAxes] = candidatePeriodicParam;
     param[degenAxes] = candidateDegenParam;
     Vector p(param[0], param[1], 0.0);
-    m->getFirstDerivative(g, param, uTan, vTan);
+    m->getFirstDerivative(g, p, uTan, vTan);
     double uTanSize = uTan.getLength();
     double vTanSize = vTan.getLength();
 #ifdef HAVE_CAPSTONE

From 465b21e9e8d5ea03ab062f883cb6c7192723c2ed Mon Sep 17 00:00:00 2001
From: Morteza HS <hakimi1364@gmail.com>
Date: Sun, 10 May 2020 22:48:51 -0400
Subject: [PATCH 146/555] Moves 1d poly basis in apfNedelec to a new file

This to use them in L2_Basis as well.
---
 apf/CMakeLists.txt    |   1 +
 apf/apfNedelec.cc     | 171 +--------------------------------------
 apf/apfPolyBasis1D.cc | 184 ++++++++++++++++++++++++++++++++++++++++++
 apf/apfPolyBasis1D.h  |  52 ++++++++++++
 4 files changed, 238 insertions(+), 170 deletions(-)
 create mode 100644 apf/apfPolyBasis1D.cc
 create mode 100644 apf/apfPolyBasis1D.h

diff --git a/apf/CMakeLists.txt b/apf/CMakeLists.txt
index 43484f1c7..588baac83 100644
--- a/apf/CMakeLists.txt
+++ b/apf/CMakeLists.txt
@@ -25,6 +25,7 @@ set(SOURCES
   apfShape.cc
   apfIPShape.cc
   apfHierarchic.cc
+	apfPolyBasis1D.cc
   apfNedelec.cc
   apfVector.cc
   apfVectorElement.cc
diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index 574c800d5..44551036d 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -10,6 +10,7 @@
 #include "apfFieldOf.h"
 #include "apfElement.h"
 #include "apfVectorElement.h"
+#include "apfPolyBasis1D.h"
 #include <mth.h>
 #include <mth_def.h>
 #include <mthQR.h>
@@ -180,176 +181,6 @@ static void alignFaceNodes(
 
 static unsigned const MAX_ND_ORDER = 10;
 
-enum {
-  GAUSS_LEGENDRE,
-  GAUSS_LOBATTO
-};
-
-void getGaussLegendrePoints(int np, double* pts)
-{
-  switch (np)
-  {
-    case 1:
-      pts[0] = 0.5;
-      return;
-    case 2:
-      pts[0] = 0.21132486540518711775;
-      pts[1] = 0.78867513459481288225;
-      return;
-    case 3:
-      pts[0] = 0.11270166537925831148;
-      pts[1] = 0.5;
-      pts[2] = 0.88729833462074168852;
-      return;
-  }
-
-  const int n = np;
-  const int m = (n+1)/2;
-
-  for (int i = 1; i <= m; i++)
-  {
-     double z = cos(M_PI * (i - 0.25) / (n + 0.5));
-     double pp, p1, dz, xi = 0.;
-     bool done = false;
-     while (1)
-     {
-        double p2 = 1;
-        p1 = z;
-        for (int j = 2; j <= n; j++)
-        {
-           double p3 = p2;
-           p2 = p1;
-           p1 = ((2 * j - 1) * z * p2 - (j - 1) * p3) / j;
-        }
-        // p1 is Legendre polynomial
-        pp = n * (z*p1-p2) / (z*z - 1);
-        if (done) { break; }
-        dz = p1/pp;
-        if (fabs(dz) < 1e-16)
-        {
-           done = true;
-           // map the new point (z-dz) to (0,1):
-           xi = ((1 - z) + dz)/2; // (1 - (z - dz))/2 has bad round-off
-           // continue the computation: get pp at the new point, then exit
-        }
-        // update: z = z - dz
-        z -= dz;
-     }
-     pts[i-1] = xi;
-     pts[n-i] = 1 - xi;
-  }
-}
-
-void getGaussLobattoPoints(int /*np*/, double* /*pts*/)
-{ /* implement Gauss Lobatto points. Later when needed. */
-};
-
-static void getPoints(
-    int order,
-    apf::NewArray<double>& points,
-    int type)
-{
-  int np = order + 1;
-  points.allocated() ? points.resize(np) : points.allocate(np);
-  switch (type)
-  {
-    case GAUSS_LEGENDRE:
-    {
-      getGaussLegendrePoints(np, &points[0]);
-      break;
-    }
-    case GAUSS_LOBATTO:
-    {
-      getGaussLobattoPoints(np, &points[0]);
-      break;
-    }
-    default:
-      PCU_ALWAYS_ASSERT_VERBOSE(false,
-      	  "type should be either GAUSS_LEGENDRE or GAUSS_LOBATTO!");
-  }
-}
-
-void getOpenPoints(
-    int order,
-    apf::NewArray<double>& op,
-    int type = GAUSS_LEGENDRE)
-{
-  getPoints(order, op, type);
-}
-
-void getClosedPoints(
-    int order,
-    apf::NewArray<double>& cp,
-    int type = GAUSS_LOBATTO)
-{
-    getPoints(order, cp, type);
-}
-
-void getChebyshevT(int order, double xi, double* u)
-{
-  // recursive definition, z in [-1,1]
-  // T_0(z) = 1,  T_1(z) = z
-  // T_{n+1}(z) = 2*z*T_n(z) - T_{n-1}(z)
-  double z;
-  u[0] = 1.;
-  if (order == 0) { return; }
-  u[1] = z = 2.*xi - 1.;
-  for (int n = 1; n < order; n++)
-  {
-     u[n+1] = 2*z*u[n] - u[n-1];
-  }
-}
-
-void getChebyshevT(int order, double xi, double* u, double* d)
-{
-  // recursive definition, z in [-1,1]
-  // T_0(z) = 1,  T_1(z) = z
-  // T_{n+1}(z) = 2*z*T_n(z) - T_{n-1}(z)
-  // T'_n(z) = n*U_{n-1}(z)
-  // U_0(z) = 1  U_1(z) = 2*z
-  // U_{n+1}(z) = 2*z*U_n(z) - U_{n-1}(z)
-  // U_n(z) = z*U_{n-1}(z) + T_n(z) = z*T'_n(z)/n + T_n(z)
-  // T'_{n+1}(z) = (n + 1)*(z*T'_n(z)/n + T_n(z))
-  double z;
-  u[0] = 1.;
-  d[0] = 0.;
-  if (order == 0) { return; }
-  u[1] = z = 2.*xi - 1.;
-  d[1] = 2.;
-  for (int n = 1; n < order; n++)
-  {
-     u[n+1] = 2*z*u[n] - u[n-1];
-     d[n+1] = (n + 1)*(z*d[n]/n + 2*u[n]);
-  }
-}
-
-void getChebyshevT(int order, double xi, double* u, double* d, double* dd)
-{
-  // recursive definition, z in [-1,1]
-  // T_0(z) = 1,  T_1(z) = z
-  // T_{n+1}(z) = 2*z*T_n(z) - T_{n-1}(z)
-  // T'_n(z) = n*U_{n-1}(z)
-  // U_0(z) = 1  U_1(z) = 2*z
-  // U_{n+1}(z) = 2*z*U_n(z) - U_{n-1}(z)
-  // U_n(z) = z*U_{n-1}(z) + T_n(z) = z*T'_n(z)/n + T_n(z)
-  // T'_{n+1}(z) = (n + 1)*(z*T'_n(z)/n + T_n(z))
-  // T''_{n+1}(z) = (n + 1)*(2*(n + 1)*T'_n(z) + z*T''_n(z)) / n
-  double z;
-  u[0] = 1.;
-  d[0] = 0.;
-  dd[0]= 0.;
-  if (order == 0) { return; }
-  u[1] = z = 2.*xi - 1.;
-  d[1] = 2.;
-  dd[1] = 0;
-  for (int n = 1; n < order; n++)
-  {
-     u[n+1] = 2*z*u[n] - u[n-1];
-     d[n+1] = (n + 1)*(z*d[n]/n + 2*u[n]);
-     dd[n+1] = (n + 1)*(2.*(n + 1)*d[n] + z*dd[n])/n;
-  }
-}
-
 // This is all nodes, including the nodes associated with bounding edges
 static inline int countTriNodes(int P)
 {
diff --git a/apf/apfPolyBasis1D.cc b/apf/apfPolyBasis1D.cc
new file mode 100644
index 000000000..6f0e4b4c1
--- /dev/null
+++ b/apf/apfPolyBasis1D.cc
@@ -0,0 +1,184 @@
+/*
+ * Copyright 2011 Scientific Computation Research Center
+ *
+ * This work is open source software, licensed under the terms of the
+ * BSD license as described in the LICENSE file in the top-level directory.
+ */
+
+#include "apfPolyBasis1D.h"
+
+#include <mth.h>
+#include <mth_def.h>
+#include <mthQR.h>
+#include <pcu_util.h>
+#include <PCU.h>
+
+namespace apf {
+
+static void getPoints(
+    int order,
+    apf::NewArray<double>& points,
+    int type)
+{
+  int np = order + 1;
+  points.allocated() ? points.resize(np) : points.allocate(np);
+  switch (type)
+  {
+    case GAUSS_LEGENDRE:
+    {
+      getGaussLegendrePoints(np, &points[0]);
+      break;
+    }
+    case GAUSS_LOBATTO:
+    {
+      getGaussLobattoPoints(np, &points[0]);
+      break;
+    }
+    default:
+      PCU_ALWAYS_ASSERT_VERBOSE(false,
+      	  "type should be either GAUSS_LEGENDRE or GAUSS_LOBATTO!");
+  }
+}
+
+void getGaussLegendrePoints(int np, double* pts)
+{
+  switch (np)
+  {
+    case 1:
+      pts[0] = 0.5;
+      return;
+    case 2:
+      pts[0] = 0.21132486540518711775;
+      pts[1] = 0.78867513459481288225;
+      return;
+    case 3:
+      pts[0] = 0.11270166537925831148;
+      pts[1] = 0.5;
+      pts[2] = 0.88729833462074168852;
+      return;
+  }
+
+  const int n = np;
+  const int m = (n+1)/2;
+
+  for (int i = 1; i <= m; i++)
+  {
+     double z = cos(M_PI * (i - 0.25) / (n + 0.5));
+     double pp, p1, dz, xi = 0.;
+     bool done = false;
+     while (1)
+     {
+        double p2 = 1;
+        p1 = z;
+        for (int j = 2; j <= n; j++)
+        {
+           double p3 = p2;
+           p2 = p1;
+           p1 = ((2 * j - 1) * z * p2 - (j - 1) * p3) / j;
+        }
+        // p1 is Legendre polynomial
+        pp = n * (z*p1-p2) / (z*z - 1);
+        if (done) { break; }
+        dz = p1/pp;
+        if (fabs(dz) < 1e-16)
+        {
+           done = true;
+           // map the new point (z-dz) to (0,1):
+           xi = ((1 - z) + dz)/2; // (1 - (z - dz))/2 has bad round-off
+           // continue the computation: get pp at the new point, then exit
+        }
+        // update: z = z - dz
+        z -= dz;
+     }
+     pts[i-1] = xi;
+     pts[n-i] = 1 - xi;
+  }
+}
+
+void getGaussLobattoPoints(int /*np*/, double* /*pts*/)
+{ /* implement Gauss Lobatto points. Later when needed. */
+};
+
+
+void getOpenPoints(
+    int order,
+    apf::NewArray<double>& op,
+    int type)
+{
+  getPoints(order, op, type);
+}
+
+void getClosedPoints(
+    int order,
+    apf::NewArray<double>& cp,
+    int type)
+{
+    getPoints(order, cp, type);
+}
+
+void getChebyshevT(int order, double xi, double* u)
+{
+  // recursive definition, z in [-1,1]
+  // T_0(z) = 1,  T_1(z) = z
+  // T_{n+1}(z) = 2*z*T_n(z) - T_{n-1}(z)
+  double z;
+  u[0] = 1.;
+  if (order == 0) { return; }
+  u[1] = z = 2.*xi - 1.;
+  for (int n = 1; n < order; n++)
+  {
+     u[n+1] = 2*z*u[n] - u[n-1];
+  }
+}
+
+void getChebyshevT(int order, double xi, double* u, double* d)
+{
+  // recursive definition, z in [-1,1]
+  // T_0(z) = 1,  T_1(z) = z
+  // T_{n+1}(z) = 2*z*T_n(z) - T_{n-1}(z)
+  // T'_n(z) = n*U_{n-1}(z)
+  // U_0(z) = 1  U_1(z) = 2*z
+  // U_{n+1}(z) = 2*z*U_n(z) - U_{n-1}(z)
+  // U_n(z) = z*U_{n-1}(z) + T_n(z) = z*T'_n(z)/n + T_n(z)
+  // T'_{n+1}(z) = (n + 1)*(z*T'_n(z)/n + T_n(z))
+  double z;
+  u[0] = 1.;
+  d[0] = 0.;
+  if (order == 0) { return; }
+  u[1] = z = 2.*xi - 1.;
+  d[1] = 2.;
+  for (int n = 1; n < order; n++)
+  {
+     u[n+1] = 2*z*u[n] - u[n-1];
+     d[n+1] = (n + 1)*(z*d[n]/n + 2*u[n]);
+  }
+}
+
+void getChebyshevT(int order, double xi, double* u, double* d, double* dd)
+{
+  // recursive definition, z in [-1,1]
+  // T_0(z) = 1,  T_1(z) = z
+  // T_{n+1}(z) = 2*z*T_n(z) - T_{n-1}(z)
+  // T'_n(z) = n*U_{n-1}(z)
+  // U_0(z) = 1  U_1(z) = 2*z
+  // U_{n+1}(z) = 2*z*U_n(z) - U_{n-1}(z)
+  // U_n(z) = z*U_{n-1}(z) + T_n(z) = z*T'_n(z)/n + T_n(z)
+  // T'_{n+1}(z) = (n + 1)*(z*T'_n(z)/n + T_n(z))
+  // T''_{n+1}(z) = (n + 1)*(2*(n + 1)*T'_n(z) + z*T''_n(z)) / n
+  double z;
+  u[0] = 1.;
+  d[0] = 0.;
+  dd[0]= 0.;
+  if (order == 0) { return; }
+  u[1] = z = 2.*xi - 1.;
+  d[1] = 2.;
+  dd[1] = 0;
+  for (int n = 1; n < order; n++)
+  {
+     u[n+1] = 2*z*u[n] - u[n-1];
+     d[n+1] = (n + 1)*(z*d[n]/n + 2*u[n]);
+     dd[n+1] = (n + 1)*(2.*(n + 1)*d[n] + z*dd[n])/n;
+  }
+}
+
+}
diff --git a/apf/apfPolyBasis1D.h b/apf/apfPolyBasis1D.h
new file mode 100644
index 000000000..ea123b913
--- /dev/null
+++ b/apf/apfPolyBasis1D.h
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2011 Scientific Computation Research Center
+ *
+ * This work is open source software, incensed under the terms of the
+ * BSD license as described in the LICENSE file in the top-level directroy.
+ */
+
+#ifndef APFPOLYBASES1D_H
+#define APFPOLYBASES1D_H
+
+#include "apfNew.h"
+
+namespace apf {
+
+/** \brief Gauss point types */
+enum Gauss{
+  GAUSS_LEGENDRE,
+  GAUSS_LOBATTO,
+  GAUSS
+};
+
+
+/** \brief Gauss Legendre points */
+void getGaussLegendrePoints(int np, double* pts);
+
+/** \brief Gauss Lobatto points */
+void getGaussLobattoPoints(int, double*);
+
+/** \brief get open points for a given order and Gauss point type */
+void getOpenPoints(
+    int order,
+    NewArray<double>& op,
+    int type = GAUSS_LEGENDRE);
+
+/** \brief get closed points for a given order and Gauss point type */
+void getClosedPoints(
+    int order,
+    NewArray<double>& cp,
+    int type = GAUSS_LOBATTO);
+
+/** \brief Chebyshev polynomials of the first kind */
+void getChebyshevT(int order, double xi, double* u);
+
+/** \brief Chebyshev polynomials of the first kind and 1st derivative */
+void getChebyshevT(int order, double xi, double* u, double* d);
+
+/** \brief Chebyshev polynomials of the first kind and 1st and 2nd derivative */
+void getChebyshevT(int order, double xi, double* u, double* d, double* dd);
+
+}
+
+#endif

From 8ab3ab3fc4c012219b8a6e4eaed33545c58e0a64 Mon Sep 17 00:00:00 2001
From: Morteza HS <hakimi1364@gmail.com>
Date: Mon, 11 May 2020 00:29:48 -0400
Subject: [PATCH 147/555] Implements L2 (discontinuous) field shapes

---
 apf/CMakeLists.txt |   1 +
 apf/apfL2Shapes.cc | 468 +++++++++++++++++++++++++++++++++++++++++++++
 apf/apfShape.h     |   5 +
 3 files changed, 474 insertions(+)
 create mode 100644 apf/apfL2Shapes.cc

diff --git a/apf/CMakeLists.txt b/apf/CMakeLists.txt
index 588baac83..a83dd71b8 100644
--- a/apf/CMakeLists.txt
+++ b/apf/CMakeLists.txt
@@ -27,6 +27,7 @@ set(SOURCES
   apfHierarchic.cc
 	apfPolyBasis1D.cc
   apfNedelec.cc
+	apfL2Shapes.cc
   apfVector.cc
   apfVectorElement.cc
   apfVectorField.cc
diff --git a/apf/apfL2Shapes.cc b/apf/apfL2Shapes.cc
new file mode 100644
index 000000000..c89a09cc8
--- /dev/null
+++ b/apf/apfL2Shapes.cc
@@ -0,0 +1,468 @@
+/*
+ * Copyright 2011 Scientific Computation Research Center
+ *
+ * This work is open source software, licensed under the terms of the
+ * BSD license as described in the LICENSE file in the top-level directory.
+ */
+
+#include "apfShape.h"
+#include "apfMesh.h"
+#include "apfFieldOf.h"
+#include "apfElement.h"
+#include "apfVectorElement.h"
+#include "apfPolyBasis1D.h"
+#include <mth.h>
+#include <mth_def.h>
+#include <mthQR.h>
+#include <pcu_util.h>
+#include <PCU.h>
+
+#include <iostream>
+using namespace std;
+
+namespace apf {
+
+static unsigned const MAX_ND_ORDER = 10;
+
+// For L2Shapes there are only internal nodes
+static inline int countTriNodes(int P)
+{
+  return (P+1)*(P+2)/2;
+}
+// For L2Shapes there are only internal nodes
+static inline int countTetNodes(int P)
+{
+  return (P+1)*(P+2)*(P+3)/6;
+}
+
+static void computeTriangleTi(
+    int P, /*order*/
+    mth::Matrix<double>& Q, /*Q in QR factorization of Ti*/
+    mth::Matrix<double>& R) /*R in QR factorization of Ti*/
+{
+  int non = countTriNodes(P);
+
+  apf::NewArray<double> op;
+  getOpenPoints(P, op);
+
+
+  const int p = P;
+  apf::NewArray<double> shape_x(p+1);
+  apf::NewArray<double> shape_y(p+1);
+  apf::NewArray<double> shape_l(p+1);
+
+  apf::DynamicArray<apf::Vector3> nodes (non);
+
+  int o = 0;
+  for (int j = 0; j <= p; j++) {
+    for (int i = 0; i + j <= p; i++)
+      {
+        double w = op[i] + op[j] + op[p-i-j];
+        nodes[o][0] = op[i]/w;  nodes[o][1] = op[j]/w;  nodes[o][2] = 0.;
+        o++;
+      }
+  }
+
+  // Populate T
+  mth::Matrix<double> T(non,non);
+  for (int m = 0; m < non; m++)
+  {
+    o = 0;
+
+    double x = nodes[m][0]; double y = nodes[m][1];
+
+    getChebyshevT(p, x, &shape_x[0]);
+    getChebyshevT(p, y, &shape_y[0]);
+    getChebyshevT(p, 1. - x - y, &shape_l[0]);
+
+    for (int j = 0; j <= p; j++)
+      for (int i = 0; i + j <= p; i++)
+      	T(o++, m) = shape_x[i]*shape_y[j]*shape_l[p-i-j];
+  }
+  mth::decomposeQR(T, Q, R);
+}
+
+static void computeTetTi(
+    int P, /*order*/
+    mth::Matrix<double>& Q, /*Q in QR factorization of Ti*/
+    mth::Matrix<double>& R) /*R in QR factorization of Ti*/
+{
+  int non = countTetNodes(P);
+
+  apf::NewArray<double> op;
+  getOpenPoints(P, op);
+
+  const int p = P;
+
+  apf::NewArray<double> shape_x(p+1);
+  apf::NewArray<double> shape_y(p+1);
+  apf::NewArray<double> shape_z(p+1);
+  apf::NewArray<double> shape_l(p+1);
+
+  apf::DynamicArray<apf::Vector3> nodes (non);
+  nodes.setSize(non);
+
+  int o = 0;
+  // Region loops to get nodes and dof2tk for regions
+  for (int k = 0; k <= p; k++) {
+    for (int j = 0; j + k <= p; j++) {
+      for (int i = 0; i + j + k <= p; i++) {
+        double w = op[i] + op[j] + op[k] + op[p-i-j-k];
+        nodes[o][0] = op[i]/w;  nodes[o][1] = op[j]/w;  nodes[o][2] = op[k]/w;
+        o++;
+      }
+    }
+  }
+
+  // Populate T
+  mth::Matrix<double> T(non,non);
+  for (int m = 0; m < non; m++)
+  {
+    o = 0;
+    double x = nodes[m][0]; double y = nodes[m][1]; double z = nodes[m][2];
+
+    getChebyshevT(p, x, &shape_x[0]);
+    getChebyshevT(p, y, &shape_y[0]);
+    getChebyshevT(p, z, &shape_z[0]);
+    getChebyshevT(p, 1. - x - y - z, &shape_l[0]);
+
+    for (int k = 0; k <= p; k++)
+      for (int j = 0; j + k <= p; j++)
+        for (int i = 0; i + j + k <= p; i++)
+          T(o++, m) = shape_x[i]*shape_y[j]*shape_z[k]*shape_l[p-i-j-k];
+	}
+  mth::decomposeQR(T, Q, R);
+}
+
+static void getTi(
+    int P,
+    int type,
+    mth::Matrix<double>& Q,
+    mth::Matrix<double>& R)
+{
+
+  bool cond = (type == apf::Mesh::TRIANGLE || type == apf::Mesh::TET);
+  PCU_ALWAYS_ASSERT_VERBOSE(cond,
+      "type should be either apf::Mesh::TRIANGLE or apf::Mesh::TET!");
+
+  static apf::NewArray<double> transformQ[apf::Mesh::TYPES][MAX_ND_ORDER+1];
+  static apf::NewArray<double> transformR[apf::Mesh::TYPES][MAX_ND_ORDER+1];
+  int n = type == apf::Mesh::TRIANGLE ? countTriNodes(P) : countTetNodes(P);
+
+  // get the transform matrices if the are not already computed
+  if (!transformQ[type][P].allocated()) {
+    mth::Matrix<double> LQ(n,n);
+    mth::Matrix<double> LR(n,n);
+    type == apf::Mesh::TRIANGLE ?
+    	  computeTriangleTi(P, LQ, LR) : computeTetTi(P, LQ, LR);
+
+    transformQ[type][P].allocate(n*n);
+    transformR[type][P].allocate(n*n);
+
+    for (int i = 0; i < n; i++) {
+      for (int j = 0; j < n; j++) {
+	      transformQ[type][P][i*n+j] = LQ(i,j);
+      	transformR[type][P][i*n+j] = LR(i,j);
+      }
+    }
+  }
+
+  // set Q and R using transformQ and transformR
+  for (int i = 0; i < n; i++) {
+    for (int j = 0; j < n; j++) {
+      Q(i,j) = transformQ[type][P][i*n+j];
+      R(i,j) = transformR[type][P][i*n+j];
+    }
+  }
+}
+
+template<int P>
+class L2ShapeTri: public FieldShape {
+  public:
+    const char* getName() const { return "L2ShapeTri"; }
+    bool isVectorShape() {return false;}
+    L2ShapeTri()
+    {}
+    class Triangle : public apf::EntityShape
+    {
+    public:
+      int getOrder() {return P;}
+      void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const& xi, apf::NewArray<double>& shapes) const
+      {
+        const int p = P;
+
+      	apf::NewArray<double> shape_x(p+1);
+      	apf::NewArray<double> shape_y(p+1);
+      	apf::NewArray<double> shape_l(p+1);
+
+        int dof = countNodes();
+        mth::Matrix<double> u(dof, 1);
+
+        double x = xi[0]; double y = xi[1];
+
+        getChebyshevT(p, x, &shape_x[0]);
+        getChebyshevT(p, y, &shape_y[0]);
+        getChebyshevT(p, 1. - x - y, &shape_l[0]);
+
+        int n = 0;
+        for (int j = 0; j <= p; j++)
+          for (int i = 0; i + j <= p; i++)
+            u(n++, 0) = shape_x[i]*shape_y[j]*shape_l[p-i-j];
+
+	      mth::Matrix<double> Q(dof, dof);
+      	mth::Matrix<double> R(dof, dof);
+      	getTi(P, apf::Mesh::TRIANGLE, Q, R);
+
+        mth::Matrix<double> S(dof, 1);
+      	for(int i = 0; i < 1; i++) // S = Ti * u
+        {
+          mth::Vector<double> B (dof);
+          mth::Vector<double> X (dof);
+          for (int j = 0; j < dof; j++) B[j] = u(j,i); // populate b in QR x = b
+          mth::solveFromQR(Q, R, B, X);
+          for (int j = 0; j < dof; j++)  S(j,i) = X[j]; // populate S with x
+        }
+
+        shapes.allocate(dof);
+        for (int i = 0; i < dof; i++) // populate y
+        {
+      	  shapes[i] = S(i,0);
+        }
+      }
+      void getLocalGradients(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
+      {
+      	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getLocalGradients not \
+      	    implemented for L2ShapeTri. Aborting()!");
+      }
+      int countNodes() const {return countTriNodes(P);}
+      void getVectorValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
+      {
+      	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getVectorValues not implemented \
+      	    for L2ShapeTri. Try getValues. Aborting()!");
+      }
+    };
+    EntityShape* getEntityShape(int type)
+    {
+    	PCU_ALWAYS_ASSERT_VERBOSE(type == Mesh::TRIANGLE,
+    			"L2ShapeTri only has entity shapes for TRIANGLEs");
+      static Triangle tri;
+      return &tri;
+    }
+    // For the following to member functions we only need to
+    // consider the interior nodes, i.e.,
+    // Faces: no need to count the nodes associated with bounding edges
+    // Tets: no need to count the nodes associated with bounding edges/faces
+    bool hasNodesIn(int dimension)
+    { 
+    	if (dimension == Mesh::typeDimension[Mesh::TRIANGLE])
+    		return true;
+      return false;
+    }
+    int countNodesOn(int type)
+    {
+      if (type == apf::Mesh::TRIANGLE) return countTriNodes(P);
+      return 0;
+    }
+    int getOrder() {return P;}
+    void getNodeXi(int type, int node, Vector3& xi)
+    {
+    	PCU_ALWAYS_ASSERT_VERBOSE(type == Mesh::TRIANGLE,
+    			"getNodeXi for L2ShapeTri can be called only for TRIANGLEs");
+      apf::NewArray<double> op;
+			getOpenPoints(P, op);
+			int c = 0;
+
+			for (int j = 0; j <= P; j++) {
+				for (int i = 0; i + j <= P; i++) {
+					if (node == c) { 
+						double w = op[i] + op[j] + op[P-i-j];
+						xi = Vector3( op[i]/w, op[j]/w, 0. );
+						return;
+					}
+					else
+						c++;
+				}
+			}
+    }
+};
+
+template<int P>
+class L2ShapeTet: public FieldShape {
+  public:
+    const char* getName() const { return "L2ShapeTet"; }
+    bool isVectorShape() {return false;}
+    L2ShapeTet()
+    {}
+    class Tetrahedron : public apf::EntityShape
+    {
+    public:
+      int getOrder() {return P;}
+      void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const& xi, apf::NewArray<double>& shapes) const
+      {
+        const int p = P;
+
+      	apf::NewArray<double> shape_x(p+1);
+      	apf::NewArray<double> shape_y(p+1);
+      	apf::NewArray<double> shape_z(p+1);
+      	apf::NewArray<double> shape_l(p+1);
+
+        int dof = countNodes();
+        mth::Matrix<double> u(dof, 1);
+
+        double x = xi[0]; double y = xi[1]; double z = xi[2];
+
+        getChebyshevT(p, x, &shape_x[0]);
+        getChebyshevT(p, y, &shape_y[0]);
+        getChebyshevT(p, z, &shape_z[0]);
+        getChebyshevT(p, 1. - x - y - z, &shape_l[0]);
+
+        int n = 0;
+        for (int k = 0; k <= p; k++)
+          for (int j = 0; j + k <= p; j++)
+            for (int i = 0; i + j + k <= p; i++)
+            	u(n++, 0) = shape_x[i]*shape_y[j]*shape_z[k]*shape_l[p-i-j-k];
+
+
+      	mth::Matrix<double> Q(dof, dof);
+      	mth::Matrix<double> R(dof, dof);
+      	getTi(P, apf::Mesh::TET, Q, R);
+
+        mth::Matrix<double> S(dof, 1);
+      	for(int i = 0; i < 1; i++) // S = Ti * u
+        {
+          mth::Vector<double> B (dof);
+          mth::Vector<double> X (dof);
+          for (int j = 0; j < dof; j++) B[j] = u(j,i); // populate b
+          mth::solveFromQR(Q, R, B, X);
+          for (int j = 0; j < dof; j++)  S(j,i) = X[j]; // populate S with x
+        }
+
+        shapes.allocate(dof);
+        for (int i = 0; i < dof; i++) // populate y
+        {
+      	  shapes[i] = S(i,0);
+        }
+      }
+      void getLocalGradients(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
+      {
+      	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getLocalGradients not \
+      	    implemented for L2ShapeTet. Aborting()!");
+      }
+      int countNodes() const {return countTetNodes(P);}
+      void getVectorValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const& xi, apf::NewArray<apf::Vector3>& shapes) const
+      {
+      	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getVectorValues not implemented for \
+      	    L2ShapeTet. Try getValues. Aborting()!");
+      }
+    };
+    EntityShape* getEntityShape(int type)
+    {
+    	PCU_ALWAYS_ASSERT_VERBOSE(type == Mesh::TRIANGLE,
+    			"L2ShapeTet only has entity shapes for TETs");
+      static Tetrahedron tet;
+      return &tet;
+    }
+    // For the following to member functions we only need to
+    // consider the interior nodes, i.e.,
+    // Faces: no need to count the nodes associated with bounding edges
+    // Tets: no need to count the nodes associated with bounding edges/faces
+    bool hasNodesIn(int dimension)
+    { 
+    	if (dimension == Mesh::typeDimension[Mesh::TET])
+    		return true;
+      return false;
+    }
+    int countNodesOn(int type)
+    {
+      if (type == apf::Mesh::TET) return countTetNodes(P);
+      return 0;
+    }
+    int getOrder() {return P;}
+    void getNodeXi(int type, int node, Vector3& xi)
+    {
+    	PCU_ALWAYS_ASSERT_VERBOSE(type == Mesh::TRIANGLE,
+    			"getNodeXi for L2ShapeTet can be called only for TETs");
+      apf::NewArray<double> op;
+			getOpenPoints(P, op);
+			int c = 0;
+
+			for (int k = 0; k <= P; k++) {
+				for (int j = 0; j + k <= P; j++) {
+					for (int i = 0; i + j + k <= P; i++) {
+						if( node == c) { 
+							double w = op[i] + op[j] + op[k] + op[P-i-j-k];
+							xi = Vector3( op[i]/w, op[j]/w,  op[k]/w );
+							return;
+						}
+						else
+							c++;
+					}
+				}
+			}
+    }
+};
+
+
+static apf::FieldShape* getL2ShapeTri(int order)
+{
+  PCU_ALWAYS_ASSERT_VERBOSE(order >= 0,
+      "order is expected to be bigger than or equal to 0!");
+  PCU_ALWAYS_ASSERT_VERBOSE(order <= 10,
+      "order is expected to be less than or equal to 10!");
+  static L2ShapeTri<0>  l2_0;
+  static L2ShapeTri<1>  l2_1;
+  static L2ShapeTri<2>  l2_2;
+  static L2ShapeTri<3>  l2_3;
+  static L2ShapeTri<4>  l2_4;
+  static L2ShapeTri<5>  l2_5;
+  static L2ShapeTri<6>  l2_6;
+  static L2ShapeTri<7>  l2_7;
+  static L2ShapeTri<8>  l2_8;
+  static L2ShapeTri<9>  l2_9;
+  static L2ShapeTri<10> l2_10;
+  static FieldShape* const l2Shapes[11] =
+  {&l2_0, &l2_1, &l2_2, &l2_3, &l2_4, &l2_5, &l2_6, &l2_7, &l2_8, &l2_9, &l2_10};
+  return l2Shapes[order];
+}
+
+static apf::FieldShape* getL2ShapeTet(int order)
+{
+  PCU_ALWAYS_ASSERT_VERBOSE(order >= 0,
+      "order is expected to be bigger than or equal to 0!");
+  PCU_ALWAYS_ASSERT_VERBOSE(order <= 10,
+      "order is expected to be less than or equal to 10!");
+  static L2ShapeTet<0>  l2_0;
+  static L2ShapeTet<1>  l2_1;
+  static L2ShapeTet<2>  l2_2;
+  static L2ShapeTet<3>  l2_3;
+  static L2ShapeTet<4>  l2_4;
+  static L2ShapeTet<5>  l2_5;
+  static L2ShapeTet<6>  l2_6;
+  static L2ShapeTet<7>  l2_7;
+  static L2ShapeTet<8>  l2_8;
+  static L2ShapeTet<9>  l2_9;
+  static L2ShapeTet<10> l2_10;
+  static FieldShape* const l2Shapes[11] =
+  {&l2_0, &l2_1, &l2_2, &l2_3, &l2_4, &l2_5, &l2_6, &l2_7, &l2_8, &l2_9, &l2_10};
+  return l2Shapes[order];
+}
+
+
+apf::FieldShape* getL2Shapes(int order, int type)
+{
+  if (type == Mesh::TRIANGLE)
+  	return getL2ShapeTri(order);
+	else if (type == Mesh::TET)
+  	return getL2ShapeTet(order);
+	else
+		PCU_ALWAYS_ASSERT_VERBOSE(0,
+				"L2Shapes are only implemented for tris and tets");
+}
+
+};
diff --git a/apf/apfShape.h b/apf/apfShape.h
index 37e259408..3a6c4c794 100644
--- a/apf/apfShape.h
+++ b/apf/apfShape.h
@@ -167,6 +167,11 @@ FieldShape* getHierarchic(int order);
  */
 FieldShape* getNedelec(int order);
 
+/** \brief Get the L2 shapes of a given order and entity type
+  \details
+ */
+FieldShape* getL2Shapes(int order, int type);
+
 /** \brief Project a hierarchic field */
 void projectHierarchicField(Field* to, Field* from);
 

From c99acfd24c5917d5a209a82d681c8805e845c1a4 Mon Sep 17 00:00:00 2001
From: Morteza HS <hakimi1364@gmail.com>
Date: Mon, 11 May 2020 01:39:21 -0400
Subject: [PATCH 148/555] Fixes indentations

---
 apf/apfL2Shapes.cc | 82 ++++++++++++++++++++++------------------------
 1 file changed, 40 insertions(+), 42 deletions(-)

diff --git a/apf/apfL2Shapes.cc b/apf/apfL2Shapes.cc
index c89a09cc8..e3a9c7b9f 100644
--- a/apf/apfL2Shapes.cc
+++ b/apf/apfL2Shapes.cc
@@ -269,23 +269,22 @@ class L2ShapeTri: public FieldShape {
     int getOrder() {return P;}
     void getNodeXi(int type, int node, Vector3& xi)
     {
-    	PCU_ALWAYS_ASSERT_VERBOSE(type == Mesh::TRIANGLE,
-    			"getNodeXi for L2ShapeTri can be called only for TRIANGLEs");
+      PCU_ALWAYS_ASSERT_VERBOSE(type == Mesh::TRIANGLE,
+      	  "getNodeXi for L2ShapeTri can be called only for TRIANGLEs");
       apf::NewArray<double> op;
-			getOpenPoints(P, op);
-			int c = 0;
-
-			for (int j = 0; j <= P; j++) {
-				for (int i = 0; i + j <= P; i++) {
-					if (node == c) { 
-						double w = op[i] + op[j] + op[P-i-j];
-						xi = Vector3( op[i]/w, op[j]/w, 0. );
-						return;
-					}
-					else
-						c++;
-				}
-			}
+      getOpenPoints(P, op);
+      int c = 0;
+      for (int j = 0; j <= P; j++) {
+      	for (int i = 0; i + j <= P; i++) {
+      	  if (node == c) {
+      	    double w = op[i] + op[j] + op[P-i-j];
+      	    xi = Vector3( op[i]/w, op[j]/w, 0. );
+      	    return;
+      	  }
+      	  else
+      	    c++;
+      	}
+      }
     }
 };
 
@@ -363,8 +362,8 @@ class L2ShapeTet: public FieldShape {
     };
     EntityShape* getEntityShape(int type)
     {
-    	PCU_ALWAYS_ASSERT_VERBOSE(type == Mesh::TRIANGLE,
-    			"L2ShapeTet only has entity shapes for TETs");
+      PCU_ALWAYS_ASSERT_VERBOSE(type == Mesh::TET,
+      	  "L2ShapeTet only has entity shapes for TETs");
       static Tetrahedron tet;
       return &tet;
     }
@@ -386,25 +385,24 @@ class L2ShapeTet: public FieldShape {
     int getOrder() {return P;}
     void getNodeXi(int type, int node, Vector3& xi)
     {
-    	PCU_ALWAYS_ASSERT_VERBOSE(type == Mesh::TRIANGLE,
-    			"getNodeXi for L2ShapeTet can be called only for TETs");
+      PCU_ALWAYS_ASSERT_VERBOSE(type == Mesh::TET,
+      	  "getNodeXi for L2ShapeTet can be called only for TETs");
       apf::NewArray<double> op;
-			getOpenPoints(P, op);
-			int c = 0;
-
-			for (int k = 0; k <= P; k++) {
-				for (int j = 0; j + k <= P; j++) {
-					for (int i = 0; i + j + k <= P; i++) {
-						if( node == c) { 
-							double w = op[i] + op[j] + op[k] + op[P-i-j-k];
-							xi = Vector3( op[i]/w, op[j]/w,  op[k]/w );
-							return;
-						}
-						else
-							c++;
-					}
-				}
-			}
+      getOpenPoints(P, op);
+      int c = 0;
+      for (int k = 0; k <= P; k++) {
+      	for (int j = 0; j + k <= P; j++) {
+      	  for (int i = 0; i + j + k <= P; i++) {
+      	    if( node == c) {
+      	      double w = op[i] + op[j] + op[k] + op[P-i-j-k];
+      	      xi = Vector3( op[i]/w, op[j]/w,  op[k]/w );
+      	      return;
+      	    }
+      	    else
+      	      c++;
+      	  }
+      	}
+      }
     }
 };
 
@@ -457,12 +455,12 @@ static apf::FieldShape* getL2ShapeTet(int order)
 apf::FieldShape* getL2Shapes(int order, int type)
 {
   if (type == Mesh::TRIANGLE)
-  	return getL2ShapeTri(order);
-	else if (type == Mesh::TET)
-  	return getL2ShapeTet(order);
-	else
-		PCU_ALWAYS_ASSERT_VERBOSE(0,
-				"L2Shapes are only implemented for tris and tets");
+    return getL2ShapeTri(order);
+  else if (type == Mesh::TET)
+    return getL2ShapeTet(order);
+  else
+    PCU_ALWAYS_ASSERT_VERBOSE(0,
+    	"L2Shapes are only implemented for tris and tets");
 }
 
 };

From b52c2c23eefdec96b19c0308e2f043336132ee0a Mon Sep 17 00:00:00 2001
From: Morteza HS <hakimi1364@gmail.com>
Date: Mon, 11 May 2020 01:39:49 -0400
Subject: [PATCH 149/555] Adds test for L2 (discontinuous) fields

---
 test/CMakeLists.txt |   1 +
 test/L2Shapes.cc    | 152 ++++++++++++++++++++++++++++++++++++++++++++
 test/testing.cmake  |   4 ++
 3 files changed, 157 insertions(+)
 create mode 100644 test/L2Shapes.cc

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index af2bc7936..027f43ed4 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -177,6 +177,7 @@ test_exe_func(mixedNumbering mixedNumbering.cc)
 test_exe_func(test_verify test_verify.cc)
 test_exe_func(hierarchic hierarchic.cc)
 test_exe_func(nedelecShapes nedelecShapes.cc)
+test_exe_func(L2Shapes L2Shapes.cc)
 test_exe_func(poisson poisson.cc)
 test_exe_func(ph_adapt ph_adapt.cc)
 test_exe_func(assert_timing assert_timing.cc)
diff --git a/test/L2Shapes.cc b/test/L2Shapes.cc
new file mode 100644
index 000000000..18cc7bf93
--- /dev/null
+++ b/test/L2Shapes.cc
@@ -0,0 +1,152 @@
+#include <PCU.h>
+#include <gmi_mesh.h>
+#include <gmi_null.h>
+#include <apf.h>
+#include <apfMDS.h>
+#include <apfMesh2.h>
+#include <apfShape.h>
+#include <apfField.h>
+#include <lionPrint.h>
+#include <pcu_util.h>
+
+#include <cassert>
+#include <cstdlib>
+#include <iostream>
+#include <string>
+#include <math.h>
+
+using namespace std;
+
+// User defined vector functions E(x,y,z) of order up to 6
+void E_exact(const apf::Vector3& x, apf::Vector3& value, int p);
+
+void testL2(
+    apf::Mesh2* m,
+    const apf::Vector3& testXi,
+    int ndOrder, int exactOrder);
+
+int main(int argc, char** argv)
+{
+  MPI_Init(&argc,&argv);
+  PCU_Comm_Init();
+
+  lion_set_verbosity(1);
+
+  if (argc != 3) {
+    if(0==PCU_Comm_Self())
+      std::cerr << "usage: " << argv[0]
+        << " <model.dmg or .null> <mesh.smb>\n";
+    return EXIT_FAILURE;
+  }
+
+  gmi_register_mesh();
+  gmi_register_null();
+
+  gmi_model* g = gmi_load(argv[1]);
+  apf::Mesh2* m = apf::loadMdsMesh(g,argv[2]);
+  m->verify();
+
+  for (int i = 0; i <= 6; i++) {
+    testL2(
+    	m, /* mesh */
+	apf::Vector3(1./7., 1./11., 1./3.), /* test point */
+	i, /* order of Nedelec field */
+	i); /* order of test field */
+  }
+
+  apf::destroyMesh(m);
+  PCU_Comm_Free();
+  MPI_Finalize();
+}
+
+void E_exact(const apf::Vector3& x, apf::Vector3& value, int p)
+{
+  // Polynomial coefficients for each component of exact vector field
+  double a[7] = { 1.0, -1.0,  2., -2., -1.0,  1.0, -1.0};
+  double b[7] = {-2.0,  1.0, -2.,  2., -1.0, -1.0,  1.0};
+  double c[7] = { 3.0,  0.0, -1.,  0., -1.0,  1.0,  0.0};
+
+  value[0] = 0.0;
+  value[1] = 0.0;
+  value[2] = 0.0;
+  for (int i = p;  i >= 0; i--) {
+    value[0] += pow(x[0],p)*a[p];
+    value[1] += pow(x[1],p)*b[p];
+    value[2] += pow(x[2],p)*c[p];
+  }
+}
+
+
+
+void testL2(
+    apf::Mesh2* m,
+    const apf::Vector3& testXi,
+    int ndOrder, int exactOrder)
+{
+  // Loop over all nodes and set scalar dofs.
+  int dim = m->getDimension();
+  apf::Field* l2Field = apf::createField(
+      m, "l2_test", apf::VECTOR, apf::getL2Shapes(ndOrder, apf::Mesh::simplexTypes[dim]));
+  apf::MeshEntity* ent;
+  apf::MeshIterator* it;
+
+  for (int d = 0; d <= dim; d++) {
+    if (!l2Field->getShape()->countNodesOn(apf::Mesh::simplexTypes[d])) {
+      lion_oprint(1, "no nodes in dimension %d\n", d);
+      continue;
+    }
+    else
+      lion_oprint(1, "computing dofs for dimension %d\n", d);
+    it = m->begin(d);
+    int count = 0;
+    while( (ent = m->iterate(it)) ) {
+      int type = m->getType(ent);
+      int non = l2Field->getShape()->countNodesOn(type);
+      apf::MeshElement* me = apf::createMeshElement(m, ent);
+      for (int i = 0; i < non; i++)
+      {
+        apf::Vector3 xi, p, value;
+        l2Field->getShape()->getNodeXi(type, i, xi);
+        apf::mapLocalToGlobal(me, xi, p);
+        E_exact(p, value, exactOrder);
+
+      	apf::setVector(l2Field, ent, i, value);
+      }
+      apf::destroyMeshElement(me);
+      count++;
+    }
+    m->end(it);
+  }
+
+
+  // Verify that interpolated solution field agrees with exact field.
+  double L2ErrorE = 0.;
+  it = m->begin(dim);
+  int count = 0;
+  while( (ent = m->iterate(it)) ) {
+    apf::MeshElement* me = apf::createMeshElement(m, ent);
+    apf::Vector3 x;
+    apf::mapLocalToGlobal(me, testXi, x);
+    apf::Vector3 eFieldExact;
+    E_exact(x, eFieldExact, exactOrder);
+
+    // obtain interpolated value
+    apf::Element* el = apf::createElement(l2Field, me);
+    apf::Vector3 eFieldValue;
+    apf::getVector(el, testXi, eFieldValue);
+
+    double err = ((eFieldValue - eFieldExact) * (eFieldValue - eFieldExact))
+      / (eFieldExact * eFieldExact); // normalization factor
+    L2ErrorE += err;
+    apf::destroyMeshElement(me);
+    apf::destroyElement(el);
+    count++;
+  }
+  m->end(it);
+
+  // check for field interpolation
+  PCU_ALWAYS_ASSERT_VERBOSE(L2ErrorE < 1.e-16,
+      "Fields were not interpolated correctly!");
+
+  apf::destroyField(l2Field);
+}
diff --git a/test/testing.cmake b/test/testing.cmake
index 02754de6f..f33be609f 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -517,6 +517,10 @@ mpi_test(nedelec 1
   ./nedelecShapes
   "${MDIR}/cube.dmg"
   "${MDIR}/cube.smb")
+mpi_test(l2_shape_tet 1
+  ./L2Shapes
+  "${MDIR}/cube.dmg"
+  "${MDIR}/cube.smb")
 mpi_test(pumiLoadMesh-1p 1
   ./pumiLoadMesh
   ${MDIR}/cube.dmg

From 028535057b8ff83dc0d0b9a0605fc6e89b2a9fec Mon Sep 17 00:00:00 2001
From: Morteza HS <hakimi1364@gmail.com>
Date: Mon, 11 May 2020 22:38:48 -0400
Subject: [PATCH 150/555] Adds a vtk writer for piece-wise continues fields

---
 apf/CMakeLists.txt           |   1 +
 apf/apf.h                    |   6 +
 apf/apfVtkPieceWiseFields.cc | 246 +++++++++++++++++++++++++++++++++++
 3 files changed, 253 insertions(+)
 create mode 100644 apf/apfVtkPieceWiseFields.cc

diff --git a/apf/CMakeLists.txt b/apf/CMakeLists.txt
index a83dd71b8..94d4d6e5e 100644
--- a/apf/CMakeLists.txt
+++ b/apf/CMakeLists.txt
@@ -38,6 +38,7 @@ set(SOURCES
   apfMixedNumbering.cc
   apfAdjReorder.cc
   apfVtk.cc
+  apfVtkPieceWiseFields.cc
   apfFieldData.cc
   apfTagData.cc
   apfCoordData.cc
diff --git a/apf/apf.h b/apf/apf.h
index 5c5b02755..facb7478d 100644
--- a/apf/apf.h
+++ b/apf/apf.h
@@ -679,6 +679,12 @@ void writeASCIIVtkFiles(const char* prefix, Mesh* m);
 void writeASCIIVtkFiles(const char* prefix, Mesh* m,
     std::vector<std::string> writeFields);
 
+/** \brief Output .vtk files with ASCII encoding for this part.
+  \details this function is useful for debugging meshes with Nedelec 
+  fields on them.
+  */
+void writeNedelecVtkFiles(const char* prefix, Mesh* m);
+
 /** \brief Return the location of a gaussian integration point.
   \param type the element type, from apf::Mesh::getType
   \param order the order of the integration rule
diff --git a/apf/apfVtkPieceWiseFields.cc b/apf/apfVtkPieceWiseFields.cc
new file mode 100644
index 000000000..381e3573a
--- /dev/null
+++ b/apf/apfVtkPieceWiseFields.cc
@@ -0,0 +1,246 @@
+/*
+ * Copyright 2011 Scientific Computation Research Center
+ *
+ * This work is open source software, licensed under the terms of the
+ * BSD license as described in the LICENSE file in the top-level directory.
+ */
+
+#include <PCU.h>
+#include <lionBase64.h>
+#include <lionCompress.h>
+#include "apfMesh.h"
+#include "apfNumbering.h"
+#include "apfNumberingClass.h"
+#include "apfShape.h"
+#include "apfFieldData.h"
+#include <sstream>
+#include <fstream>
+#include <pcu_util.h>
+#include <lionPrint.h>
+#include <cstdlib>
+#include <stdint.h>
+#include <vector>
+#include <apfVtk.h>
+
+// === includes for safe_mkdir ===
+#include <reel.h>
+#include <sys/types.h> /*required for mode_t for mkdir on some systems*/
+#include <sys/stat.h> /*using POSIX mkdir call for SMB "foo/" path*/
+#include <errno.h> /* for checking the error from mkdir */
+// ===============================
+
+namespace apf {
+
+static void safe_mkdir(const char* path)
+{
+  mode_t const mode = S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH;
+  int err;
+  errno = 0;
+  err = mkdir(path, mode);
+  if (err != 0 && errno != EEXIST)
+  {
+    reel_fail("MDS: could not create directory \"%s\"\n", path);
+  }
+}
+
+/* Paraview/VTK has trouble with sub-normal double precision floating point
+ * ASCII values.
+ *
+ * http://www.paraview.org/Bug/view.php?id=15925
+ *
+ * This function exists to cast "double" to "float"
+ * before writing it to file, the others are to maintain the
+ * templated design of writeCornerCoords and others */
+
+/* static float workaround(double v) */
+/* { */
+/*   return static_cast<float>(v); */
+/* } */
+/* static int workaround(int v) */
+/* { */
+/*   return v; */
+/* } */
+/* static long workaround(long v) */
+/* { */
+/*   return v; */
+/* } */
+
+static std::string getPieceFileName(int id)
+{
+  std::stringstream ss;
+  ss << id << ".vtk";
+  return ss.str();
+}
+
+
+static std::string getFileNameAndPathVtu(const char* prefix,
+    std::string fileName,
+    int id)
+{
+  int dirNum = id/1024;
+  std::stringstream ss;
+  ss << prefix << '/' << prefix << '_' << fileName;
+  return ss.str();
+}
+
+static void writeNedelecVtkFile(const char* prefix, Mesh* m,
+    std::vector<std::string> writeFields)
+{
+  double t0 = PCU_Time();
+
+  // get the number of points on this part
+  MeshEntity* e;
+  MeshIterator* it = m->begin(m->getDimension());
+  int point_count = 0;
+  int cell_count = 0;
+  bool isSimplexMesh = true;
+  while( (e = m->iterate(it)) ) {
+    int type = m->getType(e);
+    if (!apf::isSimplex(type)) {
+      isSimplexMesh = false;
+      break;
+    }
+    point_count += Mesh::adjacentCount[type][0];
+    cell_count++;
+  }
+  m->end(it);
+
+  PCU_ALWAYS_ASSERT_VERBOSE(isSimplexMesh,
+      "writeNedelecVtk only implemented for all simplex meshes");
+
+  static Vector3 xis[4] = {
+    apf::Vector3(0., 0., 0.),
+    apf::Vector3(1., 0., 0.),
+    apf::Vector3(0., 1., 0.),
+    apf::Vector3(0., 0., 1.)
+  };
+
+  std::string fileName = getPieceFileName(PCU_Comm_Self());
+  std::string fileNameAndPath = getFileNameAndPathVtu(prefix, fileName, PCU_Comm_Self());
+  std::stringstream buf;
+  buf <<
+    "# vtk DataFile Version 3.0\n"
+    "Made by SCOREC/core\n"
+    "ASCII\n"
+    "DATASET UNSTRUCTURED_GRID\n";
+  buf << "POINTS " << point_count << " double\n";
+
+  it = m->begin(m->getDimension());
+  while( (e = m->iterate(it)) ) {
+    int type = m->getType(e);
+    int n = Mesh::adjacentCount[type][0];
+    MeshElement* me = createMeshElement(m, e);
+    for(int i=0; i<n; i++) {
+      Vector3 p;
+      mapLocalToGlobal(me, xis[i], p);
+      buf << p[0] << ' ' << p[1] << ' ' << p[2] << '\n';
+    }
+    destroyMeshElement(me);
+  }
+  m->end(it);
+
+  buf << "CELLS " << cell_count << ' ' << point_count + cell_count << '\n';
+
+  it = m->begin(m->getDimension());
+  int N = 0;
+  while( (e = m->iterate(it)) ) {
+    int type = m->getType(e);
+    int n = Mesh::adjacentCount[type][0];
+    buf << n;
+    for(int i=0; i<n; i++) {
+      buf << ' ' << N;
+      N++;
+    }
+    buf << '\n';
+  }
+  m->end(it);
+
+  buf << "CELL_TYPES " << cell_count << '\n';
+  it = m->begin(m->getDimension());
+  while( (e = m->iterate(it)) ) {
+    int type = m->getType(e);
+    int vtk_type = -1;
+    switch (type) {
+      case Mesh::TRIANGLE: vtk_type = 5; break;
+      case Mesh::TET:      vtk_type = 10; break;
+      default:
+        PCU_ALWAYS_ASSERT_VERBOSE(0,
+            "only TRIANGLE and TET supported");
+        break;
+    }
+    buf << vtk_type << '\n';
+  }
+  m->end(it);
+
+  buf << "CELL_DATA " << cell_count << '\n';
+  buf << "SCALARS part_id int\n";
+  buf << "LOOKUP_TABLE default\n";
+  it = m->begin(m->getDimension());
+  while( (e = m->iterate(it)) ) {
+    buf << PCU_Comm_Self() << '\n';
+  }
+  m->end(it);
+
+  buf << "POINT_DATA " << point_count << '\n' << std::flush;
+
+  for(std::size_t i = 0; i < writeFields.size(); i++) {
+    Field* f = m->findField(writeFields[i].c_str());
+    const char* fName = f->getName();
+    buf << "VECTORS " << fName << " double\n";
+
+    it = m->begin(m->getDimension());
+    while( (e = m->iterate(it)) ) {
+      int type = m->getType(e);
+      int n = Mesh::adjacentCount[type][0];
+      MeshElement* me = createMeshElement(m, e);
+      Element* el = createElement(f, me);
+      for(int i=0; i<n; i++) {
+	Vector3 fVal;
+	getVector(el, xis[i], fVal);
+	buf << fVal[0] << ' ' << fVal[1] << ' ' << fVal[2] << '\n';
+      }
+      destroyElement(el);
+      destroyMeshElement(me);
+    }
+    m->end(it);
+  }
+  double t1 = PCU_Time();
+  if (!PCU_Comm_Self())
+  {
+    lion_oprint(1,"writeVtuFile into buffers: %f seconds\n", t1 - t0);
+  }
+  { //block forces std::ofstream destructor call
+    std::ofstream file(fileNameAndPath.c_str());
+    PCU_ALWAYS_ASSERT(file.is_open());
+    file << buf.rdbuf();
+  }
+  double t2 = PCU_Time();
+  if (!PCU_Comm_Self())
+  {
+    lion_oprint(1,"writeNedelecVtkFile buffers to disk: %f seconds\n", t2 - t1);
+  }
+}
+
+std::vector<std::string> populateNedelecFields(Mesh* m)
+{
+  std::vector<std::string> writeFields;
+  for (int i=0; i < m->countFields(); ++i)
+  {
+    Field* f = m->getField(i);
+    std::string fsName = f->getShape()->getName();
+    if (fsName == std::string("Nedelec"))
+      writeFields.push_back(f->getName());
+  }
+  return writeFields;
+}
+
+void writeNedelecVtkFiles(const char* prefix, Mesh* m)
+{
+  // bool isWritingBinary = true;
+  std::vector<std::string> writeFields = populateNedelecFields(m);
+  if (writeFields.size() == 0) return;
+  safe_mkdir(prefix);
+  writeNedelecVtkFile(prefix, m, writeFields);
+}
+
+}

From bf1e377327923dc26dec059e8f07c24ee62ccb6e Mon Sep 17 00:00:00 2001
From: Samiullah-Malik <39503305+Samiullah-Malik@users.noreply.github.com>
Date: Tue, 12 May 2020 15:45:12 -0400
Subject: [PATCH 151/555] Adds estimateError.cc which solves local BVPs.

---
 em/CMakeLists.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/em/CMakeLists.txt b/em/CMakeLists.txt
index 03b1a14c2..229ae59f9 100644
--- a/em/CMakeLists.txt
+++ b/em/CMakeLists.txt
@@ -6,7 +6,8 @@ endif()
 # Package sources
 set(SOURCES
   emResidualFunctionals.cc
-  emFluxCorrection.cc)
+  emFluxCorrection.cc
+  emEstimateError.cc)
 
 # Package headers
 set(HEADERS

From c417db586f06856cd057e03f57ef07eae4e0458c Mon Sep 17 00:00:00 2001
From: Samiullah-Malik <39503305+Samiullah-Malik@users.noreply.github.com>
Date: Tue, 12 May 2020 15:47:44 -0400
Subject: [PATCH 152/555] Moves certain general functions to em.h

---
 em/em.h                     | 43 +++++++++++++++++++++++++++++++------
 em/emResidualFunctionals.cc | 35 +++++++++++++-----------------
 2 files changed, 52 insertions(+), 26 deletions(-)

diff --git a/em/em.h b/em/em.h
index c119887ad..1845a1696 100644
--- a/em/em.h
+++ b/em/em.h
@@ -20,18 +20,49 @@
 
 namespace em {
 
+// Takes the solution electric field and corrected flux field and solves
+// local element level BVPs to estimate the error.
+// Returns a per-element scalar error field.
+apf::Field* emEstimateError(apf::Field* ef, apf::Field* correctedFlux);
 
+// Takes the solution electric field and equilibrated field (of face vectors)
+// and computes the 'correction' to the flux vectors on each face.
 apf::Field* computeFluxCorrection(apf::Field* ef, apf::Field* g);
 
+// Takes the solution electric field and computes edge equilibrations.
 apf::Field* equilibrateResiduals(apf::Field* f);
 
 
-
-
-
-
-
-
+void assembleCurlCurlElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
+    apf::Field* f, mth::Matrix<double>& elmat);
+
+void assembleVectorMassElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
+    apf::Field* f, mth::Matrix<double>& elmat);
+
+void assembleElementMatrix(apf::Mesh* mesh, apf::MeshEntity*e,
+    apf::Field* f, mth::Matrix<double>& elmat);
+
+void assembleDomainLFElementVector(apf::Mesh* mesh, apf::MeshEntity* e,
+    apf::Field* f, mth::Vector<double>& elvect)
+
+// TODO QUESTION redo this to allow user access from outside
+void pumiUserFunction(const apf::Vector3& x, mth::Vector<double>& f,
+    apf::MeshEntity* e, apf::Mesh* mesh)
+{
+  double freq = 1.;
+  double kappa = freq * M_PI;
+  int dim = apf::getDimension(mesh, e);
+  if (dim == 3) {
+      f(0) = (1. + kappa * kappa) * sin(kappa * x[1]);
+      f(1) = (1. + kappa * kappa) * sin(kappa * x[2]);
+      f(2) = (1. + kappa * kappa) * sin(kappa * x[0]);
+  }
+  else {
+     f(0) = (1. + kappa * kappa) * sin(kappa * x[1]);
+     f(1) = (1. + kappa * kappa) * sin(kappa * x[0]);
+     f(2) = 0.0;
+  }
+}
 
 
 }
diff --git a/em/emResidualFunctionals.cc b/em/emResidualFunctionals.cc
index 582fe05a5..8b6e8c4ab 100644
--- a/em/emResidualFunctionals.cc
+++ b/em/emResidualFunctionals.cc
@@ -195,7 +195,7 @@ static void assembleLHS(EdgePatch* p)
   std::cout << p->qr.R << std::endl;*/
 }
 
-static void assembleCurlCurlElementMatrix(apf::Mesh* mesh,apf::MeshEntity* e,
+void assembleCurlCurlElementMatrix(apf::Mesh* mesh,apf::MeshEntity* e,
     apf::Field* f, mth::Matrix<double>& elmat)
 {
   apf::FieldShape* fs = f->getShape();
@@ -253,7 +253,7 @@ static void assembleCurlCurlElementMatrix(apf::Mesh* mesh,apf::MeshEntity* e,
   apf::destroyMeshElement(me);
 }
 
-static void assembleVectorMassElementMatrix(apf::Mesh* mesh,apf::MeshEntity* e,
+void assembleVectorMassElementMatrix(apf::Mesh* mesh,apf::MeshEntity* e,
     apf::Field* f, mth::Matrix<double>& elmat)
 {
   apf::FieldShape* fs = f->getShape();
@@ -298,6 +298,19 @@ static void assembleVectorMassElementMatrix(apf::Mesh* mesh,apf::MeshEntity* e,
   apf::destroyMeshElement(me);
 }
 
+void assembleElementMatrix(apf::Mesh* mesh, apf::MeshEntity*e,
+    apf::Field* f, mth::Matrix<double>& elmat)
+{
+  mth::Matrix<double> curl_elmat, mass_elmat;
+  assembleCurlCurlElementMatrix(mesh, e, f, curl_elmat);
+  assembleVectorMassElementMatrix(mesh, e, f, mass_elmat);
+  
+  elmat.resize(curl_elmat.rows(), curl_elmat.cols());
+  elmat.zero();
+  elmat += curl_elmat;
+  elmat += mass_elmat;
+}
+
 // computes local bilinear form integral restricted to
 // an edge of an element
 static double getLocalBLFIntegral(EdgePatch* p, apf::MeshEntity* tet)
@@ -344,24 +357,6 @@ static double getLocalBLFIntegral(EdgePatch* p, apf::MeshEntity* tet)
   return integrals(ei);
 }
 
-// TODO QUESTION redo this to allow user access from outside
-static void pumiUserFunction(const apf::Vector3& x, mth::Vector<double>& f,
-    apf::MeshEntity* tet, apf::Mesh* mesh)
-{
-  double freq = 1.;
-  double kappa = freq * M_PI;
-  int dim = apf::getDimension(mesh, tet);
-  if (dim == 3) {
-      f(0) = (1. + kappa * kappa) * sin(kappa * x[1]);
-      f(1) = (1. + kappa * kappa) * sin(kappa * x[2]);
-      f(2) = (1. + kappa * kappa) * sin(kappa * x[0]);
-  }
-  else {
-     f(0) = (1. + kappa * kappa) * sin(kappa * x[1]);
-     f(1) = (1. + kappa * kappa) * sin(kappa * x[0]);
-     f(2) = 0.0;
-  }
-}
 
 void assembleDomainLFElementVector(apf::Mesh* mesh, apf::MeshEntity* e,
     apf::Field* f, mth::Vector<double>& elvect)

From 1dc440f02ff6f774db764df45d6a752905ec7dd8 Mon Sep 17 00:00:00 2001
From: Samiullah-Malik <39503305+Samiullah-Malik@users.noreply.github.com>
Date: Tue, 12 May 2020 17:19:39 -0400
Subject: [PATCH 153/555] Adds TODOs emEstimateError.cc to solve local BVPs

Computes lhs element matrices
Computes rhs residual data vectors
  1. bilinear form
  2. Linear form
  3. TODO lambda vector

TODOs: take care of negative dof orientations
---
 em/emEstimateError.cc | 171 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 171 insertions(+)
 create mode 100644 em/emEstimateError.cc

diff --git a/em/emEstimateError.cc b/em/emEstimateError.cc
new file mode 100644
index 000000000..c35f39cb5
--- /dev/null
+++ b/em/emEstimateError.cc
@@ -0,0 +1,171 @@
+/*
+ * Copyright (C) 2011 Scientific Computation Research Center
+ *
+ * This work is open source software, licensed under the terms of the
+ * BSD license as described in the LICENSE file in the top-level directory.
+ */
+#include <iostream>
+#include <cstdlib>
+#include <apf.h>
+#include <PCU.h>
+#include <lionPrint.h>
+
+#include "em.h"
+
+#include <apfMesh.h>
+#include <apfShape.h>
+#include <apfField.h>
+#include <apfElement.h>
+#include <apfCavityOp.h>
+#include "crv.h"
+#include "crvShape.h"
+
+#include <mthQR.h>
+#include <mth.h>
+#include <mth_def.h>
+#include <limits>
+
+#include <set>
+#include <pcu_util.h>
+
+#include "em.h"
+
+namespace em {
+
+static void computeResidualBLF(apf::Mesh* mesh, apf::MeshEntity* e,
+  apf::Field* f, mth::Vector<double>& blf)
+{
+  apf::FieldShape* fs = f->getShape();
+  int type = mesh->getType(e);
+  PCU_ALWAYS_ASSERT(type == apf::Mesh::TET);
+  int nd = apf::countElementNodes(fs, type);
+  int dim = apf::getDimension(mesh, e);
+  double w;
+
+  apf::NewArray<apf::Vector3> curlshape(nd);
+  apf::NewArray<apf::Vector3> vectorshape(nd);
+  mth::Matrix<double> phys_curlshape(nd, dim);
+  mth::Matrix<double> Vectorshape(nd, dim);
+  mth::Vector<double> curlcurl_vec (nd);
+  mth::Vector<double> mass_vec (nd);
+  blf.resize(nd);
+
+  apf::MeshElement* me = apf::createMeshElement(mesh, e);
+  apf::Element* el = apf::createElement(f, me);
+  int order = 2 * fs->getOrder() - 2;
+  int np = apf::countIntPoints(me, order); // int points required
+
+  // 1. Compute Curl Curl Integration
+  curlcurl_vec.zero();
+  apf::Vector3 p, curl;
+  for (int i = 0; i < np; i++) {
+    apf::getIntPoint(me, order, i, p);
+    double weight = apf::getIntWeight(me, order, i);
+    apf::Matrix3x3 J;
+    apf::getJacobian(me, p, J);
+    double jdet = apf::getJacobianDeterminant(J, dim);
+    w = weight; // TODO check why do not need division by jdet
+
+    // get curl vector
+    apf::getCurl(el, p, curl);
+    
+    // get curlshape values
+    el->getShape()->getLocalVectorCurls(mesh, e, p, curlshape);
+    phys_curlshape.zero();
+    for (int i = 0; i < nd; i++)
+      for (int j = 0; j < dim; j++)
+        for (int k = 0; k < dim; k++)
+          phys_curlshape(i,j) += curlshape[i][k] * J[k][j];
+
+    // multiply
+    mth::Vector<double> temp (nd);
+    temp.zero();
+    mth::Vector<double> c (dim);
+    c(0) = curl[0]; c(1) = curl[1]; c(2) = curl[2];
+    mth::multiply(phys_curlshape, c, temp);
+    temp *= w;
+    
+    curlcurl_vec += temp;
+
+  }
+    
+  // 2. Compute Vector Mass Integration
+  mass_vec.zero();
+  apf::Vector3 vvalue;
+  for (int i = 0; i < np; i++) {
+    apf::getIntPoint(me, order, i, p);
+    double weight = apf::getIntWeight(me, order, i);
+    apf::Matrix3x3 J;
+    apf::getJacobian(me, p, J);
+    double jdet = apf::getJacobianDeterminant(J, dim);
+
+    w = weight * jdet;
+
+    apf::getVector(el, p, vvalue);
+
+    apf::getVectorShapeValues(el, p, vectorshape);
+    Vectorshape.zero();
+    for (int i = 0; i < nd; i++)
+      for (int j = 0; j < dim; j++)
+        Vectorshape(i,j) = vectorshape[i][j];
+
+
+    mth::Vector<double> temp (nd);
+    temp.zero();
+    mth::Vector<double> v (dim);
+    v(0) = vvalue[0]; v(1) = vvalue[1]; v(2) = vvalue[2];
+    mth::multiply(Vectorshape, v, temp);
+    temp *= w;
+
+    mass_vec += temp;
+  }
+
+  // 3. get result
+  blf.zero();
+  blf += curlcurl_vec;
+  blf += mass_vec;
+
+  // TODO Take care of Negative Dofs 
+}
+
+
+apf::Field* emEstimateError(apf::Field* ef, apf::Field* correctedFlux)
+{
+  // 1. Create one order higher ND field
+  int order = ef->getShape()->getOrder();
+  int orderp1 = order+1;
+  apf::Field* efp1 = apf::createField(
+      apf::getMesh(ef), "higher_order_nedelec_field", apf::SCALAR, apf::getNedelec(orderp1));
+  apf::zeroField(efp1);
+
+  // 2. iterate over all elements of the mesh
+  apf::MeshEntity* el;
+  apf::MeshIterator* it = apf::getMesh(ef)->begin(3);
+  while ((el = apf::getMesh(ef)->iterate(it))) {
+    // 2(a). Assemble LHS element matrix
+    mth::Matrix<double> lhs;
+    assembleElementMatrix( apf::getMesh(ef), el, efp1, lhs);
+    // TODO Take care of negative dofs in lhs
+
+    // 2(b). Compute Bilinear Form Vector 
+    mth::Vector<double> blf;
+    computeResidualBLF(apf::getMesh(efp1), e, efp1, blf);
+    // TODO Take care of negative dofs in blf
+
+    // 2(c). Compute Linear Form Vector
+    mth::Vector<double> lf;
+    assembleDomainLFElementVector(apf::getMesh(efp1), e, efp1, lf);
+    // TODO Take care of negative dofs in lf
+    
+    // 2(c). Compute Lamda Vector
+    mth::Vector<double> lambda;
+    assembleDomainLFElementVector(apf::getMesh(efp1), e, efp1, lf);
+    // TODO Take care of negative dofs in lambda
+
+
+  }
+  apf::getMesh(ef)->end(it);
+
+
+
+}

From cddc6a4b9575e1eb50e0c93a3280bc59fcc27307 Mon Sep 17 00:00:00 2001
From: Morteza HS <hakimi1364@gmail.com>
Date: Wed, 13 May 2020 00:59:07 -0400
Subject: [PATCH 154/555] Fixes loop index (i->j) to avoid confusion

---
 apf/apfVtkPieceWiseFields.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/apf/apfVtkPieceWiseFields.cc b/apf/apfVtkPieceWiseFields.cc
index 381e3573a..63eaac9e0 100644
--- a/apf/apfVtkPieceWiseFields.cc
+++ b/apf/apfVtkPieceWiseFields.cc
@@ -194,9 +194,9 @@ static void writeNedelecVtkFile(const char* prefix, Mesh* m,
       int n = Mesh::adjacentCount[type][0];
       MeshElement* me = createMeshElement(m, e);
       Element* el = createElement(f, me);
-      for(int i=0; i<n; i++) {
+      for(int j=0; j<n; j++) {
 	Vector3 fVal;
-	getVector(el, xis[i], fVal);
+	getVector(el, xis[j], fVal);
 	buf << fVal[0] << ' ' << fVal[1] << ' ' << fVal[2] << '\n';
       }
       destroyElement(el);

From 99d75dd4ae86761b630d274e4878eb9632efcba0 Mon Sep 17 00:00:00 2001
From: Samiullah-Malik <39503305+Samiullah-Malik@users.noreply.github.com>
Date: Wed, 13 May 2020 13:29:50 -0400
Subject: [PATCH 155/555] Moves computeFaceOutwardNormal to em.h

---
 em/em.h | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/em/em.h b/em/em.h
index 1845a1696..0ec92f109 100644
--- a/em/em.h
+++ b/em/em.h
@@ -43,7 +43,7 @@ void assembleElementMatrix(apf::Mesh* mesh, apf::MeshEntity*e,
     apf::Field* f, mth::Matrix<double>& elmat);
 
 void assembleDomainLFElementVector(apf::Mesh* mesh, apf::MeshEntity* e,
-    apf::Field* f, mth::Vector<double>& elvect)
+    apf::Field* f, mth::Vector<double>& elvect);
 
 // TODO QUESTION redo this to allow user access from outside
 void pumiUserFunction(const apf::Vector3& x, mth::Vector<double>& f,
@@ -64,6 +64,9 @@ void pumiUserFunction(const apf::Vector3& x, mth::Vector<double>& f,
   }
 }
 
+apf::Vector3 computeFaceOutwardNormal(apf::Mesh* m,
+    apf::MeshEntity* t, apf::MeshEntity* f, apf::Vector3 const& p);
+
 
 }
 

From bb61c57f7165ef8eae5da3f63353b430ccadf0ce Mon Sep 17 00:00:00 2001
From: Samiullah-Malik <39503305+Samiullah-Malik@users.noreply.github.com>
Date: Wed, 13 May 2020 13:30:46 -0400
Subject: [PATCH 156/555] Adds clean up TODOs and moves facenormal to em.h

---
 em/emResidualFunctionals.cc | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/em/emResidualFunctionals.cc b/em/emResidualFunctionals.cc
index 8b6e8c4ab..6c5fd4698 100644
--- a/em/emResidualFunctionals.cc
+++ b/em/emResidualFunctionals.cc
@@ -455,7 +455,7 @@ static apf::Vector3 computeFaceNormal(apf::Mesh* m,
   return n.normalize();
 }
 
-static apf::Vector3 computeFaceOutwardNormal(apf::Mesh* m,
+apf::Vector3 computeFaceOutwardNormal(apf::Mesh* m,
     apf::MeshEntity* t, apf::MeshEntity* f, apf::Vector3 const& p)
 {
   apf::Vector3 n = computeFaceNormal(m, f, p);
@@ -545,8 +545,8 @@ static double getFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
           fnormal2 = computeFaceOutwardNormal(ep->mesh, secondTet, currentFace, p);
         else
           fnormal2 = computeFaceOutwardNormal(ep->mesh, firstTet, currentFace, p);
-        std::cout << "normal1 " << fnormal1 << std::endl;
-        std::cout << "normal2 " << fnormal2 << std::endl;
+        std::cout << "normal1 " << fnormal1 << std::endl; // REMOVE
+        std::cout << "normal2 " << fnormal2 << std::endl; // REMOVE
       }
 
 
@@ -556,8 +556,7 @@ static double getFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
       apf::MeshElement* me1 = apf::createMeshElement(ep->mesh, firstTet);
       apf::Element* el1 = apf::createElement(ep->equilibration->ef, me1);
       apf::getCurl(el1, tet1xi, curl1);
-      //curl += curl1;
-      apf::Vector3 temp1 = apf::cross(fnormal1, curl1); //
+      apf::Vector3 temp1 = apf::cross(fnormal1, curl1); // TODO clean
       curl += temp1; //
       apf::destroyElement(el1);
       apf::destroyMeshElement(me1);
@@ -568,8 +567,7 @@ static double getFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
         apf::MeshElement* me2 = apf::createMeshElement(ep->mesh, secondTet);
         apf::Element* el2 = apf::createElement(ep->equilibration->ef, me2);
         apf::getCurl(el2, tet2xi, curl2);
-        //curl += curl2;
-        apf::Vector3 temp2 = apf::cross(fnormal2, curl2); //
+        apf::Vector3 temp2 = apf::cross(fnormal2, curl2); // TODO clean
         curl += (temp2 * -1.); //
         apf::destroyElement(el2);
         apf::destroyMeshElement(me2);

From 09b760d148712073c3ec097de01c0e4b365ad979 Mon Sep 17 00:00:00 2001
From: Samiullah-Malik <39503305+Samiullah-Malik@users.noreply.github.com>
Date: Wed, 13 May 2020 13:32:27 -0400
Subject: [PATCH 157/555] Adds lambda interal vector computation for BVPs.

---
 em/emEstimateError.cc | 161 +++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 159 insertions(+), 2 deletions(-)

diff --git a/em/emEstimateError.cc b/em/emEstimateError.cc
index c35f39cb5..5f89e22b1 100644
--- a/em/emEstimateError.cc
+++ b/em/emEstimateError.cc
@@ -31,6 +31,7 @@
 #include "em.h"
 
 namespace em {
+//TODO destroy all fields created inside functions to prevent memory leaks.
 
 static void computeResidualBLF(apf::Mesh* mesh, apf::MeshEntity* e,
   apf::Field* f, mth::Vector<double>& blf)
@@ -128,7 +129,161 @@ static void computeResidualBLF(apf::Mesh* mesh, apf::MeshEntity* e,
   // TODO Take care of Negative Dofs 
 }
 
+// 
+// @ apf::Field* f --> input p-order ND field
+// @ apf::Field* fp1 --> p+1 order ND field created for local BVPs
+// @ apf::Field* THETA --> constant face field containing theta coeffs
+static void computeLambdaVector(apf::Mesh* mesh, apf::MeshEntity* e,
+  apf::Field* f, apf::Field* fp1, apf::Field* THETA, mth::Vector<double>& lambda)
+{
+  apf::FieldShape* fp1s = fp1->getShape();
+  int etype = mesh->getType(e);
+  PCU_ALWAYS_ASSERT(etype == apf::Mesh::TET);
+  int nedofs = apf::countElementNodes(fp1s, etype);
+  int edim = apf::getDimension(mesh, e);
+  lambda.resize(nedofs);
+
+  // create a 2D nedelec field 
+  int order = fp1s->getOrder();
+  apf::Field* faceNDField = apf::createField(
+      mesh, "face_nedelec_field", apf::SCALAR, apf::getNedelec(order));
+  apf::zeroField(faceNDField);
+  apf::FieldShape* faceNDFieldShape = faceNDField->getShape();
+
+  // get the downward faces of the element
+  apf::Downward faces;
+  int nf = mesh->getDownward(e, 2, faces);
+  
+  lambda.zero();
+  // assemble lambda vector LOOP OVER DOWNWARD FACES
+  for (int ii = 0; ii < nf; ii++) {
+    apf::MeshEntity* face = faces[ii];
+
+    // 1. get upward tets of the current face
+    apf::Up up;
+    mesh->getUp(face, up);
+    if (crv::isBoundaryEntity(mesh, face))
+      PCU_ALWAYS_ASSERT( up.n == 1);
+    else
+      PCU_ALWAYS_ASSERT( up.n == 2);
+
+    apf::MeshEntity* firstTet = up.e[0];
+    apf::MeshEntity* secondTet;
+    if (up.n == 2)
+      secondTet  = up.e[1];
+
+    // 2. get downward edges of the face
+    apf::Downward edges;
+    int nedges = mesh->getDownward(face, 1, edges);
+    
+    // 3. get theta coeffs on the face
+    double components[3];
+    apf::getComponents(THETA, face, 0, components);
+    mth::Vector<double> theta_coeffs(components); // TODO Warning make this work
+
+    int ftype = mesh->getType(face);
+    PCU_ALWAYS_ASSERT(ftype == apf::Mesh::TRIANGLE);
+    int nfdofs = apf::countElementNodes(faceNDFieldShape, ftype);
+    int fdim = apf::getDimension(mesh, face);
+    apf::NewArray<apf::Vector3> vectorshape(nfdofs);
+
+    apf::MeshElement* fme = apf::createMeshElement(mesh, face);
+    apf::Element* fel = apf::createElement(faceNDField, fme);
+    int np = apf::countIntPoints(fme, 2*order); // int points required
+
+    // 4. Compute integral on the face
+    apf::Vector3 p, tet1xi, tet2xi, curl1, curl2, curl,
+      fnormal1, fnormal2, tk, vshape;
+    for (int n = 0; n < np; n++) {
+
+      apf::getIntPoint(fme, 2*order, n, p);
+      double weight = apf::getIntWeight(fme, 2*order, n);
+      apf::Matrix3x3 fJ;
+      apf::getJacobian(fme, p, fJ);
+      double jdet = apf::getJacobianDeterminant(
+          fJ, apf::getDimension(mesh, face));
 
+      // evaluate theta vector using theta coeffs
+      apf::Vector3 theta_vector;
+      theta_vector.zero();
+      apf::NewArray<apf::Vector3> vector2Dshapes (nfdods);
+      apf::getVectorShapeValues(fel, p, vector2Dshapes);
+      int which, rotate; bool flip; // negative ND dofs
+      for (int i = 0; i < theta_coeffs.size(); i++) {
+        apf::getAlignment(mesh, face, edges[i], which, flip, rotate);
+        apf::Vector3 v = vector2Dshapes[i];
+        if (flip) { v = v * -1.; }
+
+        v = v * theta_coeffs[i];
+        theta_vector += v;
+      }
+
+      // compute face outward normals wrt tets
+      if (e == firstTet)
+        fnormal1 = computeFaceOutwardNormal(mesh, firstTet, face, p);
+      else
+        fnormal1 = computeFaceOutwardNormal(mesh, secondTet, face, p);
+      if (up.n == 2) {
+        if (e == firstTet)
+          fnormal2 = computeFaceOutwardNormal(mesh, secondTet, face, p);
+        else
+          fnormal2 = computeFaceOutwardNormal(mesh, firstTet, face, p);
+        std::cout << "normal1 " << fnormal1 << std::endl; // REMOVE
+        std::cout << "normal2 " << fnormal2 << std::endl; // REMOVE
+      }
+
+      curl.zero();
+      // compute curl1
+      tet1xi = apf::boundaryToElementXi(mesh, face, firstTet, p);
+      apf::MeshElement* me1 = apf::createMeshElement(mesh, firstTet);
+      apf::Element* el1 = apf::createElement(f, me1);
+      apf::getCurl(el1, tet1xi, curl1);
+      apf::Vector3 temp1 = apf::cross(fnormal1, curl1); // TODO clean
+      curl += temp1; //
+      apf::destroyElement(el1);
+      apf::destroyMeshElement(me1);
+
+      // compute curl2
+      if (up.n == 2) {
+        tet2xi = apf::boundaryToElementXi(mesh, face, secondTet, p);
+        apf::MeshElement* me2 = apf::createMeshElement(mesh, secondTet);
+        apf::Element* el2 = apf::createElement(f, me2);
+        apf::getCurl(el2, tet2xi, curl2);
+        apf::Vector3 temp2 = apf::cross(fnormal2, curl2); // TODO clean
+        curl += (temp2 * -1.); //
+        apf::destroyElement(el2);
+        apf::destroyMeshElement(me2);
+      }
+
+      // compute tk (inter-element averaged flux)
+      tk = curl * 1./2.;
+      std::cout << "tk " << tk << std::endl;
+
+      // compute p+1 order 3D vector shapes
+      apf::NewArray<apf::Vector3> vector3dshapes (nedofs);
+      apf::MeshElement* me = apf::createMeshElement(mesh, e);
+      apf::Element* el = apf::createElement(fp1, me);
+      apf::Vector3 tetxi = apf::boundaryToElementXi(mesh, face, e, p);
+      apf::getVectorShapeValues(el, tetxi, vector3dshapes);
+      // TODO take care of negative ND dofs
+      apf::destroyMeshElement(me); 
+      // compute integral
+      apf::Vector3 theta_plus_tk = theta_vec + tk;
+      double w = weight * jdet;
+      theta_plus_tk *= w;
+
+      // matrix vector multiplication
+      for (int i = 0; i < nedofs; i++)
+        lambda(i) += theta_plus_tk * vector3dshapes[i];
+
+    } // end integral loop
+    apf::destroyMeshElement(fme);
+  } // end face loop
+}
+
+// @ apf::Field* ef --> input p order ND electric field
+// @ apf::Field* ef --> input theta corrected flux field
+// *** p+1 order field is created inside this function
 apf::Field* emEstimateError(apf::Field* ef, apf::Field* correctedFlux)
 {
   // 1. Create one order higher ND field
@@ -157,10 +312,12 @@ apf::Field* emEstimateError(apf::Field* ef, apf::Field* correctedFlux)
     assembleDomainLFElementVector(apf::getMesh(efp1), e, efp1, lf);
     // TODO Take care of negative dofs in lf
     
-    // 2(c). Compute Lamda Vector
+    // 2(d). Compute Lamda Vector
     mth::Vector<double> lambda;
-    assembleDomainLFElementVector(apf::getMesh(efp1), e, efp1, lf);
+    computeLambdaVector(apf::getMesh(efp1), e, ef, efp1, correctedFlux, lambda);
     // TODO Take care of negative dofs in lambda
+   
+     
 
 
   }

From fe5931b32ac26f1f410555438f0b405dab3ee799 Mon Sep 17 00:00:00 2001
From: Samiullah-Malik <39503305+Samiullah-Malik@users.noreply.github.com>
Date: Wed, 13 May 2020 18:30:57 -0400
Subject: [PATCH 158/555] Applies DBCs to BVPs & computes per-element l2-err

---
 em/emEstimateError.cc | 217 ++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 207 insertions(+), 10 deletions(-)

diff --git a/em/emEstimateError.cc b/em/emEstimateError.cc
index 5f89e22b1..4a8a9284c 100644
--- a/em/emEstimateError.cc
+++ b/em/emEstimateError.cc
@@ -281,16 +281,181 @@ static void computeLambdaVector(apf::Mesh* mesh, apf::MeshEntity* e,
   } // end face loop
 }
 
+static void getEssentialBdrElementNDDofs(apf::Mesh* mesh, apf::MeshEntity* e,
+    apf::Field* f, apf::NewArray<int>& ess_dofs, apf::NewArray<int>& uness_dofs)
+{
+  apf::FieldShape* fs = f->getShape();
+  int etype = mesh->getType(e);
+  PCU_ALWAYS_ASSERT(etype == apf::Mesh::TET);
+  int nedofs = apf::countElementNodes(fs, etype);
+
+  apf::NewArray<int> marker_list (nedofs);
+  for (int i = 0; i < nedofs; i++)
+    marker_list[i] = 0;
+
+  // populate marker list by iterating over downward edges and faces
+  apf::Downward edges;
+  int nedges = mesh->getDownward(e, 1, edges);
+  PCU_ALWAYS_ASSERT(nedges == 6);
+  for (int i = 0; i < nedges; i++) {
+    int nodesOnEdge = fs->countNodesOn(mesh->getType(edges[i]));
+    if ( crv::isBoundaryEntity(mesh, edges[i]) ) {
+      for (int n = 0; n < nodesOnEdge; n++) {
+        marker_list[(i*nodesOnEdge)+n] = -1;
+      }
+    }
+  }
+  int nodesOnEdges = fs->countNodesOn(apf::Mesh::TRIANGLE) * nedges;
+
+  apf::Downward faces;
+  int nfaces = mesh->getDownward(e, 2, faces);
+  PCU_ALWAYS_ASSERT(nedges == 4);
+  for (int i = 0; i < nfaces; i++) {
+    int nodesOnFace = fs->countNodesOn(mesh->getType(faces[i]));
+    if ( crv::isBoundaryEntity(mesh, faces[i]) ) {
+      for (int n = 0; n < nodesOnEdge; n++) {
+        marker_list[(nodesOnEdges + i*nodesOnFace)+n] = -1;
+      }
+    }
+  }
+
+  int num_marked = 0;
+  for (int i = 0; i < nedofs; i++) {
+    if (marker_list[i])
+      num_marked++;
+  }
+
+  // use marker list to get ess_dofs list
+  ess_dofs.allocated() ? ess_dofs.resize(num_marked)
+    : ess_dofs.allocate(num_marked);
+  uness_dofs.allocated() ? uness_dofs.resize(nedofs-num_marked)
+    : uness_dofs.allocate(nedofs-num_marked);
+  int ess_dof_counter = 0;
+  int uness_dof_counter = 0;
+  for (int i = 0; i < nedofs; i++) {
+    if(marker_list[i])
+      ess_dofs[ess_dof_counter++] = i;
+    else
+      uness_dofs[uness_dofs_counter++] = i;
+  }
+}
+ 
+/**
+ * Inputs: Matrix A, Vector X, Vector B, essential dofs, not essential dofs.
+ * Output: reduced matrix A, reduced rhs B.
+ */
+static void eliminateDBCs(
+    mth::Matrix<double> const A,
+    mth::Vector<double> const X,
+    mth::Vector<double> const B, 
+    apf::NewArray<int> ess_dofs,
+    apf::NewArray<int> uness_dofs,
+    mth::Matrix<double>& Anew,
+    mth::Vector<double>& Bnew)
+{
+  int num_ess_dofs = ess_dofs.size();
+  int num_uness_dofs = uness_dofs.size();
+
+  // 1. Remove rows and cols of A corresponding to
+	// ess_dofs by copying into Anew
+  Anew.resize(num_uness_dofs, num_uness_dofs);
+	for(int rr = 0; rr < num_uness_dofs; rr++) {
+    int i = uness_dofs[rr];
+    for(int cc = 0; cc < num_uness_dofs; cc++) {
+      int j = uness_dofs[cc];
+      Anew(rr,cc) = A(i,j);
+    }    
+  }
+
+	// 2. Assemble new B
+	Bnew.resize(num_uness_dofs, num_uness_dofs);
+  for(int i = 0; i < num_uness_dofs; i++) {
+    Bnew(i) = B(uness_dofs[i]);
+  }
+
+	// 3. Subtract from Bnew: (Bnew -= Ae*Xe)
+  mth::Matrix<double> Ae(num_uness_dofs, num_ess_dofs);
+  for(int rr = 0; rr < num_uness_dofs; rr++) {
+    int i = uness_dofs[rr];
+    for(int cc = 0; cc < num_ess_dofs; cc++) {
+      int j = ess_dofs[cc];
+      Ae(rr,cc) = A(i,j);
+    }    
+  }
+
+  mth::Vector<double> Xe(num_ess_dofs);
+  for(int i = 0; i < num_ess_dofs; i++) {
+    Xe(i) = X(ess_dofs[i]);
+  }
+
+  mth::Vector<double> temp;
+  mth::multiply(Ae, Xe, temp);
+  Bnew -= temp;	
+}
+
+static double computeL2Error(apf::Mesh* mesh, apf::MeshEntity* e,
+	apf::Field* f, mth::Vector<double> const error_dofs)
+{
+	double error = 0.0;
+
+  apf::FieldShape* fs = f->getShape();
+  int type = mesh->getType(e);
+	PCU_ALWAYS_ASSERT(type == apf::Mesh::TET);
+  int nd = apf::countElementNodes(fs, type);
+  int dim = apf::getDimension(mesh, e);
+  double w;
+
+  apf::MeshElement* me = apf::createMeshElement(mesh, e);
+  apf::Element* el = apf::createElement(f, me);
+  int order = 2 * fs->getOrder();
+  int np = apf::countIntPoints(me, order);
+
+  apf::NewArray<apf::Vector3> vectorshape(nd);
+
+  for (int i = 0; i < np; i++) {
+    apf::getIntPoint(me, order, i, p);
+    double weight = apf::getIntWeight(me, order, i);
+    apf::Matrix3x3 J;
+    apf::getJacobian(me, p, J);
+    double jdet = apf::getJacobianDeterminant(J, dim);
+    w = weight * jdet;
+	
+    apf::getVectorShapeValues(el, p, vectorshape);
+		// TODO take care of negative dof values
+    mth::Matrix<double> vectorShape (nd, dim);
+    for (int j = 0; j < nd; j++)
+      for (int k = 0; k < dim; k++)
+        vectorShape(j,k) = vectorshape[j][k];
+
+		apf::Matrix<double> vectorShapeT (dim, nd);
+    mth::transpose(vectorShape, vectorShapeT);
+		
+		mth::Vector<double> err_func;
+		mth::multiply(vectorShapeT, error_dofs, err_func);
+
+		error += w * (err_func * err_func);
+	}
+	if (error < 0.0)
+		error = -error;
+
+	return sqrt(error);
+}
+
 // @ apf::Field* ef --> input p order ND electric field
 // @ apf::Field* ef --> input theta corrected flux field
 // *** p+1 order field is created inside this function
 apf::Field* emEstimateError(apf::Field* ef, apf::Field* correctedFlux)
 {
-  // 1. Create one order higher ND field
+
+	// 1. Create per-element SCALAR error field
+  apf::Field* error_field = apf::createIPField(
+		apf::getMesh(ef), "residual_error_field", apf::SCALAR, 1); // TODO maybe use getConstant here
+	
+  // 2. Create one order higher ND field
   int order = ef->getShape()->getOrder();
   int orderp1 = order+1;
-  apf::Field* efp1 = apf::createField(
-      apf::getMesh(ef), "higher_order_nedelec_field", apf::SCALAR, apf::getNedelec(orderp1));
+  apf::Field* efp1 = apf::createField(apf::getMesh(ef),
+	 "orderp1_nedelec_field", apf::SCALAR, apf::getNedelec(orderp1));
   apf::zeroField(efp1);
 
   // 2. iterate over all elements of the mesh
@@ -298,8 +463,8 @@ apf::Field* emEstimateError(apf::Field* ef, apf::Field* correctedFlux)
   apf::MeshIterator* it = apf::getMesh(ef)->begin(3);
   while ((el = apf::getMesh(ef)->iterate(it))) {
     // 2(a). Assemble LHS element matrix
-    mth::Matrix<double> lhs;
-    assembleElementMatrix( apf::getMesh(ef), el, efp1, lhs);
+    mth::Matrix<double> A;
+    assembleElementMatrix( apf::getMesh(ef), el, efp1, A);
     // TODO Take care of negative dofs in lhs
 
     // 2(b). Compute Bilinear Form Vector 
@@ -316,13 +481,45 @@ apf::Field* emEstimateError(apf::Field* ef, apf::Field* correctedFlux)
     mth::Vector<double> lambda;
     computeLambdaVector(apf::getMesh(efp1), e, ef, efp1, correctedFlux, lambda);
     // TODO Take care of negative dofs in lambda
-   
-     
-
-
+    
+    // 2(e). Assemble RHS element vector = blf - lf - lambda
+    mth::Vector<double> B = blf - lf - lambda; // TODO syntax
+
+    // 2(f). Get List of Essential Dofs
+    apf::NewArray<int> ess_dofs, uness_dofs;
+    getEssentialBdrElementNDDofs(
+        apf::getMesh(efp1), e, efp1, ess_dofs, uness_dofs);
+
+    // 2(g). eliminate Dirichlet (Essential) Boundary Conditions
+		mth::Vector<double> X, Anew, Bnew;
+		X.resize(B.size());
+		X.zero(); // initialize X with exact DBC (e = 0.0)
+		eliminateDBCs(A, X, B, ess_dofs, uness_dofs, Anew, Bnew);
+
+		// 2(h). Solve the reduced system
+	  mth::Matrix<double> Q, R;
+		mth::decompose(Anew, Q, R);
+		mth::Vector<double> Xnew;
+		mth::solveFromQR(Q, R, Bnew, Xnew);
+
+		// 2(i). Recover the solution
+		mth::Vector<double> error_dofs(B.size());
+    for(int i = 0; i < ess_dofs.size(); i++) {
+    	int index = ess_dofs[i];
+      error_dofs(index) = X(index);
+    }
+    for(int i = 0; i < uness_dofs.size(); i++) {
+      int index = uness_dofs[i];
+      error_dofs(index) = Xnew(i);
+    }
+
+		// 2(j). Compute L2 Norm Error
+		double l2_error = computeL2Error(mesh, e, efp1, error_dofs);
+		apf::setScalar(error_field, e, 0, l2_error);
   }
   apf::getMesh(ef)->end(it);
 
+	apf::destroyField(efp1);
 
-
+	return error_field;
 }

From 99869fc51829f0b67412150c694ac524590e6158 Mon Sep 17 00:00:00 2001
From: Morteza HS <hakimi1364@gmail.com>
Date: Wed, 13 May 2020 20:38:29 -0400
Subject: [PATCH 159/555] Fixes rotateTetXi

Previously this was implemented exactly the same way as unrotateTetXi,
which was not right. The current implementation does the intended task,
which is basically the inverse of unrotateTetXi. Therefore, given a xi
vector, calling unrotateTetXi(xi, rotation) and then rotateTetXi(xi,
rotation) should make xi to have its original coordinates.
---
 ma/maMesh.cc   | 10 +++++-----
 ma/maTables.cc | 18 ++++++++++++++++++
 ma/maTables.h  |  3 +++
 3 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/ma/maMesh.cc b/ma/maMesh.cc
index 348fbe220..7d9b5876f 100644
--- a/ma/maMesh.cc
+++ b/ma/maMesh.cc
@@ -141,13 +141,13 @@ void unrotateTetXi(Vector& xi, int rotation)
 void rotateTetXi(Vector& xi, int rotation)
 {
   PCU_ALWAYS_ASSERT(rotation >= 0 && rotation < 12);
-  double b[4];
-  b[0] = 1-xi[0]-xi[1]-xi[2]; b[1] = xi[0]; b[2] = xi[1]; b[3] = xi[2];
-  int const* originalIndexOf = tet_rotation[rotation];
   double a[4];
+  a[0] = 1-xi[0]-xi[1]-xi[2]; a[1] = xi[0]; a[2] = xi[1]; a[3] = xi[2];
+  int const* inverseIndexOf = tet_inv_rotation[rotation];
+  double b[4];
   for (int i = 0; i < 4; i++)
-    a[ originalIndexOf[i] ] = b[i];
-  xi[0] = a[1]; xi[1] = a[2]; xi[2] = a[3];
+    b[ inverseIndexOf[i] ] = a[i];
+  xi[0] = b[1]; xi[1] = b[2]; xi[2] = b[3];
 }
 
 void rotateOct(Entity** iv, int n, Entity** ov)
diff --git a/ma/maTables.cc b/ma/maTables.cc
index 7d80ad034..8a5f7eb7d 100644
--- a/ma/maTables.cc
+++ b/ma/maTables.cc
@@ -65,6 +65,24 @@ int const tet_rotation[12][4] =
 ,{3,1,0,2}//11
 };
 
+/* un-rotates a rotated tet to get the original. This can be
+   thought of as the inverse of tet_rotation. */
+
+int const tet_inv_rotation[12][4] =
+{{0,1,2,3} //{0,1,2,3}//0
+,{0,3,1,2} //{0,2,3,1}//1
+,{0,2,3,1} //{0,3,1,2}//2
+,{1,0,3,2} //{1,0,3,2}//3
+,{3,0,2,1} //{1,3,2,0}//4
+,{2,0,1,3} //{1,2,0,3}//5
+,{1,2,0,3} //{2,0,1,3}//6
+,{3,1,0,2} //{2,1,3,0}//7
+,{2,3,0,1} //{2,3,0,1}//8
+,{1,3,2,0} //{3,0,2,1}//9
+,{3,2,1,0} //{3,2,1,0}//10
+,{2,1,3,0} //{3,1,0,2}//11
+};
+
 /* these are the canonical edge split
    configurations that form the first
    layer of filtering for tet refinement
diff --git a/ma/maTables.h b/ma/maTables.h
index 64666ad5d..40dd38a1c 100644
--- a/ma/maTables.h
+++ b/ma/maTables.h
@@ -30,6 +30,9 @@ extern CodeMatch const* code_match[apf::Mesh::TYPES];
 /* this table defines the mapping from new to old
    vertex indices for one of the 12 rotations. */
 extern int const tet_rotation[12][4];
+/* this table defines the inverse mapping from new to old
+   vertex indices for one of the 12 rotations. */
+extern int const tet_inv_rotation[12][4];
 extern int const prism_rotation[6][6];
 extern int const pyramid_rotation[4][5];
 

From ddfb58605511fa1dbfce1ab334fc2978c3604729 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Thu, 14 May 2020 15:20:47 -0400
Subject: [PATCH 160/555] Fixes compilation and syntax errors

---
 em/em.h               |  1 +
 em/emEstimateError.cc | 78 ++++++++++++++++++++++++-------------------
 2 files changed, 44 insertions(+), 35 deletions(-)

diff --git a/em/em.h b/em/em.h
index 0ec92f109..8661945b7 100644
--- a/em/em.h
+++ b/em/em.h
@@ -13,6 +13,7 @@
  *  \brief The Elegtromagnetics Equilibrated Residual error estimator inteface
  */
 #include "apf.h"
+#include <apfMesh.h>
 
 #include <mthQR.h>
 #include <mth.h>
diff --git a/em/emEstimateError.cc b/em/emEstimateError.cc
index 4a8a9284c..f99fa931c 100644
--- a/em/emEstimateError.cc
+++ b/em/emEstimateError.cc
@@ -64,7 +64,7 @@ static void computeResidualBLF(apf::Mesh* mesh, apf::MeshEntity* e,
     double weight = apf::getIntWeight(me, order, i);
     apf::Matrix3x3 J;
     apf::getJacobian(me, p, J);
-    double jdet = apf::getJacobianDeterminant(J, dim);
+    //double jdet = apf::getJacobianDeterminant(J, dim);
     w = weight; // TODO check why do not need division by jdet
 
     // get curl vector
@@ -140,7 +140,6 @@ static void computeLambdaVector(apf::Mesh* mesh, apf::MeshEntity* e,
   int etype = mesh->getType(e);
   PCU_ALWAYS_ASSERT(etype == apf::Mesh::TET);
   int nedofs = apf::countElementNodes(fp1s, etype);
-  int edim = apf::getDimension(mesh, e);
   lambda.resize(nedofs);
 
   // create a 2D nedelec field 
@@ -179,12 +178,15 @@ static void computeLambdaVector(apf::Mesh* mesh, apf::MeshEntity* e,
     // 3. get theta coeffs on the face
     double components[3];
     apf::getComponents(THETA, face, 0, components);
-    mth::Vector<double> theta_coeffs(components); // TODO Warning make this work
+    //mth::Vector<double> theta_coeffs(components); // TODO clean
+    mth::Vector<double> theta_coeffs(*components); // TODO clean
+    /*theta_coeffs(0) = components[0];
+    theta_coeffs(1) = components[1];
+    theta_coeffs(2) = components[2];*/
 
     int ftype = mesh->getType(face);
     PCU_ALWAYS_ASSERT(ftype == apf::Mesh::TRIANGLE);
     int nfdofs = apf::countElementNodes(faceNDFieldShape, ftype);
-    int fdim = apf::getDimension(mesh, face);
     apf::NewArray<apf::Vector3> vectorshape(nfdofs);
 
     apf::MeshElement* fme = apf::createMeshElement(mesh, face);
@@ -206,10 +208,10 @@ static void computeLambdaVector(apf::Mesh* mesh, apf::MeshEntity* e,
       // evaluate theta vector using theta coeffs
       apf::Vector3 theta_vector;
       theta_vector.zero();
-      apf::NewArray<apf::Vector3> vector2Dshapes (nfdods);
+      apf::NewArray<apf::Vector3> vector2Dshapes (nfdofs);
       apf::getVectorShapeValues(fel, p, vector2Dshapes);
       int which, rotate; bool flip; // negative ND dofs
-      for (int i = 0; i < theta_coeffs.size(); i++) {
+      for (int i = 0; i < nedges; i++) {
         apf::getAlignment(mesh, face, edges[i], which, flip, rotate);
         apf::Vector3 v = vector2Dshapes[i];
         if (flip) { v = v * -1.; }
@@ -268,9 +270,9 @@ static void computeLambdaVector(apf::Mesh* mesh, apf::MeshEntity* e,
       // TODO take care of negative ND dofs
       apf::destroyMeshElement(me); 
       // compute integral
-      apf::Vector3 theta_plus_tk = theta_vec + tk;
+      apf::Vector3 theta_plus_tk = theta_vector + tk;
       double w = weight * jdet;
-      theta_plus_tk *= w;
+      theta_plus_tk = theta_plus_tk * w;
 
       // matrix vector multiplication
       for (int i = 0; i < nedofs; i++)
@@ -313,7 +315,7 @@ static void getEssentialBdrElementNDDofs(apf::Mesh* mesh, apf::MeshEntity* e,
   for (int i = 0; i < nfaces; i++) {
     int nodesOnFace = fs->countNodesOn(mesh->getType(faces[i]));
     if ( crv::isBoundaryEntity(mesh, faces[i]) ) {
-      for (int n = 0; n < nodesOnEdge; n++) {
+      for (int n = 0; n < nodesOnFace; n++) {
         marker_list[(nodesOnEdges + i*nodesOnFace)+n] = -1;
       }
     }
@@ -336,7 +338,7 @@ static void getEssentialBdrElementNDDofs(apf::Mesh* mesh, apf::MeshEntity* e,
     if(marker_list[i])
       ess_dofs[ess_dof_counter++] = i;
     else
-      uness_dofs[uness_dofs_counter++] = i;
+      uness_dofs[uness_dof_counter++] = i;
   }
 }
  
@@ -345,13 +347,13 @@ static void getEssentialBdrElementNDDofs(apf::Mesh* mesh, apf::MeshEntity* e,
  * Output: reduced matrix A, reduced rhs B.
  */
 static void eliminateDBCs(
-    mth::Matrix<double> const A,
-    mth::Vector<double> const X,
-    mth::Vector<double> const B, 
-    apf::NewArray<int> ess_dofs,
-    apf::NewArray<int> uness_dofs,
-    mth::Matrix<double>& Anew,
-    mth::Vector<double>& Bnew)
+    mth::Matrix<double> const &A,
+    mth::Vector<double> const &X,
+    mth::Vector<double> const &B,
+    apf::NewArray<int>  const &ess_dofs,
+    apf::NewArray<int>  const &uness_dofs,
+    mth::Matrix<double> &Anew,
+    mth::Vector<double> &Bnew)
 {
   int num_ess_dofs = ess_dofs.size();
   int num_uness_dofs = uness_dofs.size();
@@ -364,11 +366,11 @@ static void eliminateDBCs(
     for(int cc = 0; cc < num_uness_dofs; cc++) {
       int j = uness_dofs[cc];
       Anew(rr,cc) = A(i,j);
-    }    
+    }
   }
 
 	// 2. Assemble new B
-	Bnew.resize(num_uness_dofs, num_uness_dofs);
+	Bnew.resize(num_uness_dofs);
   for(int i = 0; i < num_uness_dofs; i++) {
     Bnew(i) = B(uness_dofs[i]);
   }
@@ -412,6 +414,7 @@ static double computeL2Error(apf::Mesh* mesh, apf::MeshEntity* e,
 
   apf::NewArray<apf::Vector3> vectorshape(nd);
 
+  apf::Vector3 p;
   for (int i = 0; i < np; i++) {
     apf::getIntPoint(me, order, i, p);
     double weight = apf::getIntWeight(me, order, i);
@@ -427,7 +430,7 @@ static double computeL2Error(apf::Mesh* mesh, apf::MeshEntity* e,
       for (int k = 0; k < dim; k++)
         vectorShape(j,k) = vectorshape[j][k];
 
-		apf::Matrix<double> vectorShapeT (dim, nd);
+    mth::Matrix<double> vectorShapeT (dim, nd);
     mth::transpose(vectorShape, vectorShapeT);
 		
 		mth::Vector<double> err_func;
@@ -450,7 +453,7 @@ apf::Field* emEstimateError(apf::Field* ef, apf::Field* correctedFlux)
 	// 1. Create per-element SCALAR error field
   apf::Field* error_field = apf::createIPField(
 		apf::getMesh(ef), "residual_error_field", apf::SCALAR, 1); // TODO maybe use getConstant here
-	
+
   // 2. Create one order higher ND field
   int order = ef->getShape()->getOrder();
   int orderp1 = order+1;
@@ -467,55 +470,58 @@ apf::Field* emEstimateError(apf::Field* ef, apf::Field* correctedFlux)
     assembleElementMatrix( apf::getMesh(ef), el, efp1, A);
     // TODO Take care of negative dofs in lhs
 
-    // 2(b). Compute Bilinear Form Vector 
+    // 2(b). Compute Bilinear Form Vector
     mth::Vector<double> blf;
-    computeResidualBLF(apf::getMesh(efp1), e, efp1, blf);
+    computeResidualBLF(apf::getMesh(efp1), el, efp1, blf);
     // TODO Take care of negative dofs in blf
 
     // 2(c). Compute Linear Form Vector
     mth::Vector<double> lf;
-    assembleDomainLFElementVector(apf::getMesh(efp1), e, efp1, lf);
+    assembleDomainLFElementVector(apf::getMesh(efp1), el, efp1, lf);
     // TODO Take care of negative dofs in lf
-    
+
     // 2(d). Compute Lamda Vector
     mth::Vector<double> lambda;
-    computeLambdaVector(apf::getMesh(efp1), e, ef, efp1, correctedFlux, lambda);
+    computeLambdaVector(apf::getMesh(ef), el, ef, efp1, correctedFlux, lambda);
     // TODO Take care of negative dofs in lambda
-    
+
     // 2(e). Assemble RHS element vector = blf - lf - lambda
-    mth::Vector<double> B = blf - lf - lambda; // TODO syntax
+    mth::Vector<double> B(blf.size());
+    B.zero();
+    B += blf; B -= lf; B -= lambda;
 
     // 2(f). Get List of Essential Dofs
     apf::NewArray<int> ess_dofs, uness_dofs;
     getEssentialBdrElementNDDofs(
-        apf::getMesh(efp1), e, efp1, ess_dofs, uness_dofs);
+        apf::getMesh(efp1), el, efp1, ess_dofs, uness_dofs);
 
     // 2(g). eliminate Dirichlet (Essential) Boundary Conditions
-		mth::Vector<double> X, Anew, Bnew;
+		mth::Vector<double> X, Bnew;
+    mth::Matrix<double> Anew;
 		X.resize(B.size());
 		X.zero(); // initialize X with exact DBC (e = 0.0)
 		eliminateDBCs(A, X, B, ess_dofs, uness_dofs, Anew, Bnew);
 
 		// 2(h). Solve the reduced system
 	  mth::Matrix<double> Q, R;
-		mth::decompose(Anew, Q, R);
+		mth::decomposeQR(Anew, Q, R);
 		mth::Vector<double> Xnew;
 		mth::solveFromQR(Q, R, Bnew, Xnew);
 
 		// 2(i). Recover the solution
 		mth::Vector<double> error_dofs(B.size());
-    for(int i = 0; i < ess_dofs.size(); i++) {
+    for(unsigned int i = 0; i < ess_dofs.size(); i++) {
     	int index = ess_dofs[i];
       error_dofs(index) = X(index);
     }
-    for(int i = 0; i < uness_dofs.size(); i++) {
+    for(unsigned int i = 0; i < uness_dofs.size(); i++) {
       int index = uness_dofs[i];
       error_dofs(index) = Xnew(i);
     }
 
 		// 2(j). Compute L2 Norm Error
-		double l2_error = computeL2Error(mesh, e, efp1, error_dofs);
-		apf::setScalar(error_field, e, 0, l2_error);
+		double l2_error = computeL2Error(apf::getMesh(ef), el, efp1, error_dofs);
+		apf::setScalar(error_field, el, 0, l2_error);
   }
   apf::getMesh(ef)->end(it);
 
@@ -523,3 +529,5 @@ apf::Field* emEstimateError(apf::Field* ef, apf::Field* correctedFlux)
 
 	return error_field;
 }
+
+}

From ab67224991fe5e3f524dd64770420d74517fc79d Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Thu, 14 May 2020 16:19:01 -0400
Subject: [PATCH 161/555] Cleans #includes in the em directory

---
 em/em.h                     |  9 +++++++++
 em/emEstimateError.cc       | 21 ---------------------
 em/emFluxCorrection.cc      | 21 ---------------------
 em/emResidualFunctionals.cc | 20 +-------------------
 4 files changed, 10 insertions(+), 61 deletions(-)

diff --git a/em/em.h b/em/em.h
index 8661945b7..c58c7a40d 100644
--- a/em/em.h
+++ b/em/em.h
@@ -15,6 +15,15 @@
 #include "apf.h"
 #include <apfMesh.h>
 
+#include <PCU.h>
+#include <lionPrint.h>
+
+#include "apfShape.h"
+#include "apfField.h"
+#include "apfElement.h"
+#include "crv.h"
+#include "crvShape.h"
+
 #include <mthQR.h>
 #include <mth.h>
 #include <mth_def.h>
diff --git a/em/emEstimateError.cc b/em/emEstimateError.cc
index f99fa931c..148c04646 100644
--- a/em/emEstimateError.cc
+++ b/em/emEstimateError.cc
@@ -6,27 +6,6 @@
  */
 #include <iostream>
 #include <cstdlib>
-#include <apf.h>
-#include <PCU.h>
-#include <lionPrint.h>
-
-#include "em.h"
-
-#include <apfMesh.h>
-#include <apfShape.h>
-#include <apfField.h>
-#include <apfElement.h>
-#include <apfCavityOp.h>
-#include "crv.h"
-#include "crvShape.h"
-
-#include <mthQR.h>
-#include <mth.h>
-#include <mth_def.h>
-#include <limits>
-
-#include <set>
-#include <pcu_util.h>
 
 #include "em.h"
 
diff --git a/em/emFluxCorrection.cc b/em/emFluxCorrection.cc
index c1dfbff47..c621cc426 100644
--- a/em/emFluxCorrection.cc
+++ b/em/emFluxCorrection.cc
@@ -6,27 +6,6 @@
  */
 #include <iostream>
 #include <cstdlib>
-#include <apf.h>
-#include <PCU.h>
-#include <lionPrint.h>
-
-#include "em.h"
-
-#include <apfMesh.h>
-#include <apfShape.h>
-#include <apfField.h>
-#include <apfElement.h>
-#include <apfCavityOp.h>
-#include "crv.h"
-#include "crvShape.h"
-
-#include <mthQR.h>
-#include <mth.h>
-#include <mth_def.h>
-#include <limits>
-
-#include <set>
-#include <pcu_util.h>
 
 #include "em.h"
 
diff --git a/em/emResidualFunctionals.cc b/em/emResidualFunctionals.cc
index 6c5fd4698..c00559ff2 100644
--- a/em/emResidualFunctionals.cc
+++ b/em/emResidualFunctionals.cc
@@ -6,28 +6,10 @@
  */
 #include <iostream>
 #include <cstdlib>
-#include <apf.h>
-#include <PCU.h>
-#include <lionPrint.h>
 
-#include "em.h"
-
-#include <apfMesh.h>
-#include <apfShape.h>
-#include <apfField.h>
-#include <apfElement.h>
 #include <apfCavityOp.h>
-#include "crv.h"
-#include "crvShape.h"
-
-#include <mthQR.h>
-#include <mth.h>
-#include <mth_def.h>
 
-#include <limits>
-
-#include <set>
-#include <pcu_util.h>
+#include "em.h"
 
 namespace em {
 

From e3a932b7e69cb3d879cbcc7ecabbf1cc8104b9b5 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Sat, 16 May 2020 21:36:46 -0400
Subject: [PATCH 162/555] Cleans 2D vector shapes Piola transformation

---
 apf/apf.cc | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/apf/apf.cc b/apf/apf.cc
index 043d3a0ff..7a9538507 100644
--- a/apf/apf.cc
+++ b/apf/apf.cc
@@ -485,8 +485,7 @@ void getVectorShapeValues(Element* e, Vector3 const& local,
     // TODO clean up
     apf::Matrix3x3 J;
     apf::getJacobian(e->getParent(), local, J);
-    std::cout << "J (3x3) last row zero expected" << std::endl;
-    std::cout << J << std::endl;
+
     apf::Matrix3x3 JT = apf::transpose(J);
     apf::Matrix3x3 JJT = J * JT;
     // take inverse of JTJ and transpose it
@@ -496,8 +495,7 @@ void getVectorShapeValues(Element* e, Vector3 const& local,
         jjt[i][j] = JJT[i][j];
     apf::Matrix<2,2> JJTinv = apf::invert(jjt);
     apf::Matrix<2,2> JJTinvT = apf::transpose(JJTinv);
-    std::cout << "JJTinvT (2x2)" << std::endl;
-    std::cout << JJTinvT << std::endl;
+
     apf::Matrix<2,3> JJTinvTJ; // JJTinvT * J
     for( int i = 0; i < 2; i++ ) {
       for ( int j = 0; j < 3; j++ ) {
@@ -506,6 +504,7 @@ void getVectorShapeValues(Element* e, Vector3 const& local,
           JJTinvTJ[i][j] += JJTinvT[i][k] * J[k][j];
       }
     }
+
     // u(x_hat) * J(x_hat)^{-1}
     for( size_t i = 0; i < values.size(); i++ ) {
       for ( int j = 0; j < 3; j++ ) {
@@ -514,8 +513,6 @@ void getVectorShapeValues(Element* e, Vector3 const& local,
           values[i][j] += vvals[i][k] * JJTinvTJ[k][j];
       }
     }
-    /*PCU_ALWAYS_ASSERT_VERBOSE(false,
-    	"not yet implemented for 3D surface meshes (i.e., manifolds)!");*/
   }
 }
 

From 7fb121cfbf8de3c66cebbe85b1da0e8460b6a4d4 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Sat, 16 May 2020 21:39:05 -0400
Subject: [PATCH 163/555] Fixes header files and includes

---
 em/emEstimateError.cc | 96 ++++++++++++++++++++++++++++---------------
 1 file changed, 64 insertions(+), 32 deletions(-)

diff --git a/em/emEstimateError.cc b/em/emEstimateError.cc
index 148c04646..07c82e9c6 100644
--- a/em/emEstimateError.cc
+++ b/em/emEstimateError.cc
@@ -7,8 +7,12 @@
 #include <iostream>
 #include <cstdlib>
 
-#include "em.h"
+#include "apfElement.h"
+#include "crv.h"
+#include "crvShape.h"
 
+#include "em.h"
+using namespace std;
 namespace em {
 //TODO destroy all fields created inside functions to prevent memory leaks.
 
@@ -108,12 +112,17 @@ static void computeResidualBLF(apf::Mesh* mesh, apf::MeshEntity* e,
   // TODO Take care of Negative Dofs 
 }
 
-// 
-// @ apf::Field* f --> input p-order ND field
-// @ apf::Field* fp1 --> p+1 order ND field created for local BVPs
+// @ apf::Field* f --> input p-order tet ND field
+// @ apf::Field* fp1 --> p+1 order tet ND field created for local BVPs
 // @ apf::Field* THETA --> constant face field containing theta coeffs
-static void computeLambdaVector(apf::Mesh* mesh, apf::MeshEntity* e,
-  apf::Field* f, apf::Field* fp1, apf::Field* THETA, mth::Vector<double>& lambda)
+static void computeLambdaVector(
+    apf::Mesh* mesh,
+    apf::MeshEntity* e,
+    apf::Field* f,
+    apf::Field* fp1,
+    apf::Field* faceNDField,
+    apf::Field* THETA,
+    mth::Vector<double>& lambda)
 {
   apf::FieldShape* fp1s = fp1->getShape();
   int etype = mesh->getType(e);
@@ -123,9 +132,6 @@ static void computeLambdaVector(apf::Mesh* mesh, apf::MeshEntity* e,
 
   // create a 2D nedelec field 
   int order = fp1s->getOrder();
-  apf::Field* faceNDField = apf::createField(
-      mesh, "face_nedelec_field", apf::SCALAR, apf::getNedelec(order));
-  apf::zeroField(faceNDField);
   apf::FieldShape* faceNDFieldShape = faceNDField->getShape();
 
   // get the downward faces of the element
@@ -158,10 +164,11 @@ static void computeLambdaVector(apf::Mesh* mesh, apf::MeshEntity* e,
     double components[3];
     apf::getComponents(THETA, face, 0, components);
     //mth::Vector<double> theta_coeffs(components); // TODO clean
-    mth::Vector<double> theta_coeffs(*components); // TODO clean
-    /*theta_coeffs(0) = components[0];
+    //mth::Vector<double> theta_coeffs(*components); // TODO clean
+    mth::Vector<double> theta_coeffs(3); // TODO clean
+    theta_coeffs(0) = components[0];
     theta_coeffs(1) = components[1];
-    theta_coeffs(2) = components[2];*/
+    theta_coeffs(2) = components[2];
 
     int ftype = mesh->getType(face);
     PCU_ALWAYS_ASSERT(ftype == apf::Mesh::TRIANGLE);
@@ -290,7 +297,7 @@ static void getEssentialBdrElementNDDofs(apf::Mesh* mesh, apf::MeshEntity* e,
 
   apf::Downward faces;
   int nfaces = mesh->getDownward(e, 2, faces);
-  PCU_ALWAYS_ASSERT(nedges == 4);
+  PCU_ALWAYS_ASSERT(nfaces == 4);
   for (int i = 0; i < nfaces; i++) {
     int nodesOnFace = fs->countNodesOn(mesh->getType(faces[i]));
     if ( crv::isBoundaryEntity(mesh, faces[i]) ) {
@@ -354,24 +361,26 @@ static void eliminateDBCs(
     Bnew(i) = B(uness_dofs[i]);
   }
 
-	// 3. Subtract from Bnew: (Bnew -= Ae*Xe)
-  mth::Matrix<double> Ae(num_uness_dofs, num_ess_dofs);
-  for(int rr = 0; rr < num_uness_dofs; rr++) {
-    int i = uness_dofs[rr];
-    for(int cc = 0; cc < num_ess_dofs; cc++) {
-      int j = ess_dofs[cc];
-      Ae(rr,cc) = A(i,j);
-    }    
-  }
+  if (num_ess_dofs > 0) {
+	  // 3. Subtract from Bnew: (Bnew -= Ae*Xe)
+    mth::Matrix<double> Ae(num_uness_dofs, num_ess_dofs);
+    for(int rr = 0; rr < num_uness_dofs; rr++) {
+      int i = uness_dofs[rr];
+      for(int cc = 0; cc < num_ess_dofs; cc++) {
+        int j = ess_dofs[cc];
+        Ae(rr,cc) = A(i,j);
+      }
+    }
 
-  mth::Vector<double> Xe(num_ess_dofs);
-  for(int i = 0; i < num_ess_dofs; i++) {
-    Xe(i) = X(ess_dofs[i]);
-  }
+    mth::Vector<double> Xe(num_ess_dofs);
+    for(int i = 0; i < num_ess_dofs; i++) {
+      Xe(i) = X(ess_dofs[i]);
+    }
 
-  mth::Vector<double> temp;
-  mth::multiply(Ae, Xe, temp);
-  Bnew -= temp;	
+    mth::Vector<double> temp;
+    mth::multiply(Ae, Xe, temp);
+    Bnew -= temp;
+  }
 }
 
 static double computeL2Error(apf::Mesh* mesh, apf::MeshEntity* e,
@@ -433,13 +442,22 @@ apf::Field* emEstimateError(apf::Field* ef, apf::Field* correctedFlux)
   apf::Field* error_field = apf::createIPField(
 		apf::getMesh(ef), "residual_error_field", apf::SCALAR, 1); // TODO maybe use getConstant here
 
-  // 2. Create one order higher ND field
+  cout << "ELEMENT ERROR FIELD CREATED" << endl;
+
+  // 2. Create p+1 order tet ND field
+  // and p order triangle ND field
   int order = ef->getShape()->getOrder();
   int orderp1 = order+1;
   apf::Field* efp1 = apf::createField(apf::getMesh(ef),
 	 "orderp1_nedelec_field", apf::SCALAR, apf::getNedelec(orderp1));
   apf::zeroField(efp1);
 
+  apf::Field* faceNDField = apf::createField(apf::getMesh(ef),
+      "face_nedelec_field", apf::SCALAR, apf::getNedelec(order));
+  apf::zeroField(faceNDField);;
+
+  cout << "P+1 ORDER FIELD CREATED" << endl;
+
   // 2. iterate over all elements of the mesh
   apf::MeshEntity* el;
   apf::MeshIterator* it = apf::getMesh(ef)->begin(3);
@@ -448,31 +466,38 @@ apf::Field* emEstimateError(apf::Field* ef, apf::Field* correctedFlux)
     mth::Matrix<double> A;
     assembleElementMatrix( apf::getMesh(ef), el, efp1, A);
     // TODO Take care of negative dofs in lhs
+    cout << "LHS matrix assembled" << endl;
 
     // 2(b). Compute Bilinear Form Vector
     mth::Vector<double> blf;
     computeResidualBLF(apf::getMesh(efp1), el, efp1, blf);
     // TODO Take care of negative dofs in blf
+    cout << "RHS bilinear form vector assembled" << endl;
 
     // 2(c). Compute Linear Form Vector
     mth::Vector<double> lf;
     assembleDomainLFElementVector(apf::getMesh(efp1), el, efp1, lf);
     // TODO Take care of negative dofs in lf
+    cout << "RHS linear form vector assembled" << endl;
 
     // 2(d). Compute Lamda Vector
     mth::Vector<double> lambda;
-    computeLambdaVector(apf::getMesh(ef), el, ef, efp1, correctedFlux, lambda);
+    computeLambdaVector(
+        apf::getMesh(ef), el, ef, efp1, faceNDField, correctedFlux, lambda);
     // TODO Take care of negative dofs in lambda
+    cout << "RHS lambda vector assembled" << endl;
 
     // 2(e). Assemble RHS element vector = blf - lf - lambda
     mth::Vector<double> B(blf.size());
     B.zero();
     B += blf; B -= lf; B -= lambda;
+    cout << "RHS Vector assembled" << endl;
 
     // 2(f). Get List of Essential Dofs
     apf::NewArray<int> ess_dofs, uness_dofs;
     getEssentialBdrElementNDDofs(
         apf::getMesh(efp1), el, efp1, ess_dofs, uness_dofs);
+    cout << "Get list of Essential Dofs" << endl;
 
     // 2(g). eliminate Dirichlet (Essential) Boundary Conditions
 		mth::Vector<double> X, Bnew;
@@ -480,12 +505,14 @@ apf::Field* emEstimateError(apf::Field* ef, apf::Field* correctedFlux)
 		X.resize(B.size());
 		X.zero(); // initialize X with exact DBC (e = 0.0)
 		eliminateDBCs(A, X, B, ess_dofs, uness_dofs, Anew, Bnew);
+		cout << "eliminate DBCs" << endl;
 
 		// 2(h). Solve the reduced system
 	  mth::Matrix<double> Q, R;
 		mth::decomposeQR(Anew, Q, R);
 		mth::Vector<double> Xnew;
 		mth::solveFromQR(Q, R, Bnew, Xnew);
+		cout << "Solve the reduced system" << endl;
 
 		// 2(i). Recover the solution
 		mth::Vector<double> error_dofs(B.size());
@@ -497,14 +524,19 @@ apf::Field* emEstimateError(apf::Field* ef, apf::Field* correctedFlux)
       int index = uness_dofs[i];
       error_dofs(index) = Xnew(i);
     }
+    cout << "Recover the solution" << endl;
 
 		// 2(j). Compute L2 Norm Error
 		double l2_error = computeL2Error(apf::getMesh(ef), el, efp1, error_dofs);
+		cout << "Compute L2 element error" << endl;
+
 		apf::setScalar(error_field, el, 0, l2_error);
+    cout << "Write the L2 error to error_field" << endl;
   }
   apf::getMesh(ef)->end(it);
-
+  cout << "End loop over elements" << endl;
 	apf::destroyField(efp1);
+  apf::destroyField(faceNDField);
 
 	return error_field;
 }

From 32969f7f28ebdea69ada4a4b89e5b97ed7d16184 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Sat, 16 May 2020 21:40:02 -0400
Subject: [PATCH 164/555] Clean up, renames, and fix headers and includes

---
 em/em.h                     |  21 ---
 em/emResidualFunctionals.cc | 248 ++++++++++++++++++++----------------
 2 files changed, 137 insertions(+), 132 deletions(-)

diff --git a/em/em.h b/em/em.h
index c58c7a40d..84bf238e5 100644
--- a/em/em.h
+++ b/em/em.h
@@ -20,9 +20,6 @@
 
 #include "apfShape.h"
 #include "apfField.h"
-#include "apfElement.h"
-#include "crv.h"
-#include "crvShape.h"
 
 #include <mthQR.h>
 #include <mth.h>
@@ -55,24 +52,6 @@ void assembleElementMatrix(apf::Mesh* mesh, apf::MeshEntity*e,
 void assembleDomainLFElementVector(apf::Mesh* mesh, apf::MeshEntity* e,
     apf::Field* f, mth::Vector<double>& elvect);
 
-// TODO QUESTION redo this to allow user access from outside
-void pumiUserFunction(const apf::Vector3& x, mth::Vector<double>& f,
-    apf::MeshEntity* e, apf::Mesh* mesh)
-{
-  double freq = 1.;
-  double kappa = freq * M_PI;
-  int dim = apf::getDimension(mesh, e);
-  if (dim == 3) {
-      f(0) = (1. + kappa * kappa) * sin(kappa * x[1]);
-      f(1) = (1. + kappa * kappa) * sin(kappa * x[2]);
-      f(2) = (1. + kappa * kappa) * sin(kappa * x[0]);
-  }
-  else {
-     f(0) = (1. + kappa * kappa) * sin(kappa * x[1]);
-     f(1) = (1. + kappa * kappa) * sin(kappa * x[0]);
-     f(2) = 0.0;
-  }
-}
 
 apf::Vector3 computeFaceOutwardNormal(apf::Mesh* m,
     apf::MeshEntity* t, apf::MeshEntity* f, apf::Vector3 const& p);
diff --git a/em/emResidualFunctionals.cc b/em/emResidualFunctionals.cc
index c00559ff2..3e0feaa44 100644
--- a/em/emResidualFunctionals.cc
+++ b/em/emResidualFunctionals.cc
@@ -8,9 +8,14 @@
 #include <cstdlib>
 
 #include <apfCavityOp.h>
+#include "apfElement.h"
+#include "crv.h"
+#include "crvShape.h"
 
 #include "em.h"
 
+using namespace std;
+
 namespace em {
 
 /* overall information useful during equilibration */
@@ -23,7 +28,7 @@ struct Equilibration {
   /* input scalar field containing Nedelec dofs for electric field */
   apf::Field* ef;
   /* output field containing correction values */
-  apf::Field* g; 
+  apf::Field* g;
 };
 
 static void setupEquilibration(Equilibration* eq, apf::Field* f, apf::Field* g)
@@ -33,6 +38,14 @@ static void setupEquilibration(Equilibration* eq, apf::Field* f, apf::Field* g)
   eq->ef = f;
   eq->order = f->getShape()->getOrder();
   eq->g = g;
+
+  double zeros[3] = {0., 0., 0.};
+  apf::MeshEntity* face;
+  apf::MeshIterator* it = apf::getMesh(f)->begin(2);
+  while ((face = apf::getMesh(f)->iterate(it))) {
+    apf::setComponents(eq->g, face, 0, zeros);
+  }
+  apf::getMesh(f)->end(it);
 }
 
 struct QRDecomp {
@@ -56,7 +69,6 @@ struct EdgePatch {
   mth::Vector<double> b;
   mth::Vector<double> x;
   QRDecomp qr;
-
 };
 
 static void setupEdgePatch(EdgePatch* p, Equilibration* eq)
@@ -130,11 +142,10 @@ static bool buildEdgePatch(EdgePatch* p, apf::CavityOp* o)
   return true;
 }
 
-static void assembleLHS(EdgePatch* p)
+static void assembleEdgePatchLHS(EdgePatch* p)
 {
   int ne = p->tets.size();
   int nf = p->faces.size();
-  //printf("ne %d nf %d \n", ne, nf); // REMOVE
   if( crv::isBoundaryEntity(p->mesh, p->entity) ) {
     p->A.resize(ne+nf, ne+nf);
     p->A.zero();
@@ -146,9 +157,6 @@ static void assembleLHS(EdgePatch* p)
     }
     p->A(ne+nf-1, ne-1) = 1.; p->A(ne+nf-1, ne) = 1.;
     p->A(ne-1, ne+nf-1) = 1.; p->A(ne, ne+nf-1) = 1.;
-
-    //std::cout << "boundary" << std::endl; // REMOVE
-    //std::cout << p->A << std::endl; // REMOVE
   }
   else if( ! crv::isBoundaryEntity(p->mesh, p->entity) ) {
     mth::Matrix<double> m(ne, nf);
@@ -167,46 +175,41 @@ static void assembleLHS(EdgePatch* p)
     for (int i = 0; i < ne; i++)
       for (int j = 0; j < ne; j++)
         p->A(i,j) += 1.;
-    //std::cout << "interior" << std::endl; // REMOVE
-    //std::cout << p->A << std::endl; // REMOVE
   }
   mth::decomposeQR(p->A, p->qr.Q, p->qr.R);
-  /*std::cout << "Q" << std::endl;
-  std::cout << p->qr.Q << std::endl;
-  std::cout << "R" << std::endl;
-  std::cout << p->qr.R << std::endl;*/
 }
 
-void assembleCurlCurlElementMatrix(apf::Mesh* mesh,apf::MeshEntity* e,
+void assembleCurlCurlElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
     apf::Field* f, mth::Matrix<double>& elmat)
 {
   apf::FieldShape* fs = f->getShape();
   int type = mesh->getType(e);
+  PCU_ALWAYS_ASSERT(type == apf::Mesh::TET); // TODO add Triangle
   int nd = apf::countElementNodes(fs, type);
   int dim = apf::getDimension(mesh, e);
   int dimc = (dim == 3) ? 3 : 1;
   double w;
 
   apf::NewArray<apf::Vector3> curlshape(nd);
-  mth::Matrix<double> phys_curlshape(nd, dimc);
+  mth::Matrix<double> phys_curlshape(nd, dimc); // TODO remove once curl Piola is in place in apfNedelec
   elmat.resize(nd,nd);
 
   apf::MeshElement* me = apf::createMeshElement(mesh, e);
   apf::Element* el = apf::createElement(f, me);
-  int order = 2 * fs->getOrder() - 2;
-  int np = apf::countIntPoints(me, order); // int points required
+  int int_order = 2 * fs->getOrder() - 2;
+  int np = apf::countIntPoints(me, int_order);
 
   elmat.zero();
   apf::Vector3 p;
   for (int i = 0; i < np; i++) {
-    apf::getIntPoint(me, order, i, p);
-    double weight = apf::getIntWeight(me, order, i);
+    apf::getIntPoint(me, int_order, i, p);
+    double weight = apf::getIntWeight(me, int_order, i);
     apf::Matrix3x3 J;
     apf::getJacobian(me, p, J);
     double jdet = apf::getJacobianDeterminant(J, dim);
     w = weight / jdet;
 
-    if (dim == 3) {
+    if (dim == 3) { // TODO this is Piola transformation. Put it in apfNedelec
       el->getShape()->getLocalVectorCurls(mesh, e, p, curlshape);
       phys_curlshape.zero();
       for (int i = 0; i < nd; i++)
@@ -225,6 +228,7 @@ void assembleCurlCurlElementMatrix(apf::Mesh* mesh,apf::MeshEntity* e,
     }
     mth::Matrix<double> phys_curlshapeT;
     mth::transpose(phys_curlshape, phys_curlshapeT);
+
     mth::Matrix<double> M (nd, nd);
     M.zero();
     mth::multiply(phys_curlshape, phys_curlshapeT, M);
@@ -235,44 +239,46 @@ void assembleCurlCurlElementMatrix(apf::Mesh* mesh,apf::MeshEntity* e,
   apf::destroyMeshElement(me);
 }
 
-void assembleVectorMassElementMatrix(apf::Mesh* mesh,apf::MeshEntity* e,
+void assembleVectorMassElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
     apf::Field* f, mth::Matrix<double>& elmat)
 {
   apf::FieldShape* fs = f->getShape();
   int type = mesh->getType(e);
+  PCU_ALWAYS_ASSERT(type == apf::Mesh::TET || type == apf::Mesh::TRIANGLE);
   int nd = apf::countElementNodes(fs, type);
   int dim = apf::getDimension(mesh, e);
   double w;
 
-  apf::NewArray<apf::Vector3> vectorshape(nd);
+  apf::NewArray<apf::Vector3> vectorshapes(nd);
   elmat.resize(nd,nd);
 
   apf::MeshElement* me = apf::createMeshElement(mesh, e);
   apf::Element* el = apf::createElement(f, me);
-  int order = 2 * fs->getOrder();
-  int np = apf::countIntPoints(me, order); // int points required
+  int int_order = 2 * fs->getOrder();
+  int np = apf::countIntPoints(me, int_order); // int points required
 
   elmat.zero();
   apf::Vector3 p;
   for (int i = 0; i < np; i++) {
-    apf::getIntPoint(me, order, i, p);
-    double weight = apf::getIntWeight(me, order, i);
+    apf::getIntPoint(me, int_order, i, p);
+    double weight = apf::getIntWeight(me, int_order, i);
     apf::Matrix3x3 J;
     apf::getJacobian(me, p, J);
     double jdet = apf::getJacobianDeterminant(J, dim);
     w = weight * jdet;
 
-    apf::getVectorShapeValues(el, p, vectorshape);
-    mth::Matrix<double> vectorShape (nd, dim);
+    apf::getVectorShapeValues(el, p, vectorshapes);
+    mth::Matrix<double> vectorShapes (nd, dim); // TODO opt clean
     for (int j = 0; j < nd; j++)
       for (int k = 0; k < dim; k++)
-        vectorShape(j,k) = vectorshape[j][k];
+        vectorShapes(j,k) = vectorshapes[j][k];
+
+    mth::Matrix<double> vectorShapesT (dim, nd);
+    mth::transpose(vectorShapes, vectorShapesT);
 
-    mth::Matrix<double> vectorShapeT (dim, nd);
-    mth::transpose(vectorShape, vectorShapeT);
     mth::Matrix<double> M (nd,nd);
     M.zero();
-    mth::multiply(vectorShape, vectorShapeT, M);
+    mth::multiply(vectorShapes, vectorShapesT, M);
     M *= w;
     elmat += M;
   }
@@ -286,7 +292,7 @@ void assembleElementMatrix(apf::Mesh* mesh, apf::MeshEntity*e,
   mth::Matrix<double> curl_elmat, mass_elmat;
   assembleCurlCurlElementMatrix(mesh, e, f, curl_elmat);
   assembleVectorMassElementMatrix(mesh, e, f, mass_elmat);
-  
+
   elmat.resize(curl_elmat.rows(), curl_elmat.cols());
   elmat.zero();
   elmat += curl_elmat;
@@ -294,12 +300,13 @@ void assembleElementMatrix(apf::Mesh* mesh, apf::MeshEntity*e,
 }
 
 // computes local bilinear form integral restricted to
-// an edge of an element
-static double getLocalBLFIntegral(EdgePatch* p, apf::MeshEntity* tet)
+// an edge of a tet element
+static double getLocalEdgeBLF(EdgePatch* p, apf::MeshEntity* tet)
 {
-  // find position of edge in downward edges of element
+  // findIn edge in downward edges of tet
   apf::Downward e;
   int ne = p->mesh->getDownward(tet, 1, e);
+  PCU_ALWAYS_ASSERT(ne == 6);
   int ei = apf::findIn(e, ne, p->entity);
   // get Element Dofs
   apf::MeshElement* me = apf::createMeshElement(p->mesh, tet);
@@ -325,74 +332,97 @@ static double getLocalBLFIntegral(EdgePatch* p, apf::MeshEntity* tet)
   elmat += curl_elmat;
   elmat += mass_elmat;
   // multiply element matrix with element dofs
-  mth::Vector<double> integrals (nd);
-  mth::multiply(elmat, dofs, integrals);
+  mth::Vector<double> blf_integrals (nd);
+  mth::multiply(elmat, dofs, blf_integrals);
 
   apf::destroyElement(el);
   apf::destroyMeshElement(me);
 
   // pick edge index from the resulting vector
+  // negation of negative ND dofs
   int which, rotate; bool flip;
   apf::getAlignment(p->mesh, tet, p->entity, which, flip, rotate);
   if (flip)
-    return -1*integrals(ei);
-  return integrals(ei);
+    blf_integrals(ei) = -1*blf_integrals(ei);
+  return blf_integrals(ei);
 }
 
 
+// TODO QUESTION redo this to allow user access from outside
+void pumiUserFunction(const apf::Vector3& x, mth::Vector<double>& f,
+    apf::MeshEntity* e, apf::Mesh* mesh)
+{
+  double freq = 1.;
+  double kappa = freq * M_PI;
+  int dim = apf::getDimension(mesh, e);
+  if (dim == 3) {
+      f(0) = (1. + kappa * kappa) * sin(kappa * x[1]);
+      f(1) = (1. + kappa * kappa) * sin(kappa * x[2]);
+      f(2) = (1. + kappa * kappa) * sin(kappa * x[0]);
+  }
+  else {
+     f(0) = (1. + kappa * kappa) * sin(kappa * x[1]);
+     f(1) = (1. + kappa * kappa) * sin(kappa * x[0]);
+     f(2) = 0.0;
+  }
+}
 void assembleDomainLFElementVector(apf::Mesh* mesh, apf::MeshEntity* e,
     apf::Field* f, mth::Vector<double>& elvect)
 {
-  apf::MeshElement* me = apf::createMeshElement(mesh, e);
-  apf::Element* el = apf::createElement(f, me);
+  apf::FieldShape* fs = f->getShape();
   int type = mesh->getType(e);
-  int nd = apf::countElementNodes(el->getFieldShape(), type);
+  PCU_ALWAYS_ASSERT(type == apf::Mesh::TET);
+  int nd = apf::countElementNodes(fs, type);
   int dim = apf::getDimension(mesh, e);
+  double w;
 
-  apf::NewArray<apf::Vector3> vectorshape(nd);
+  apf::NewArray<apf::Vector3> vectorshapes(nd);
   elvect.resize(nd);
-  mth::Vector<double> val (3);
+  mth::Vector<double> val (dim);
   val.zero();
   apf::Vector3 p;
-  double w;
 
-  apf::FieldShape* fs = f->getShape();
-  int order = 2 * fs->getOrder();
-  int np = apf::countIntPoints(me, order); // int points required
+  apf::MeshElement* me = apf::createMeshElement(mesh, e);
+  apf::Element* el = apf::createElement(f, me);
+  int int_order = 2 * fs->getOrder();
+  int np = apf::countIntPoints(me, int_order);
 
   elvect.zero();
   for (int i = 0; i < np; i++) {
-    apf::getIntPoint(me, order, i, p);
-    double weight = apf::getIntWeight(me, order, i);
+    apf::getIntPoint(me, int_order, i, p);
+    double weight = apf::getIntWeight(me, int_order, i);
     apf::Matrix3x3 J;
     apf::getJacobian(me, p, J);
     double jdet = apf::getJacobianDeterminant(J, dim);
     w = weight * jdet;
 
-    apf::getVectorShapeValues(el, p, vectorshape);
+    apf::getVectorShapeValues(el, p, vectorshapes);
     apf::Vector3 global;
     apf::mapLocalToGlobal(me, p, global);
     pumiUserFunction(global, val, e, apf::getMesh(f)); // eval vector function
     val *= w;
 
-    mth::Matrix<double> vectorShape (nd, dim);
+    mth::Matrix<double> vectorShapes (nd, dim);
     for (int i = 0; i < nd; i++)
       for (int j = 0; j < dim; j++)
-        vectorShape(i,j) = vectorshape[i][j];
-    mth::Vector<double> temp (nd);
-    temp.zero();
-    mth::multiply(vectorShape, val, temp);
-    elvect += temp;
+        vectorShapes(i,j) = vectorshapes[i][j];
+    mth::Vector<double> V (nd);
+    V.zero();
+    mth::multiply(vectorShapes, val, V);
+    elvect += V;
   }
   apf::destroyElement(el);
   apf::destroyMeshElement(me);
 }
 
-static double getLocalLFIntegral(EdgePatch* p, apf::MeshEntity* tet)
+// computes local linear form integral restricted to
+// an edge of a tet element
+static double getLocalEdgeLF(EdgePatch* p, apf::MeshEntity* tet)
 {
-  // find position of edge in downward edges of element
+  // findIn edge in downward edges of tet
   apf::Downward e;
   int ne = p->mesh->getDownward(tet, 1, e);
+  PCU_ALWAYS_ASSERT(ne == 6);
   int ei = apf::findIn(e, ne, p->entity);
   // assemble Domain LF Vector
   mth::Vector<double> elvect;
@@ -401,7 +431,7 @@ static double getLocalLFIntegral(EdgePatch* p, apf::MeshEntity* tet)
   int which, rotate; bool flip;
   apf::getAlignment(p->mesh, tet, p->entity, which, flip, rotate);
   if (flip)
-    return -1*elvect(ei);
+    elvect(ei) = -1*elvect(ei);
   return elvect(ei);
 }
 
@@ -464,13 +494,15 @@ apf::Vector3 computeFaceOutwardNormal(apf::Mesh* m,
   return n;
 }
 
-
-static double getFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
+// computes local flux integral restricted to
+// an edge of a tet element
+static double getLocalFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
 {
   double fluxIntegral = 0.0;
-  // 1. find position of edge in downward edges of element
+  // 1. findIn edge in downward edges of tet
   apf::Downward e;
   int ne = ep->mesh->getDownward(tet, 1, e);
+  PCU_ALWAYS_ASSERT(ne == 6);
   int ei = apf::findIn(e, ne, ep->entity);
   // 2. get faces of the tet in the patch
   apf::Downward f;
@@ -501,17 +533,17 @@ static double getFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
 
     // 5. count integration points for flux face integral
     apf::FieldShape* fs = ep->equilibration->ef->getShape();
-    int order = 2 * fs->getOrder() - 2;
+    int int_order = 2 * fs->getOrder();
     apf::MeshElement* fme = apf::createMeshElement(ep->mesh, currentFace);
-    int np = apf::countIntPoints(fme, order);
-    std::cout << "np " << np << std::endl;
+    int np = apf::countIntPoints(fme, int_order);
+    std::cout << "np " << np << std::endl; // REMOVE
 
     // loop over integration points
     apf::Vector3 p, tet1xi, tet2xi, curl1, curl2, curl,
       fnormal1, fnormal2, tk, vshape;
     for (int n = 0; n < np; n++) {
-      apf::getIntPoint(fme, order, n, p);
-      double weight = apf::getIntWeight(fme, order, n);
+      apf::getIntPoint(fme, int_order, n, p);
+      double weight = apf::getIntWeight(fme, int_order, n);
       apf::Matrix3x3 fJ;
       apf::getJacobian(fme, p, fJ);
       double jdet = apf::getJacobianDeterminant(
@@ -531,15 +563,14 @@ static double getFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
         std::cout << "normal2 " << fnormal2 << std::endl; // REMOVE
       }
 
-
       curl.zero();
       // compute curl1
       tet1xi = apf::boundaryToElementXi(ep->mesh, currentFace, firstTet, p);
       apf::MeshElement* me1 = apf::createMeshElement(ep->mesh, firstTet);
       apf::Element* el1 = apf::createElement(ep->equilibration->ef, me1);
       apf::getCurl(el1, tet1xi, curl1);
-      apf::Vector3 temp1 = apf::cross(fnormal1, curl1); // TODO clean
-      curl += temp1; //
+      apf::Vector3 temp1 = apf::cross(fnormal1, curl1);
+      curl += temp1;
       apf::destroyElement(el1);
       apf::destroyMeshElement(me1);
 
@@ -549,8 +580,8 @@ static double getFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
         apf::MeshElement* me2 = apf::createMeshElement(ep->mesh, secondTet);
         apf::Element* el2 = apf::createElement(ep->equilibration->ef, me2);
         apf::getCurl(el2, tet2xi, curl2);
-        apf::Vector3 temp2 = apf::cross(fnormal2, curl2); // TODO clean
-        curl += (temp2 * -1.); //
+        apf::Vector3 temp2 = apf::cross(fnormal2, curl2);
+        curl += (temp2 * -1.);
         apf::destroyElement(el2);
         apf::destroyMeshElement(me2);
       }
@@ -558,18 +589,19 @@ static double getFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
       // compute tk (inter-element averaged flux)
       //tk = apf::cross(fnormal, curl);
       tk = curl * 1./2.;
-      std::cout << "tk " << tk << std::endl;
+      std::cout << "tk " << tk << std::endl; // REMOVE
 
       // compute vector shape
       int type = ep->mesh->getType(tet);
+      PCU_ALWAYS_ASSERT(type == apf::Mesh::TET);
       int nd = apf::countElementNodes(fs, type);
       apf::NewArray<apf::Vector3> vectorshapes (nd);
       apf::MeshElement* me = apf::createMeshElement(ep->mesh, tet);
       apf::Element* el = apf::createElement(ep->equilibration->ef, me);
       apf::Vector3 tetxi = apf::boundaryToElementXi(ep->mesh, currentFace, tet, p);
       apf::getVectorShapeValues(el, tetxi, vectorshapes);
-      std::cout << "p " << p << std::endl;
-      std::cout << "tetxi " << tetxi << std::endl;
+      std::cout << "p " << p << std::endl; // REMOVE
+      std::cout << "tetxi " << tetxi << std::endl; // REMOVE
       vshape = vectorshapes[ei];
       apf::destroyElement(el);
       apf::destroyMeshElement(me);
@@ -579,13 +611,10 @@ static double getFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
       apf::getAlignment(ep->mesh, tet, ep->entity, which, flip, rotate);
       if (flip) {
         vshape = vshape * -1.;
-        std::cout << "flip " << flip << std::endl;
-        std::cout << "vshape " << vshape << std::endl;
+        std::cout << "flip " << flip << std::endl; // REMOVE
+        std::cout << "vshape " << vshape << std::endl; // REMOVE
       }
 
-
-
-
       // compute integral
       fluxFaceIntegral += (tk * vshape) * weight * jdet;
     }
@@ -596,41 +625,29 @@ static double getFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
   return fluxIntegral;
 }
 
-static void assembleRHS(EdgePatch* p)
+static void assembleEdgePatchRHS(EdgePatch* p)
 {
   if (p->isOnBdry) {
-    std::cout << "boundary" << std::endl; // REMOVE
     p->b.resize(p->tets.size() + p->faces.size());
     p->b.zero();
   }
   else {
-    std::cout << "interior" << std::endl; // REMOVE
     p->b.resize(p->tets.size());
     p->b.zero();
   }
-  printf("RHS B\n");
-  double testlfblf = 0.0; // REMOVE
-  double testflux = 0.0; // REMOVE
   int ne = p->tets.size();
   int nf = p->faces.size();
   for (int i = 0; i < ne; i++) {
     EntitySet::iterator it = std::next(p->tets.begin(), i);
     apf::MeshEntity* tet = *it;
-    double blfIntegral = getLocalBLFIntegral(p, tet);
-    double lfIntegral = getLocalLFIntegral(p, tet);
-    double fluxIntegral = getFluxIntegral(p, tet);
-    testlfblf += blfIntegral - lfIntegral;
-    testflux += fluxIntegral;
+    double blfIntegral = getLocalEdgeBLF(p, tet);
+    double lfIntegral = getLocalEdgeLF(p, tet);
+    double fluxIntegral = getLocalFluxIntegral(p, tet);
     if(p->isOnBdry)
       p->b(nf+i) = blfIntegral - lfIntegral - fluxIntegral;
     else
       p->b(i) = blfIntegral - lfIntegral - fluxIntegral;
   }
-  std::cout << p->b << std::endl; // REMOVE
-  std::cout << "Sum of bilinear - linear integrals " << testlfblf << std::endl; // want this to be zero
-  if (abs(testlfblf) > 1e-08 && (!p->isOnBdry)) std::cout << "nonzero" << std::endl;
-  std::cout << "Sum of flux integrals over the edge patch " << testflux << std::endl; // want this to be zero
-  if (abs(testflux) > 1e-08 && (!p->isOnBdry)) std::cout << "nonzero" << std::endl;
 }
 
 // The following two functions help order tets and faces in a cavity in a
@@ -717,10 +734,30 @@ static void getOrderedTetsandFaces(apf::Mesh* mesh, apf::MeshEntity* edge,
 static void runErm(EdgePatch* p)
 {
   getOrderedTetsandFaces(p->mesh, p->entity, p->tets, p->faces);
-  assembleLHS(p);
-  assembleRHS(p);
+  assembleEdgePatchLHS(p);
+  assembleEdgePatchRHS(p);
   mth::solveFromQR(p->qr.Q, p->qr.R, p->b, p->x);
 
+  bool debug = true; // REMOVE DEBUG
+  if (debug) {
+    if (p->isOnBdry)
+      cout << "Boundary Patch" << endl;
+    else
+      cout << "Interior Patch" << endl;
+    cout << "ne " << p->tets.size() << " nf " << p->faces.size() << endl;
+    cout << "LHS Matrix" << endl;
+    cout << p->A << endl;
+
+    cout << "RHS Vector" << endl;
+    cout << p->b << endl;
+    double rhs_sum = 0.0;
+    for(unsigned int i = 0; i < p->b.size(); i++)
+      rhs_sum += p->b(i);
+    cout << "rhs_sum " << rhs_sum << endl;
+    if ( (abs(rhs_sum) > 1e-8) && (!p->isOnBdry) )
+      cout << "nonzero! error" << endl;
+  }
+
   int nf = p->faces.size();
   for(int i = 0; i < nf; i++) {
     // get face entitiy
@@ -738,7 +775,6 @@ static void runErm(EdgePatch* p)
     apf::setComponents(p->equilibration->g, face, 0, components);
   }
 
-
   // REMOVE ALL BELOW
   int ne = p->tets.size();
   //int nf = p->faces.size();
@@ -789,17 +825,7 @@ class EdgePatchOp : public apf::CavityOp
 
 apf::Field* equilibrateResiduals(apf::Field* f)
 {
-  // initialize the field 'g' with zeros
   apf::Field* g = createPackedField( apf::getMesh(f), "g", 3, apf::getConstant(2) );
-  double zeros[3] = {0., 0., 0.};
-  apf::MeshEntity* face;
-  apf::MeshIterator* it = apf::getMesh(f)->begin(2);
-  while ((face = apf::getMesh(f)->iterate(it))) {
-    apf::setComponents(g, face, 0, zeros);
-  }
-  apf::getMesh(f)->end(it);
-
-
   Equilibration equilibration;
   setupEquilibration(&equilibration, f, g);
   EdgePatchOp op(&equilibration);

From 5032f20303ef585ded6e57f9a304a856aa7b50ac Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Sat, 16 May 2020 21:41:06 -0400
Subject: [PATCH 165/555] Fixes bug in assembleFaceMassMatrix. clean up.

---
 em/emFluxCorrection.cc | 81 +++++++++++++++++++++++++-----------------
 1 file changed, 49 insertions(+), 32 deletions(-)

diff --git a/em/emFluxCorrection.cc b/em/emFluxCorrection.cc
index c621cc426..8e6ffae44 100644
--- a/em/emFluxCorrection.cc
+++ b/em/emFluxCorrection.cc
@@ -7,8 +7,12 @@
 #include <iostream>
 #include <cstdlib>
 
-#include "em.h"
+#include "crv.h"
+#include "crvShape.h"
+#include "apfElement.h"
 
+#include "em.h"
+using namespace std;
 namespace em {
 
 struct QRDecomp {
@@ -16,58 +20,61 @@ struct QRDecomp {
   mth::Matrix<double> R;
 };
 
-static void assembleFaceMassMatrix(apf::Mesh* mesh,apf::MeshEntity* e,
+static void assembleFaceMassMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
     apf::Field* f, mth::Matrix<double>& elmat)
 {
   apf::FieldShape* fs = f->getShape();
   int type = mesh->getType(e);
   int nd = apf::countElementNodes(fs, type);
   int dim = apf::getDimension(mesh, e);
+  int sdim = mesh->getDimension();
+  PCU_ALWAYS_ASSERT(type == apf::Mesh::TRIANGLE && dim == 2);
   double w;
 
-  apf::NewArray<apf::Vector3> vectorshape(nd);
+  apf::NewArray<apf::Vector3> vectorshapes(nd);
   elmat.resize(nd,nd);
 
   apf::MeshElement* me = apf::createMeshElement(mesh, e);
   apf::Element* el = apf::createElement(f, me);
-  int order = 2 * fs->getOrder();
-  int np = apf::countIntPoints(me, order); // int points required
+  int int_order = 2 * fs->getOrder();
+  int np = apf::countIntPoints(me, int_order); // int points required
 
   apf::Downward edges;
   int ned =  mesh->getDownward(e, 1, edges);
-  PCU_ALWAYS_ASSERT(ned == 3);
   int which, rotate; bool flip;
 
   elmat.zero();
   apf::Vector3 p;
   for (int i = 0; i < np; i++) {
-    apf::getIntPoint(me, order, i, p);
-    double weight = apf::getIntWeight(me, order, i);
+    apf::getIntPoint(me, int_order, i, p);
+    double weight = apf::getIntWeight(me, int_order, i);
     apf::Matrix3x3 J;
     apf::getJacobian(me, p, J);
     double jdet = apf::getJacobianDeterminant(J, dim);
     w = weight * jdet;
 
-    apf::getVectorShapeValues(el, p, vectorshape);
-    mth::Matrix<double> vectorShape (nd, dim);
+    apf::getVectorShapeValues(el, p, vectorshapes);
+    mth::Matrix<double> vectorShapes (nd, sdim);
     for (int j = 0; j < nd; j++)
-      for (int k = 0; k < dim; k++)
-        vectorShape(j,k) = vectorshape[j][k];
+      for (int k = 0; k < sdim; k++)
+        vectorShapes(j,k) = vectorshapes[j][k];
 
-    // negate negatve dof indices
+    // negate negative dof indices
+    // TODO maybe can do this outside the function and therfore
+    // merge this function with assembleVectorMassElementMatrix
     for (int ei = 0; ei < ned; ei++) {
       apf::getAlignment(mesh, e, edges[ei], which, flip, rotate);
       if(flip) {
-        for (int j = 0; j < dim; j++)
-          vectorShape(ei, j) = -1*vectorShape(ei, j);
+        for (int j = 0; j < sdim; j++)
+          vectorShapes(ei, j) = -1*vectorShapes(ei, j);
       }
     }
 
-    mth::Matrix<double> vectorShapeT (dim, nd);
-    mth::transpose(vectorShape, vectorShapeT);
+    mth::Matrix<double> vectorShapesT (sdim, nd);
+    mth::transpose(vectorShapes, vectorShapesT);
     mth::Matrix<double> M (nd,nd);
     M.zero();
-    mth::multiply(vectorShape, vectorShapeT, M);
+    mth::multiply(vectorShapes, vectorShapesT, M);
     M *= w;
     elmat += M;
   }
@@ -81,33 +88,41 @@ static void assembleFaceMassMatrix(apf::Mesh* mesh,apf::MeshEntity* e,
 apf::Field* computeFluxCorrection(apf::Field* ef, apf::Field* g)
 {
   int order = ef->getShape()->getOrder();
-  apf::Field* faceNedelecField = apf::createField(
-      apf::getMesh(g), "face_nedelec_field", apf::SCALAR, apf::getNedelec(order));
-  apf::zeroField(faceNedelecField);
-
+  cout << "tri_field_order " << order << endl; // REMOVE
+  apf::Field* triNedelecField = apf::createField(
+      apf::getMesh(ef), "tri_nedelec_field", apf::SCALAR, apf::getNedelec(order));
+  apf::zeroField(triNedelecField);
 
   apf::Field* correctedFluxField =  createPackedField(
-      apf::getMesh(g), "corrected_flux_field", 3, apf::getConstant(2));
+      apf::getMesh(ef), "corrected_flux_field", 3, apf::getConstant(2));
 
   // iterate over all faces of the mesh
   apf::MeshEntity* face;
-  apf::MeshIterator* it = apf::getMesh(g)->begin(2);
-  while ((face = apf::getMesh(g)->iterate(it))) {
-    // assemble RHS vector
+  apf::MeshIterator* it = apf::getMesh(ef)->begin(2);
+  while ((face = apf::getMesh(ef)->iterate(it))) {
+    // 1. assemble RHS vector
     double components[3];
     apf::getComponents(g, face, 0, components);
     mth::Vector<double> rhs(3);
     rhs(0) = components[0]; rhs(1) = components[1]; rhs(2) = components[2];
 
-    // assemlbe face mass matrix
+    bool debug = true;
+    if (debug) {
+      cout << "RHS VECTOR" << endl;
+      cout << components[0] << " " << components[1] << " " << components[2] << endl;
+      cout << rhs << endl;
+      cout << "==============" << endl;
+    }
+
+    // 2. assemble face mass matrix
     mth::Matrix<double> M;
     assembleFaceMassMatrix(
-        apf::getMesh(g), face, faceNedelecField, M);
+        apf::getMesh(ef), face, triNedelecField, M);
 
-    // solve the system
+    // 3. solve the system
     QRDecomp qr;
     mth::decomposeQR(M, qr.Q, qr.R);
-    std::cout << "M" << std::endl;
+    std::cout << "M" << std::endl; // REMOVE DEBUG
     std::cout << M << std::endl;
     std::cout << "Q" << std::endl;
     std::cout << qr.Q << std::endl;
@@ -122,14 +137,16 @@ apf::Field* computeFluxCorrection(apf::Field* ef, apf::Field* g)
     std::cout << theta << std::endl;
 
     // set solution vector on face field
-    //theta.toArray(components);
+    //theta.toArray(components); TODO clean
     components[0] = theta(0);
     components[1] = theta(1);
     components[2] = theta(2);
     apf::setComponents(
         correctedFluxField, face, 0, components);
   }
-  apf::getMesh(g)->end(it);
+  apf::getMesh(ef)->end(it);
+
+  apf::destroyField(triNedelecField);
 
   return correctedFluxField;
 }

From ae9f58e2ecc055a9908e04d2634d38b1c298bc10 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Sun, 17 May 2020 16:40:12 -0400
Subject: [PATCH 166/555] Fixes bugs in emEstimateError.cc and clean ups

---
 em/em.h               |   1 -
 em/emEstimateError.cc | 116 ++++++++++++++++++++++--------------------
 2 files changed, 60 insertions(+), 57 deletions(-)

diff --git a/em/em.h b/em/em.h
index 84bf238e5..3e60c1ef3 100644
--- a/em/em.h
+++ b/em/em.h
@@ -52,7 +52,6 @@ void assembleElementMatrix(apf::Mesh* mesh, apf::MeshEntity*e,
 void assembleDomainLFElementVector(apf::Mesh* mesh, apf::MeshEntity* e,
     apf::Field* f, mth::Vector<double>& elvect);
 
-
 apf::Vector3 computeFaceOutwardNormal(apf::Mesh* m,
     apf::MeshEntity* t, apf::MeshEntity* f, apf::Vector3 const& p);
 
diff --git a/em/emEstimateError.cc b/em/emEstimateError.cc
index 07c82e9c6..92296681c 100644
--- a/em/emEstimateError.cc
+++ b/em/emEstimateError.cc
@@ -29,22 +29,22 @@ static void computeResidualBLF(apf::Mesh* mesh, apf::MeshEntity* e,
   apf::NewArray<apf::Vector3> curlshape(nd);
   apf::NewArray<apf::Vector3> vectorshape(nd);
   mth::Matrix<double> phys_curlshape(nd, dim);
-  mth::Matrix<double> Vectorshape(nd, dim);
+  mth::Matrix<double> vectorShape(nd, dim);
   mth::Vector<double> curlcurl_vec (nd);
   mth::Vector<double> mass_vec (nd);
   blf.resize(nd);
 
   apf::MeshElement* me = apf::createMeshElement(mesh, e);
   apf::Element* el = apf::createElement(f, me);
-  int order = 2 * fs->getOrder() - 2;
-  int np = apf::countIntPoints(me, order); // int points required
+  int int_order = 2 * fs->getOrder();
+  int np = apf::countIntPoints(me, int_order); // int points required
 
   // 1. Compute Curl Curl Integration
   curlcurl_vec.zero();
   apf::Vector3 p, curl;
   for (int i = 0; i < np; i++) {
-    apf::getIntPoint(me, order, i, p);
-    double weight = apf::getIntWeight(me, order, i);
+    apf::getIntPoint(me, int_order, i, p);
+    double weight = apf::getIntWeight(me, int_order, i);
     apf::Matrix3x3 J;
     apf::getJacobian(me, p, J);
     //double jdet = apf::getJacobianDeterminant(J, dim);
@@ -52,8 +52,8 @@ static void computeResidualBLF(apf::Mesh* mesh, apf::MeshEntity* e,
 
     // get curl vector
     apf::getCurl(el, p, curl);
-    
-    // get curlshape values
+
+    // get curlshape values // TODO CLEAN use getCurlShapeValues
     el->getShape()->getLocalVectorCurls(mesh, e, p, curlshape);
     phys_curlshape.zero();
     for (int i = 0; i < nd; i++)
@@ -62,23 +62,25 @@ static void computeResidualBLF(apf::Mesh* mesh, apf::MeshEntity* e,
           phys_curlshape(i,j) += curlshape[i][k] * J[k][j];
 
     // multiply
-    mth::Vector<double> temp (nd);
-    temp.zero();
+    mth::Vector<double> V (nd);
+    V.zero();
     mth::Vector<double> c (dim);
     c(0) = curl[0]; c(1) = curl[1]; c(2) = curl[2];
-    mth::multiply(phys_curlshape, c, temp);
-    temp *= w;
-    
-    curlcurl_vec += temp;
+    mth::multiply(phys_curlshape, c, V);
+    V *= w;
 
+    curlcurl_vec += V;
   }
-    
+
   // 2. Compute Vector Mass Integration
+  int_order = 2 * fs->getOrder();
+  np = apf::countIntPoints(me, int_order); // int points required
+
   mass_vec.zero();
   apf::Vector3 vvalue;
   for (int i = 0; i < np; i++) {
-    apf::getIntPoint(me, order, i, p);
-    double weight = apf::getIntWeight(me, order, i);
+    apf::getIntPoint(me, int_order, i, p);
+    double weight = apf::getIntWeight(me, int_order, i);
     apf::Matrix3x3 J;
     apf::getJacobian(me, p, J);
     double jdet = apf::getJacobianDeterminant(J, dim);
@@ -88,20 +90,20 @@ static void computeResidualBLF(apf::Mesh* mesh, apf::MeshEntity* e,
     apf::getVector(el, p, vvalue);
 
     apf::getVectorShapeValues(el, p, vectorshape);
-    Vectorshape.zero();
+    vectorShape.zero();
     for (int i = 0; i < nd; i++)
       for (int j = 0; j < dim; j++)
-        Vectorshape(i,j) = vectorshape[i][j];
+        vectorShape(i,j) = vectorshape[i][j];
 
 
-    mth::Vector<double> temp (nd);
-    temp.zero();
+    mth::Vector<double> V (nd);
+    V.zero();
     mth::Vector<double> v (dim);
     v(0) = vvalue[0]; v(1) = vvalue[1]; v(2) = vvalue[2];
-    mth::multiply(Vectorshape, v, temp);
-    temp *= w;
+    mth::multiply(vectorShape, v, V);
+    V *= w;
 
-    mass_vec += temp;
+    mass_vec += V;
   }
 
   // 3. get result
@@ -109,7 +111,7 @@ static void computeResidualBLF(apf::Mesh* mesh, apf::MeshEntity* e,
   blf += curlcurl_vec;
   blf += mass_vec;
 
-  // TODO Take care of Negative Dofs 
+  // TODO Take care of Negative Dofs
 }
 
 // @ apf::Field* f --> input p-order tet ND field
@@ -120,7 +122,7 @@ static void computeLambdaVector(
     apf::MeshEntity* e,
     apf::Field* f,
     apf::Field* fp1,
-    apf::Field* faceNDField,
+    apf::Field* tri_nedelec_field,
     apf::Field* THETA,
     mth::Vector<double>& lambda)
 {
@@ -130,14 +132,14 @@ static void computeLambdaVector(
   int nedofs = apf::countElementNodes(fp1s, etype);
   lambda.resize(nedofs);
 
-  // create a 2D nedelec field 
+  // create a 2D nedelec field
   int order = fp1s->getOrder();
-  apf::FieldShape* faceNDFieldShape = faceNDField->getShape();
+  apf::FieldShape* tri_nedelec_fieldShape = tri_nedelec_field->getShape();
 
   // get the downward faces of the element
   apf::Downward faces;
   int nf = mesh->getDownward(e, 2, faces);
-  
+
   lambda.zero();
   // assemble lambda vector LOOP OVER DOWNWARD FACES
   for (int ii = 0; ii < nf; ii++) {
@@ -159,7 +161,7 @@ static void computeLambdaVector(
     // 2. get downward edges of the face
     apf::Downward edges;
     int nedges = mesh->getDownward(face, 1, edges);
-    
+
     // 3. get theta coeffs on the face
     double components[3];
     apf::getComponents(THETA, face, 0, components);
@@ -172,11 +174,11 @@ static void computeLambdaVector(
 
     int ftype = mesh->getType(face);
     PCU_ALWAYS_ASSERT(ftype == apf::Mesh::TRIANGLE);
-    int nfdofs = apf::countElementNodes(faceNDFieldShape, ftype);
+    int nfdofs = apf::countElementNodes(tri_nedelec_fieldShape, ftype);
     apf::NewArray<apf::Vector3> vectorshape(nfdofs);
 
     apf::MeshElement* fme = apf::createMeshElement(mesh, face);
-    apf::Element* fel = apf::createElement(faceNDField, fme);
+    apf::Element* fel = apf::createElement(tri_nedelec_field, fme);
     int np = apf::countIntPoints(fme, 2*order); // int points required
 
     // 4. Compute integral on the face
@@ -194,12 +196,12 @@ static void computeLambdaVector(
       // evaluate theta vector using theta coeffs
       apf::Vector3 theta_vector;
       theta_vector.zero();
-      apf::NewArray<apf::Vector3> vector2Dshapes (nfdofs);
-      apf::getVectorShapeValues(fel, p, vector2Dshapes);
+      apf::NewArray<apf::Vector3> triVectorShapes (nfdofs);
+      apf::getVectorShapeValues(fel, p, triVectorShapes);
       int which, rotate; bool flip; // negative ND dofs
       for (int i = 0; i < nedges; i++) {
         apf::getAlignment(mesh, face, edges[i], which, flip, rotate);
-        apf::Vector3 v = vector2Dshapes[i];
+        apf::Vector3 v = triVectorShapes[i];
         if (flip) { v = v * -1.; }
 
         v = v * theta_coeffs[i];
@@ -226,8 +228,8 @@ static void computeLambdaVector(
       apf::MeshElement* me1 = apf::createMeshElement(mesh, firstTet);
       apf::Element* el1 = apf::createElement(f, me1);
       apf::getCurl(el1, tet1xi, curl1);
-      apf::Vector3 temp1 = apf::cross(fnormal1, curl1); // TODO clean
-      curl += temp1; //
+      apf::Vector3 temp1 = apf::cross(fnormal1, curl1);
+      curl += temp1;
       apf::destroyElement(el1);
       apf::destroyMeshElement(me1);
 
@@ -237,24 +239,26 @@ static void computeLambdaVector(
         apf::MeshElement* me2 = apf::createMeshElement(mesh, secondTet);
         apf::Element* el2 = apf::createElement(f, me2);
         apf::getCurl(el2, tet2xi, curl2);
-        apf::Vector3 temp2 = apf::cross(fnormal2, curl2); // TODO clean
-        curl += (temp2 * -1.); //
+        apf::Vector3 temp2 = apf::cross(fnormal2, curl2);
+        curl += (temp2 * -1.);
         apf::destroyElement(el2);
         apf::destroyMeshElement(me2);
       }
 
       // compute tk (inter-element averaged flux)
       tk = curl * 1./2.;
-      std::cout << "tk " << tk << std::endl;
+      std::cout << "tk " << tk << std::endl; // REMOVE
 
       // compute p+1 order 3D vector shapes
-      apf::NewArray<apf::Vector3> vector3dshapes (nedofs);
+      apf::NewArray<apf::Vector3> tetVectorShapes (nedofs);
       apf::MeshElement* me = apf::createMeshElement(mesh, e);
       apf::Element* el = apf::createElement(fp1, me);
       apf::Vector3 tetxi = apf::boundaryToElementXi(mesh, face, e, p);
-      apf::getVectorShapeValues(el, tetxi, vector3dshapes);
+      apf::getVectorShapeValues(el, tetxi, tetVectorShapes);
       // TODO take care of negative ND dofs
-      apf::destroyMeshElement(me); 
+      apf::destroyElement(el);
+      apf::destroyMeshElement(me);
+
       // compute integral
       apf::Vector3 theta_plus_tk = theta_vector + tk;
       double w = weight * jdet;
@@ -262,14 +266,14 @@ static void computeLambdaVector(
 
       // matrix vector multiplication
       for (int i = 0; i < nedofs; i++)
-        lambda(i) += theta_plus_tk * vector3dshapes[i];
+        lambda(i) += theta_plus_tk * tetVectorShapes[i];
 
     } // end integral loop
     apf::destroyMeshElement(fme);
   } // end face loop
 }
 
-static void getEssentialBdrElementNDDofs(apf::Mesh* mesh, apf::MeshEntity* e,
+static void getEssentialElementNDDofs(apf::Mesh* mesh, apf::MeshEntity* e,
     apf::Field* f, apf::NewArray<int>& ess_dofs, apf::NewArray<int>& uness_dofs)
 {
   apf::FieldShape* fs = f->getShape();
@@ -293,7 +297,7 @@ static void getEssentialBdrElementNDDofs(apf::Mesh* mesh, apf::MeshEntity* e,
       }
     }
   }
-  int nodesOnEdges = fs->countNodesOn(apf::Mesh::TRIANGLE) * nedges;
+  int nodesOnEdges = fs->countNodesOn(apf::Mesh::EDGE) * nedges;
 
   apf::Downward faces;
   int nfaces = mesh->getDownward(e, 2, faces);
@@ -327,7 +331,7 @@ static void getEssentialBdrElementNDDofs(apf::Mesh* mesh, apf::MeshEntity* e,
       uness_dofs[uness_dof_counter++] = i;
   }
 }
- 
+
 /**
  * Inputs: Matrix A, Vector X, Vector B, essential dofs, not essential dofs.
  * Output: reduced matrix A, reduced rhs B.
@@ -344,7 +348,7 @@ static void eliminateDBCs(
   int num_ess_dofs = ess_dofs.size();
   int num_uness_dofs = uness_dofs.size();
 
-  // 1. Remove rows and cols of A corresponding to
+  // 1. Remove rows of A corresponding to
 	// ess_dofs by copying into Anew
   Anew.resize(num_uness_dofs, num_uness_dofs);
 	for(int rr = 0; rr < num_uness_dofs; rr++) {
@@ -410,7 +414,7 @@ static double computeL2Error(apf::Mesh* mesh, apf::MeshEntity* e,
     apf::getJacobian(me, p, J);
     double jdet = apf::getJacobianDeterminant(J, dim);
     w = weight * jdet;
-	
+
     apf::getVectorShapeValues(el, p, vectorshape);
 		// TODO take care of negative dof values
     mth::Matrix<double> vectorShape (nd, dim);
@@ -420,7 +424,7 @@ static double computeL2Error(apf::Mesh* mesh, apf::MeshEntity* e,
 
     mth::Matrix<double> vectorShapeT (dim, nd);
     mth::transpose(vectorShape, vectorShapeT);
-		
+
 		mth::Vector<double> err_func;
 		mth::multiply(vectorShapeT, error_dofs, err_func);
 
@@ -442,7 +446,7 @@ apf::Field* emEstimateError(apf::Field* ef, apf::Field* correctedFlux)
   apf::Field* error_field = apf::createIPField(
 		apf::getMesh(ef), "residual_error_field", apf::SCALAR, 1); // TODO maybe use getConstant here
 
-  cout << "ELEMENT ERROR FIELD CREATED" << endl;
+  cout << "ELEMENT ERROR FIELD CREATED" << endl; // REMOVE
 
   // 2. Create p+1 order tet ND field
   // and p order triangle ND field
@@ -452,11 +456,11 @@ apf::Field* emEstimateError(apf::Field* ef, apf::Field* correctedFlux)
 	 "orderp1_nedelec_field", apf::SCALAR, apf::getNedelec(orderp1));
   apf::zeroField(efp1);
 
-  apf::Field* faceNDField = apf::createField(apf::getMesh(ef),
+  apf::Field* tri_nedelec_field = apf::createField(apf::getMesh(ef),
       "face_nedelec_field", apf::SCALAR, apf::getNedelec(order));
-  apf::zeroField(faceNDField);;
+  apf::zeroField(tri_nedelec_field);;
 
-  cout << "P+1 ORDER FIELD CREATED" << endl;
+  cout << "P+1 ORDER FIELD CREATED" << endl; // REMOVE
 
   // 2. iterate over all elements of the mesh
   apf::MeshEntity* el;
@@ -483,7 +487,7 @@ apf::Field* emEstimateError(apf::Field* ef, apf::Field* correctedFlux)
     // 2(d). Compute Lamda Vector
     mth::Vector<double> lambda;
     computeLambdaVector(
-        apf::getMesh(ef), el, ef, efp1, faceNDField, correctedFlux, lambda);
+        apf::getMesh(ef), el, ef, efp1, tri_nedelec_field, correctedFlux, lambda);
     // TODO Take care of negative dofs in lambda
     cout << "RHS lambda vector assembled" << endl;
 
@@ -495,7 +499,7 @@ apf::Field* emEstimateError(apf::Field* ef, apf::Field* correctedFlux)
 
     // 2(f). Get List of Essential Dofs
     apf::NewArray<int> ess_dofs, uness_dofs;
-    getEssentialBdrElementNDDofs(
+    getEssentialElementNDDofs(
         apf::getMesh(efp1), el, efp1, ess_dofs, uness_dofs);
     cout << "Get list of Essential Dofs" << endl;
 
@@ -536,7 +540,7 @@ apf::Field* emEstimateError(apf::Field* ef, apf::Field* correctedFlux)
   apf::getMesh(ef)->end(it);
   cout << "End loop over elements" << endl;
 	apf::destroyField(efp1);
-  apf::destroyField(faceNDField);
+  apf::destroyField(tri_nedelec_field);
 
 	return error_field;
 }

From 3af8db141411aff6e27a2f26b4bfff1e96e555a0 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Wed, 20 May 2020 14:43:21 -0400
Subject: [PATCH 167/555] Fixes edge patch flux canceling by 2D tri shapes

---
 em/emResidualFunctionals.cc | 58 +++++++++++++++++--------------------
 1 file changed, 27 insertions(+), 31 deletions(-)

diff --git a/em/emResidualFunctionals.cc b/em/emResidualFunctionals.cc
index 3e0feaa44..dc850eaef 100644
--- a/em/emResidualFunctionals.cc
+++ b/em/emResidualFunctionals.cc
@@ -340,10 +340,12 @@ static double getLocalEdgeBLF(EdgePatch* p, apf::MeshEntity* tet)
 
   // pick edge index from the resulting vector
   // negation of negative ND dofs
-  int which, rotate; bool flip;
+  /*int which, rotate; bool flip;
   apf::getAlignment(p->mesh, tet, p->entity, which, flip, rotate);
   if (flip)
-    blf_integrals(ei) = -1*blf_integrals(ei);
+    blf_integrals(ei) = -1*blf_integrals(ei);*/
+
+  cout << "local blf " << blf_integrals(ei) << endl;
   return blf_integrals(ei);
 }
 
@@ -428,10 +430,11 @@ static double getLocalEdgeLF(EdgePatch* p, apf::MeshEntity* tet)
   mth::Vector<double> elvect;
   assembleDomainLFElementVector(p->mesh, tet,
       p->equilibration->ef, elvect);
-  int which, rotate; bool flip;
+  /*int which, rotate; bool flip;
   apf::getAlignment(p->mesh, tet, p->entity, which, flip, rotate);
   if (flip)
-    elvect(ei) = -1*elvect(ei);
+    elvect(ei) = -1*elvect(ei);*/
+  cout << "local lf " << elvect(ei) << endl;
   return elvect(ei);
 }
 
@@ -499,12 +502,7 @@ apf::Vector3 computeFaceOutwardNormal(apf::Mesh* m,
 static double getLocalFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
 {
   double fluxIntegral = 0.0;
-  // 1. findIn edge in downward edges of tet
-  apf::Downward e;
-  int ne = ep->mesh->getDownward(tet, 1, e);
-  PCU_ALWAYS_ASSERT(ne == 6);
-  int ei = apf::findIn(e, ne, ep->entity);
-  // 2. get faces of the tet in the patch
+  // 1. get faces of the tet in the patch
   apf::Downward f;
   int nf = ep->mesh->getDownward(tet, 2, f);
   PCU_ALWAYS_ASSERT(nf == 4);
@@ -514,10 +512,11 @@ static double getLocalFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
       patchFaces.push_back(f[i]);
   }
   PCU_ALWAYS_ASSERT(patchFaces.size() == 2);
-  //  3. loop over the patch faces
+
+  //  2. loop over the patch faces
   for (unsigned int i = 0; i < patchFaces.size(); i++) {
     double fluxFaceIntegral = 0.0;
-    // 4. get upward tets of the current face
+    // 3. get upward tets of the current face
     apf::Up up;
     apf::MeshEntity* currentFace = patchFaces[i];
     ep->mesh->getUp(currentFace, up);
@@ -531,10 +530,17 @@ static double getLocalFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
     if (up.n == 2)
       secondTet  = up.e[1];
 
+    // 4. findIn edge in downward edges of current face
+    apf::Downward e;
+    int ne = ep->mesh->getDownward(currentFace, 1, e);
+    PCU_ALWAYS_ASSERT(ne == 3);
+    int ei = apf::findIn(e, ne, ep->entity);
+
     // 5. count integration points for flux face integral
     apf::FieldShape* fs = ep->equilibration->ef->getShape();
     int int_order = 2 * fs->getOrder();
     apf::MeshElement* fme = apf::createMeshElement(ep->mesh, currentFace);
+    apf::Element* fel = apf::createElement(ep->equilibration->ef, fme);
     int np = apf::countIntPoints(fme, int_order);
     std::cout << "np " << np << std::endl; // REMOVE
 
@@ -592,32 +598,16 @@ static double getLocalFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
       std::cout << "tk " << tk << std::endl; // REMOVE
 
       // compute vector shape
-      int type = ep->mesh->getType(tet);
-      PCU_ALWAYS_ASSERT(type == apf::Mesh::TET);
+      int type = apf::Mesh::TRIANGLE;
       int nd = apf::countElementNodes(fs, type);
       apf::NewArray<apf::Vector3> vectorshapes (nd);
-      apf::MeshElement* me = apf::createMeshElement(ep->mesh, tet);
-      apf::Element* el = apf::createElement(ep->equilibration->ef, me);
-      apf::Vector3 tetxi = apf::boundaryToElementXi(ep->mesh, currentFace, tet, p);
-      apf::getVectorShapeValues(el, tetxi, vectorshapes);
-      std::cout << "p " << p << std::endl; // REMOVE
-      std::cout << "tetxi " << tetxi << std::endl; // REMOVE
+      apf::getVectorShapeValues(fel, p, vectorshapes);
       vshape = vectorshapes[ei];
-      apf::destroyElement(el);
-      apf::destroyMeshElement(me);
-
-      // negate if edge is flipped
-      int which, rotate; bool flip;
-      apf::getAlignment(ep->mesh, tet, ep->entity, which, flip, rotate);
-      if (flip) {
-        vshape = vshape * -1.;
-        std::cout << "flip " << flip << std::endl; // REMOVE
-        std::cout << "vshape " << vshape << std::endl; // REMOVE
-      }
 
       // compute integral
       fluxFaceIntegral += (tk * vshape) * weight * jdet;
     }
+    apf::destroyElement(fel);
     apf::destroyMeshElement(fme);
     fluxIntegral += fluxFaceIntegral;
     std::cout << "flux Face integral " << fluxFaceIntegral << std::endl; // REMOVE
@@ -635,6 +625,8 @@ static void assembleEdgePatchRHS(EdgePatch* p)
     p->b.resize(p->tets.size());
     p->b.zero();
   }
+  double testblflf = 0.0; // REMOVE
+  double testflux = 0.0; // REMOVE
   int ne = p->tets.size();
   int nf = p->faces.size();
   for (int i = 0; i < ne; i++) {
@@ -643,11 +635,15 @@ static void assembleEdgePatchRHS(EdgePatch* p)
     double blfIntegral = getLocalEdgeBLF(p, tet);
     double lfIntegral = getLocalEdgeLF(p, tet);
     double fluxIntegral = getLocalFluxIntegral(p, tet);
+    testblflf += (blfIntegral - lfIntegral);
+    testflux += fluxIntegral;
     if(p->isOnBdry)
       p->b(nf+i) = blfIntegral - lfIntegral - fluxIntegral;
     else
       p->b(i) = blfIntegral - lfIntegral - fluxIntegral;
   }
+  std::cout << "Blf - lf integrals = " << testblflf << std::endl;
+  std::cout << "Flux Integral Sum  = " << testflux << std::endl;
 }
 
 // The following two functions help order tets and faces in a cavity in a

From d326e0f56ce19ef3e022aa320d1350a7e15b2a7e Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Wed, 20 May 2020 17:56:10 -0400
Subject: [PATCH 168/555] Fixes getOrderedTetsAndFaces.

Replaces std::set with std::vector.
---
 em/emResidualFunctionals.cc | 45 +++++++++++++++++--------------------
 1 file changed, 21 insertions(+), 24 deletions(-)

diff --git a/em/emResidualFunctionals.cc b/em/emResidualFunctionals.cc
index dc850eaef..0aa73d6e4 100644
--- a/em/emResidualFunctionals.cc
+++ b/em/emResidualFunctionals.cc
@@ -6,6 +6,7 @@
  */
 #include <iostream>
 #include <cstdlib>
+#include <algorithm>
 
 #include <apfCavityOp.h>
 #include "apfElement.h"
@@ -53,7 +54,7 @@ struct QRDecomp {
   mth::Matrix<double> R;
 };
 
-typedef std::set<apf::MeshEntity*> EntitySet;
+typedef std::vector<apf::MeshEntity*> EntityVector;
 
 struct EdgePatch {
   apf::Mesh* mesh;
@@ -63,8 +64,8 @@ struct EdgePatch {
      (faces & tets) around this edge entity */
   apf::MeshEntity* entity;
   bool isOnBdry;
-  EntitySet tets;
-  EntitySet faces;
+  EntityVector tets;
+  EntityVector faces;
   mth::Matrix<double> A;
   mth::Vector<double> b;
   mth::Vector<double> x;
@@ -89,9 +90,9 @@ static void startEdgePatch(EdgePatch* p, apf::MeshEntity* e)
 static void addEntityToPatch(EdgePatch* p, apf::MeshEntity* e)
 {
   if(p->mesh->getType(e) == apf::Mesh::TRIANGLE)
-    p->faces.insert(e);
+    p->faces.push_back(e);
   if(p->mesh->getType(e) == apf::Mesh::TET)
-    p->tets.insert(e);
+    p->tets.push_back(e);
 }
 
 static void addEntitiesToPatch(EdgePatch* p, apf::DynamicArray<apf::MeshEntity*>& es)
@@ -508,7 +509,7 @@ static double getLocalFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
   PCU_ALWAYS_ASSERT(nf == 4);
   std::vector<apf::MeshEntity*> patchFaces;
   for (int i = 0; i < nf; i++) {
-    if(ep->faces.count(f[i]))
+    if(std::find(ep->faces.begin(), ep->faces.end(), f[i]) != ep->faces.end())
       patchFaces.push_back(f[i]);
   }
   PCU_ALWAYS_ASSERT(patchFaces.size() == 2);
@@ -630,8 +631,7 @@ static void assembleEdgePatchRHS(EdgePatch* p)
   int ne = p->tets.size();
   int nf = p->faces.size();
   for (int i = 0; i < ne; i++) {
-    EntitySet::iterator it = std::next(p->tets.begin(), i);
-    apf::MeshEntity* tet = *it;
+    apf::MeshEntity* tet = p->tets[i];
     double blfIntegral = getLocalEdgeBLF(p, tet);
     double lfIntegral = getLocalEdgeLF(p, tet);
     double fluxIntegral = getLocalFluxIntegral(p, tet);
@@ -663,7 +663,7 @@ static apf::MeshEntity* getTetOppFaceSharingEdge(
   return 0;
 }
 static void getOrderedTetsandFaces(apf::Mesh* mesh, apf::MeshEntity* edge,
-     EntitySet& tets, EntitySet& faces)
+     EntityVector& tets, EntityVector& faces)
 {
   tets.clear();
   faces.clear();
@@ -674,14 +674,14 @@ static void getOrderedTetsandFaces(apf::Mesh* mesh, apf::MeshEntity* edge,
     PCU_ALWAYS_ASSERT(up.n == 2);
     apf::MeshEntity* firstTet = up.e[0];
     apf::MeshEntity* nextTet  = up.e[1];
-    tets.insert(firstTet);
+    tets.push_back(firstTet);
     apf::MeshEntity* firstFace = getTetOppFaceSharingEdge(mesh, firstTet,
         currentFace, edge);
-    faces.insert(firstFace);
+    faces.push_back(firstFace);
 
     while (nextTet != firstTet) {
-      tets.insert(nextTet);
-      faces.insert(currentFace);
+      tets.push_back(nextTet);
+      faces.push_back(currentFace);
       currentFace = getTetOppFaceSharingEdge(mesh, nextTet, currentFace, edge);
       PCU_ALWAYS_ASSERT(currentFace);
       apf::Up up;
@@ -702,28 +702,28 @@ static void getOrderedTetsandFaces(apf::Mesh* mesh, apf::MeshEntity* edge,
         firstFace = up.e[i]; break;
       }
     }
-    faces.insert(firstFace);
+    faces.push_back(firstFace);
     mesh->getUp(firstFace, up);
     PCU_ALWAYS_ASSERT(up.n == 1);
     apf::MeshEntity* firstTet = up.e[0];
-    tets.insert(firstTet);
+    tets.push_back(firstTet);
 
     apf::MeshEntity* nextFace = getTetOppFaceSharingEdge(mesh, firstTet,
         firstFace, edge);
     apf::MeshEntity* nextTet = firstTet;
     mesh->getUp(nextFace, up);
     while( up.n == 2) {
-      faces.insert(nextFace);
+      faces.push_back(nextFace);
       if (nextTet != up.e[0])
         nextTet = up.e[0];
       else
         nextTet = up.e[1];
-      tets.insert(nextTet);
+      tets.push_back(nextTet);
 
       nextFace = getTetOppFaceSharingEdge(mesh, nextTet, nextFace, edge);
       mesh->getUp(nextFace, up);
     }
-    faces.insert(nextFace);
+    faces.push_back(nextFace);
   }
 }
 
@@ -757,8 +757,7 @@ static void runErm(EdgePatch* p)
   int nf = p->faces.size();
   for(int i = 0; i < nf; i++) {
     // get face entitiy
-    EntitySet::iterator it = std::next(p->faces.begin(), i);
-    apf::MeshEntity* face = *it;
+    apf::MeshEntity* face = p->faces[i];
     // get ei of edge in face downward edges
     apf::Downward e;
     int ned = p->mesh->getDownward(face, 1, e);
@@ -777,15 +776,13 @@ static void runErm(EdgePatch* p)
   std::cout << " Reordered Tets, ne: " << ne << " nf " << nf << std::endl; // REMOVE
   std::cout << " Center of Reordered tets" << std::endl;
   for (int i = 0; i < ne; i++) {
-    EntitySet::iterator it = std::next(p->tets.begin(), i);
-    apf::MeshEntity* tet = *it;
+    apf::MeshEntity* tet = p->tets[i];
     apf::Vector3 center = apf::getLinearCentroid(p->mesh, tet);
     std::cout << i << ":  " << center << std::endl;
   }
   std::cout << " Center of Reordered Faces" << std::endl;
   for (int i = 0; i < nf; i++) {
-    EntitySet::iterator it = std::next(p->faces.begin(), i);
-    apf::MeshEntity* face = *it;
+    apf::MeshEntity* face = p->faces[i];
     apf::Vector3 center = apf::getLinearCentroid(p->mesh, face);
     std::cout << i << ":  " << center << std::endl;
   }

From dcba62334635fa6bd49a060c3ff507d5cef2d4bd Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Wed, 20 May 2020 23:05:45 -0400
Subject: [PATCH 169/555] Cleans getBLFIntegral and getLFIntegral.

TODO: remove negation of negative dofs
---
 em/emResidualFunctionals.cc | 107 +++++++++++++++++++++++++++++++++---
 1 file changed, 99 insertions(+), 8 deletions(-)

diff --git a/em/emResidualFunctionals.cc b/em/emResidualFunctionals.cc
index 0aa73d6e4..fe0515fc5 100644
--- a/em/emResidualFunctionals.cc
+++ b/em/emResidualFunctionals.cc
@@ -302,9 +302,9 @@ void assembleElementMatrix(apf::Mesh* mesh, apf::MeshEntity*e,
 
 // computes local bilinear form integral restricted to
 // an edge of a tet element
-static double getLocalEdgeBLF(EdgePatch* p, apf::MeshEntity* tet)
+static double getLocalEdgeBLF(EdgePatch* ep, apf::MeshEntity* tet)
 {
-  // findIn edge in downward edges of tet
+  /* findIn edge in downward edges of tet
   apf::Downward e;
   int ne = p->mesh->getDownward(tet, 1, e);
   PCU_ALWAYS_ASSERT(ne == 6);
@@ -315,7 +315,7 @@ static double getLocalEdgeBLF(EdgePatch* p, apf::MeshEntity* tet)
   int type = p->mesh->getType(tet);
   int nd = apf::countElementNodes(el->getFieldShape(), type);
   apf::NewArray<double> d (nd);
-  el->getElementDofs(d);
+  el-getElementDofs(d);
   mth::Vector<double> dofs (nd);
   for (int i = 0; i < nd; i++) // TODO cleanup
     dofs(i) = d[i];
@@ -337,7 +337,7 @@ static double getLocalEdgeBLF(EdgePatch* p, apf::MeshEntity* tet)
   mth::multiply(elmat, dofs, blf_integrals);
 
   apf::destroyElement(el);
-  apf::destroyMeshElement(me);
+  apf::destroyMeshElement(me);*/
 
   // pick edge index from the resulting vector
   // negation of negative ND dofs
@@ -346,8 +346,97 @@ static double getLocalEdgeBLF(EdgePatch* p, apf::MeshEntity* tet)
   if (flip)
     blf_integrals(ei) = -1*blf_integrals(ei);*/
 
-  cout << "local blf " << blf_integrals(ei) << endl;
-  return blf_integrals(ei);
+  /*cout << "local blf " << blf_integrals(ei) << endl;
+  return blf_integrals(ei);*/
+  // 0. findIn edge in downward edges of tet
+  apf::Downward e;
+  int ne = ep->mesh->getDownward(tet, 1, e);
+  PCU_ALWAYS_ASSERT(ne == 6);
+  int ei = apf::findIn(e, ne, ep->entity);
+
+
+  apf::FieldShape* fs = ep->equilibration->ef->getShape();
+  int type = ep->mesh->getType(tet);
+  int dim = apf::getDimension(ep->mesh, tet);
+
+
+  apf::MeshElement* me = apf::createMeshElement(ep->mesh, tet);
+  apf::Element* el = apf::createElement(ep->equilibration->ef, me);
+  int nd = apf::countElementNodes(el->getFieldShape(), type);
+  int int_order = 2 * fs->getOrder();
+  int np = apf::countIntPoints(me, int_order);
+  double w;
+
+
+  apf::NewArray<apf::Vector3> curlshapes(nd);
+  apf::NewArray<apf::Vector3> vectorshapes(nd);
+  mth::Matrix<double> phys_curlshapes(nd, dim);  // TODO clean
+
+
+  // 1. Curl Curl Integration
+  double curlcurl = 0.0;
+
+  apf::Vector3 p, curl;
+  for (int i = 0; i < np; i++) {
+    apf::getIntPoint(me, int_order, i, p);
+    double weight = apf::getIntWeight(me, int_order, i);
+    apf::Matrix3x3 J;
+    apf::getJacobian(me, p, J);
+    //double jdet = apf::getJacobianDeterminant(J, dim);
+    w = weight; // TODO check why do not need division by jdet
+
+    // get curl vector
+    apf::getCurl(el, p, curl);
+
+    // get curlshape values // TODO CLEAN use getCurlShapeValues
+    el->getShape()->getLocalVectorCurls(ep->mesh, tet, p, curlshapes);
+    phys_curlshapes.zero();
+    for (int i = 0; i < nd; i++)
+      for (int j = 0; j < dim; j++)
+        for (int k = 0; k < dim; k++)
+          phys_curlshapes(i,j) += curlshapes[i][k] * J[k][j];
+
+    // pick the ei curlshape // TODO Clean apf::Vector
+    apf::Vector3 cshape;
+    cshape[0] = phys_curlshapes(ei, 0);
+    cshape[1] = phys_curlshapes(ei, 1);
+    cshape[2] = phys_curlshapes(ei, 2);
+
+    // multiply
+    curlcurl += (curl * cshape) * w;
+  }
+
+  // 2. Vector Mass Integration
+  int_order = 2 * fs->getOrder();
+  np = apf::countIntPoints(me, int_order); // int points required
+
+  double vectormass = 0.0;
+  apf::Vector3 vvalue;
+  for (int i = 0; i < np; i++) {
+    apf::getIntPoint(me, int_order, i, p);
+    double weight = apf::getIntWeight(me, int_order, i);
+    apf::Matrix3x3 J;
+    apf::getJacobian(me, p, J);
+    double jdet = apf::getJacobianDeterminant(J, dim);
+
+    w = weight * jdet;
+
+    apf::getVector(el, p, vvalue);
+
+    apf::getVectorShapeValues(el, p, vectorshapes);
+    apf::Vector3 vshape = vectorshapes[ei];
+
+    vectormass += (vvalue * vshape) * w;
+  }
+
+  double blf_integral = curlcurl + vectormass;
+
+  int which, rotate; bool flip;
+  apf::getAlignment(ep->mesh, tet, ep->entity, which, flip, rotate);
+  if (flip)
+    blf_integral = -1*blf_integral;
+  cout << "local blf " << blf_integral << endl;
+  return blf_integral;
 }
 
 
@@ -431,10 +520,10 @@ static double getLocalEdgeLF(EdgePatch* p, apf::MeshEntity* tet)
   mth::Vector<double> elvect;
   assembleDomainLFElementVector(p->mesh, tet,
       p->equilibration->ef, elvect);
-  /*int which, rotate; bool flip;
+  int which, rotate; bool flip;
   apf::getAlignment(p->mesh, tet, p->entity, which, flip, rotate);
   if (flip)
-    elvect(ei) = -1*elvect(ei);*/
+    elvect(ei) = -1*elvect(ei);
   cout << "local lf " << elvect(ei) << endl;
   return elvect(ei);
 }
@@ -637,6 +726,8 @@ static void assembleEdgePatchRHS(EdgePatch* p)
     double fluxIntegral = getLocalFluxIntegral(p, tet);
     testblflf += (blfIntegral - lfIntegral);
     testflux += fluxIntegral;
+    cout << "Blf Integral" << blfIntegral << endl;
+    cout << "Lf Integral" << lfIntegral << endl;
     if(p->isOnBdry)
       p->b(nf+i) = blfIntegral - lfIntegral - fluxIntegral;
     else

From 1fa0e87dfbdfc6697f4ddcf212f1cabe2a00ab7c Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Thu, 21 May 2020 00:44:33 -0400
Subject: [PATCH 170/555] removes negation in face matrix, fixes gs.

Removes negation of shape functions of flipped edges in
computation of face mass matrix.
Stores gs on a face on a tri Nedelec field so that the
negation of the dofs is handled automatically when solving
the face system of equations for theta coefficients.
---
 em/emFluxCorrection.cc | 82 ++++++++++++++++++++----------------------
 1 file changed, 38 insertions(+), 44 deletions(-)

diff --git a/em/emFluxCorrection.cc b/em/emFluxCorrection.cc
index 8e6ffae44..31b454c8e 100644
--- a/em/emFluxCorrection.cc
+++ b/em/emFluxCorrection.cc
@@ -37,11 +37,7 @@ static void assembleFaceMassMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
   apf::MeshElement* me = apf::createMeshElement(mesh, e);
   apf::Element* el = apf::createElement(f, me);
   int int_order = 2 * fs->getOrder();
-  int np = apf::countIntPoints(me, int_order); // int points required
-
-  apf::Downward edges;
-  int ned =  mesh->getDownward(e, 1, edges);
-  int which, rotate; bool flip;
+  int np = apf::countIntPoints(me, int_order);
 
   elmat.zero();
   apf::Vector3 p;
@@ -59,17 +55,6 @@ static void assembleFaceMassMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
       for (int k = 0; k < sdim; k++)
         vectorShapes(j,k) = vectorshapes[j][k];
 
-    // negate negative dof indices
-    // TODO maybe can do this outside the function and therfore
-    // merge this function with assembleVectorMassElementMatrix
-    for (int ei = 0; ei < ned; ei++) {
-      apf::getAlignment(mesh, e, edges[ei], which, flip, rotate);
-      if(flip) {
-        for (int j = 0; j < sdim; j++)
-          vectorShapes(ei, j) = -1*vectorShapes(ei, j);
-      }
-    }
-
     mth::Matrix<double> vectorShapesT (sdim, nd);
     mth::transpose(vectorShapes, vectorShapesT);
     mth::Matrix<double> M (nd,nd);
@@ -87,53 +72,62 @@ static void assembleFaceMassMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
 // correction vectors on faces and stores them in a field
 apf::Field* computeFluxCorrection(apf::Field* ef, apf::Field* g)
 {
+  apf::Mesh* mesh = apf::getMesh(ef);
   int order = ef->getShape()->getOrder();
-  cout << "tri_field_order " << order << endl; // REMOVE
+
   apf::Field* triNedelecField = apf::createField(
-      apf::getMesh(ef), "tri_nedelec_field", apf::SCALAR, apf::getNedelec(order));
+      mesh, "tri_nedelec_field", apf::SCALAR, apf::getNedelec(order));
   apf::zeroField(triNedelecField);
 
   apf::Field* correctedFluxField =  createPackedField(
-      apf::getMesh(ef), "corrected_flux_field", 3, apf::getConstant(2));
+      mesh, "corrected_flux_field", 3, apf::getConstant(2));
 
-  // iterate over all faces of the mesh
   apf::MeshEntity* face;
-  apf::MeshIterator* it = apf::getMesh(ef)->begin(2);
-  while ((face = apf::getMesh(ef)->iterate(it))) {
+  apf::MeshIterator* it = mesh->begin(2);
+  while ((face = mesh->iterate(it))) {
+
     // 1. assemble RHS vector
     double components[3];
     apf::getComponents(g, face, 0, components);
-    mth::Vector<double> rhs(3);
-    rhs(0) = components[0]; rhs(1) = components[1]; rhs(2) = components[2];
-
-    bool debug = true;
-    if (debug) {
-      cout << "RHS VECTOR" << endl;
-      cout << components[0] << " " << components[1] << " " << components[2] << endl;
-      cout << rhs << endl;
-      cout << "==============" << endl;
+    cout << "initial gs without negation" << endl; // REMOVE
+    cout << components[0] << " " << components[1] << " " << components[2] << endl; // REMOVE
+
+    apf::Downward edges;
+    int ne = mesh->getDownward(face, 1, edges);
+    for (int i = 0; i < ne; i++) {
+      apf::setScalar(triNedelecField, edges[i], 0, components[i]);
+    }
+
+    apf::MeshElement* me = apf::createMeshElement(mesh, face);
+    apf::Element* el = apf::createElement(triNedelecField, me);
+    apf::NewArray<double> facegs;
+    el->getElementDofs(facegs);
+
+    mth::Vector<double> rhs(facegs.size()); // TODO clean
+    for (size_t i = 0; i < facegs.size(); i++) {
+      rhs(i) = facegs[i];
     }
+    cout << "final gs with negation" << endl; // REMOVE
+    cout << rhs << endl; // REMOVE
 
-    // 2. assemble face mass matrix
+    apf::destroyElement(el);
+    apf::destroyMeshElement(me);
+
+    // 2. assemble LHS face mass matrix
     mth::Matrix<double> M;
     assembleFaceMassMatrix(
-        apf::getMesh(ef), face, triNedelecField, M);
+        mesh, face, triNedelecField, M);
 
     // 3. solve the system
     QRDecomp qr;
     mth::decomposeQR(M, qr.Q, qr.R);
-    std::cout << "M" << std::endl; // REMOVE DEBUG
-    std::cout << M << std::endl;
-    std::cout << "Q" << std::endl;
-    std::cout << qr.Q << std::endl;
-    std::cout << "R" << std::endl;
-    std::cout << qr.R << std::endl;
-    std::cout << "RHS" << std::endl;
-    std::cout << rhs << std::endl;
-    
     mth::Vector<double> theta;
     mth::solveFromQR(qr.Q, qr.R, rhs, theta);
-    std::cout << "theta" << std::endl;
+    std::cout << "LHS Face Mass Matrix" << std::endl; // REMOVE DEBUG
+    std::cout << M << std::endl;
+    std::cout << "RHS Vector" << std::endl;
+    std::cout << rhs << std::endl;
+    std::cout << "theta coeffs" << std::endl;
     std::cout << theta << std::endl;
 
     // set solution vector on face field
@@ -144,7 +138,7 @@ apf::Field* computeFluxCorrection(apf::Field* ef, apf::Field* g)
     apf::setComponents(
         correctedFluxField, face, 0, components);
   }
-  apf::getMesh(ef)->end(it);
+  mesh->end(it);
 
   apf::destroyField(triNedelecField);
 

From ef734510da91a10fe46ed3482445cb37544329b5 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Fri, 22 May 2020 15:05:46 -0400
Subject: [PATCH 171/555] Fixes At*mu=g bug in solving edge patch system.

---
 em/emResidualFunctionals.cc | 108 +++++++++++++++++++++++-------------
 1 file changed, 70 insertions(+), 38 deletions(-)

diff --git a/em/emResidualFunctionals.cc b/em/emResidualFunctionals.cc
index fe0515fc5..9c809881e 100644
--- a/em/emResidualFunctionals.cc
+++ b/em/emResidualFunctionals.cc
@@ -67,6 +67,8 @@ struct EdgePatch {
   EntityVector tets;
   EntityVector faces;
   mth::Matrix<double> A;
+  mth::Matrix<double> At;
+  mth::Matrix<double> T; // T = A*At + 1
   mth::Vector<double> b;
   mth::Vector<double> x;
   QRDecomp qr;
@@ -148,36 +150,35 @@ static void assembleEdgePatchLHS(EdgePatch* p)
   int ne = p->tets.size();
   int nf = p->faces.size();
   if( crv::isBoundaryEntity(p->mesh, p->entity) ) {
-    p->A.resize(ne+nf, ne+nf);
-    p->A.zero();
+    p->T.resize(ne+nf, ne+nf);
+    p->T.zero();
     for (int i = 0; i < nf; i++)
-      p->A(i,i) = 2.;
+      p->T(i,i) = 2.;
     for (int i = 0; i < ne-1; i++) {
-      p->A(i+nf,i) = 1.; p->A(i+nf,i+1) = -1.;
-      p->A(i,i+nf) = 1.; p->A(i+1,i+nf) = -1.;
+      p->T(i+nf,i) = 1.; p->T(i+nf,i+1) = -1.;
+      p->T(i,i+nf) = 1.; p->T(i+1,i+nf) = -1.;
     }
-    p->A(ne+nf-1, ne-1) = 1.; p->A(ne+nf-1, ne) = 1.;
-    p->A(ne-1, ne+nf-1) = 1.; p->A(ne, ne+nf-1) = 1.;
+    p->T(ne+nf-1, ne-1) = 1.; p->T(ne+nf-1, ne) = 1.;
+    p->T(ne-1, ne+nf-1) = 1.; p->T(ne, ne+nf-1) = 1.;
   }
   else if( ! crv::isBoundaryEntity(p->mesh, p->entity) ) {
-    mth::Matrix<double> m(ne, nf);
-    m.zero();
+    p->A.resize(ne, nf);
+    p->A.zero();
     for (int i = 0; i < ne-1; i++) {
-      m(i,i) = 1.;  m(i,i+1) = -1.;
+      p->A(i,i) = 1.;  p->A(i,i+1) = -1.;
     }
-    m(ne-1,0) = -1.; m(ne-1,ne-1) = 1.;
-
-    // m is singular so do (m*mt) + 1.0
+    p->A(ne-1,0) = -1.; p->A(ne-1,ne-1) = 1.;
+    // A is singular so do (A*At) + 1.0
     // to pick a particular solution
-    mth::Matrix<double> mt(nf,ne);
-    mth::transpose(m, mt);
-    mth::multiply(m, mt, p->A);
+    p->At.resize(nf,ne);
+    mth::transpose(p->A, p->At);
+    mth::multiply(p->A, p->At, p->T);
 
     for (int i = 0; i < ne; i++)
       for (int j = 0; j < ne; j++)
-        p->A(i,j) += 1.;
+        p->T(i,j) += 1.;
   }
-  mth::decomposeQR(p->A, p->qr.Q, p->qr.R);
+  mth::decomposeQR(p->T, p->qr.Q, p->qr.R);
 }
 
 void assembleCurlCurlElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
@@ -304,29 +305,30 @@ void assembleElementMatrix(apf::Mesh* mesh, apf::MeshEntity*e,
 // an edge of a tet element
 static double getLocalEdgeBLF(EdgePatch* ep, apf::MeshEntity* tet)
 {
-  /* findIn edge in downward edges of tet
+  // findIn edge in downward edges of tet
   apf::Downward e;
-  int ne = p->mesh->getDownward(tet, 1, e);
+  int ne = ep->mesh->getDownward(tet, 1, e);
   PCU_ALWAYS_ASSERT(ne == 6);
-  int ei = apf::findIn(e, ne, p->entity);
+  int ei = apf::findIn(e, ne, ep->entity);
   // get Element Dofs
-  apf::MeshElement* me = apf::createMeshElement(p->mesh, tet);
-  apf::Element* el = apf::createElement(p->equilibration->ef, me);
-  int type = p->mesh->getType(tet);
+  apf::MeshElement* me = apf::createMeshElement(ep->mesh, tet);
+  apf::Element* el = apf::createElement(ep->equilibration->ef, me);
+  int type = ep->mesh->getType(tet);
   int nd = apf::countElementNodes(el->getFieldShape(), type);
   apf::NewArray<double> d (nd);
-  el-getElementDofs(d);
+  el->getElementDofs(d);
   mth::Vector<double> dofs (nd);
   for (int i = 0; i < nd; i++) // TODO cleanup
     dofs(i) = d[i];
+  cout << "Element Dofs "<< dofs << endl;
   // assemble curl curl element matrix
   mth::Matrix<double> curl_elmat;
-  assembleCurlCurlElementMatrix(p->mesh, tet,
-      p->equilibration->ef, curl_elmat);
+  assembleCurlCurlElementMatrix(ep->mesh, tet,
+      ep->equilibration->ef, curl_elmat);
   // assemble vector mass element matrix
   mth::Matrix<double> mass_elmat;
-  assembleVectorMassElementMatrix(p->mesh, tet,
-      p->equilibration->ef, mass_elmat);
+  assembleVectorMassElementMatrix(ep->mesh, tet,
+      ep->equilibration->ef, mass_elmat);
   // add element matrices
   mth::Matrix<double> elmat(nd, nd);
   elmat.zero();
@@ -337,18 +339,19 @@ static double getLocalEdgeBLF(EdgePatch* ep, apf::MeshEntity* tet)
   mth::multiply(elmat, dofs, blf_integrals);
 
   apf::destroyElement(el);
-  apf::destroyMeshElement(me);*/
+  apf::destroyMeshElement(me);
 
   // pick edge index from the resulting vector
   // negation of negative ND dofs
-  /*int which, rotate; bool flip;
-  apf::getAlignment(p->mesh, tet, p->entity, which, flip, rotate);
+  int which, rotate; bool flip;
+  apf::getAlignment(ep->mesh, tet, ep->entity, which, flip, rotate);
   if (flip)
-    blf_integrals(ei) = -1*blf_integrals(ei);*/
+    blf_integrals(ei) = -1*blf_integrals(ei);
 
-  /*cout << "local blf " << blf_integrals(ei) << endl;
-  return blf_integrals(ei);*/
-  // 0. findIn edge in downward edges of tet
+  cout << "local blf " << blf_integrals(ei) << endl;
+  return blf_integrals(ei);
+
+  /* 0. findIn edge in downward edges of tet
   apf::Downward e;
   int ne = ep->mesh->getDownward(tet, 1, e);
   PCU_ALWAYS_ASSERT(ne == 6);
@@ -436,7 +439,7 @@ static double getLocalEdgeBLF(EdgePatch* ep, apf::MeshEntity* tet)
   if (flip)
     blf_integral = -1*blf_integral;
   cout << "local blf " << blf_integral << endl;
-  return blf_integral;
+  return blf_integral;*/
 }
 
 
@@ -694,6 +697,14 @@ static double getLocalFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
       apf::getVectorShapeValues(fel, p, vectorshapes);
       vshape = vectorshapes[ei];
 
+      int which, rotate; bool flip;
+      apf::getAlignment(
+          ep->mesh, currentFace, ep->entity, which, flip, rotate);
+      if (flip) {
+      vshape = vshape * -1.;
+      }
+
+
       // compute integral
       fluxFaceIntegral += (tk * vshape) * weight * jdet;
     }
@@ -820,11 +831,30 @@ static void getOrderedTetsandFaces(apf::Mesh* mesh, apf::MeshEntity* edge,
 
 static void runErm(EdgePatch* p)
 {
+  // DEBUG
+  cout << "PUMI edge vertex coordinates" << endl;
+  apf::Downward ev;
+  int nev = p->mesh->getDownward(p->entity, 0, ev);
+  PCU_ALWAYS_ASSERT(nev == 2);
+  for (int i = 0; i < nev; i++) {
+    apf::Vector3 pt;
+    p->mesh->getPoint(ev[i], 0, pt);
+    cout << pt << endl;
+  } // REMOVE DEBUG
+
   getOrderedTetsandFaces(p->mesh, p->entity, p->tets, p->faces);
   assembleEdgePatchLHS(p);
   assembleEdgePatchRHS(p);
   mth::solveFromQR(p->qr.Q, p->qr.R, p->b, p->x);
 
+  if (!p->isOnBdry) { // solve At*mu = g for g
+    mth::Vector<double> temp(p->tets.size());
+    mth::multiply(p->At, p->x, temp);
+    for (size_t i = 0; i < p->tets.size(); i++) {
+      p->x(i) = temp(i);
+    }
+  }
+
   bool debug = true; // REMOVE DEBUG
   if (debug) {
     if (p->isOnBdry)
@@ -833,7 +863,7 @@ static void runErm(EdgePatch* p)
       cout << "Interior Patch" << endl;
     cout << "ne " << p->tets.size() << " nf " << p->faces.size() << endl;
     cout << "LHS Matrix" << endl;
-    cout << p->A << endl;
+    cout << p->T << endl;
 
     cout << "RHS Vector" << endl;
     cout << p->b << endl;
@@ -881,6 +911,8 @@ static void runErm(EdgePatch* p)
 
   std::cout << "x" << std::endl; // REMOVE
   std::cout << p->x << std::endl; // REMOVE
+
+  cout << "======================================================" << endl;
 }
 
 

From 23857c81143b252f89cd04d11d47fbaf96f01b46 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Fri, 22 May 2020 17:21:55 -0400
Subject: [PATCH 172/555] Fixes tk for boundary edge patch flux integrals.

For boundary face, tk = outward face normal x curl of solution.

For interior face, tk = ( (nK x curl K) - (nL x curl L) ) / 2
---
 em/emResidualFunctionals.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/em/emResidualFunctionals.cc b/em/emResidualFunctionals.cc
index 9c809881e..3743deb51 100644
--- a/em/emResidualFunctionals.cc
+++ b/em/emResidualFunctionals.cc
@@ -681,13 +681,13 @@ static double getLocalFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
         apf::getCurl(el2, tet2xi, curl2);
         apf::Vector3 temp2 = apf::cross(fnormal2, curl2);
         curl += (temp2 * -1.);
+        curl = curl * 1./2.; //
         apf::destroyElement(el2);
         apf::destroyMeshElement(me2);
       }
 
       // compute tk (inter-element averaged flux)
-      //tk = apf::cross(fnormal, curl);
-      tk = curl * 1./2.;
+      tk = curl;
       std::cout << "tk " << tk << std::endl; // REMOVE
 
       // compute vector shape

From b07de8a6a57dd79d3bdbfc483ee6351392a9bb2e Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Sat, 23 May 2020 17:04:05 -0400
Subject: [PATCH 173/555] Fixes tk computation for boundary face

---
 em/emEstimateError.cc | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/em/emEstimateError.cc b/em/emEstimateError.cc
index 92296681c..3ea731948 100644
--- a/em/emEstimateError.cc
+++ b/em/emEstimateError.cc
@@ -241,13 +241,16 @@ static void computeLambdaVector(
         apf::getCurl(el2, tet2xi, curl2);
         apf::Vector3 temp2 = apf::cross(fnormal2, curl2);
         curl += (temp2 * -1.);
+        curl = curl * 1./2.; //
         apf::destroyElement(el2);
         apf::destroyMeshElement(me2);
       }
 
       // compute tk (inter-element averaged flux)
-      tk = curl * 1./2.;
-      std::cout << "tk " << tk << std::endl; // REMOVE
+      tk = curl;
+      std::cout << "tk           " << tk << std::endl; // REMOVE
+      std::cout << "theta_coeffs " << theta_coeffs << std::endl; // REMOVE
+      std::cout << "theta_vector " << theta_vector << std::endl; // REMOVE
 
       // compute p+1 order 3D vector shapes
       apf::NewArray<apf::Vector3> tetVectorShapes (nedofs);
@@ -465,7 +468,9 @@ apf::Field* emEstimateError(apf::Field* ef, apf::Field* correctedFlux)
   // 2. iterate over all elements of the mesh
   apf::MeshEntity* el;
   apf::MeshIterator* it = apf::getMesh(ef)->begin(3);
+  int elemNo = 0; // remove
   while ((el = apf::getMesh(ef)->iterate(it))) {
+    cout << "elemNo " << elemNo << endl;
     // 2(a). Assemble LHS element matrix
     mth::Matrix<double> A;
     assembleElementMatrix( apf::getMesh(ef), el, efp1, A);
@@ -536,6 +541,8 @@ apf::Field* emEstimateError(apf::Field* ef, apf::Field* correctedFlux)
 
 		apf::setScalar(error_field, el, 0, l2_error);
     cout << "Write the L2 error to error_field" << endl;
+
+    elemNo++;
   }
   apf::getMesh(ef)->end(it);
   cout << "End loop over elements" << endl;

From 4aab41145d7bd3c9e4371cb6f6ea692b34de37e0 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Tue, 26 May 2020 08:58:11 -0400
Subject: [PATCH 174/555] pumi version 2.2.3

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 207ca93f2..bb3ddd839 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -6,7 +6,7 @@ endif()
 # This is the top level CMake file for the SCOREC build
 cmake_minimum_required(VERSION 3.0)
 
-project(SCOREC VERSION 2.2.2 LANGUAGES CXX C)
+project(SCOREC VERSION 2.2.3 LANGUAGES CXX C)
 
 include(cmake/bob.cmake)
 include(cmake/xsdk.cmake)

From 918d05db8cbf1d8f515a3de3bb2f4166e9119db4 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Fri, 29 May 2020 14:26:47 -0400
Subject: [PATCH 175/555] Fixes curl electric field in blf vector integral

---
 em/emEstimateError.cc | 55 ++++++++++++++++++++++++++++++-------------
 1 file changed, 39 insertions(+), 16 deletions(-)

diff --git a/em/emEstimateError.cc b/em/emEstimateError.cc
index 3ea731948..bbe16827e 100644
--- a/em/emEstimateError.cc
+++ b/em/emEstimateError.cc
@@ -17,12 +17,12 @@ namespace em {
 //TODO destroy all fields created inside functions to prevent memory leaks.
 
 static void computeResidualBLF(apf::Mesh* mesh, apf::MeshEntity* e,
-  apf::Field* f, mth::Vector<double>& blf)
+  apf::Field* f, apf::Field* fp1, mth::Vector<double>& blf)
 {
-  apf::FieldShape* fs = f->getShape();
+  apf::FieldShape* fp1s = fp1->getShape();
   int type = mesh->getType(e);
   PCU_ALWAYS_ASSERT(type == apf::Mesh::TET);
-  int nd = apf::countElementNodes(fs, type);
+  int nd = apf::countElementNodes(fp1s, type);
   int dim = apf::getDimension(mesh, e);
   double w;
 
@@ -35,8 +35,9 @@ static void computeResidualBLF(apf::Mesh* mesh, apf::MeshEntity* e,
   blf.resize(nd);
 
   apf::MeshElement* me = apf::createMeshElement(mesh, e);
-  apf::Element* el = apf::createElement(f, me);
-  int int_order = 2 * fs->getOrder();
+  apf::Element* fp1el = apf::createElement(fp1, me);
+  apf::Element* fel = apf::createElement(f, me);
+  int int_order = 2 * fp1s->getOrder();
   int np = apf::countIntPoints(me, int_order); // int points required
 
   // 1. Compute Curl Curl Integration
@@ -51,10 +52,10 @@ static void computeResidualBLF(apf::Mesh* mesh, apf::MeshEntity* e,
     w = weight; // TODO check why do not need division by jdet
 
     // get curl vector
-    apf::getCurl(el, p, curl);
+    apf::getCurl(fel, p, curl);
 
     // get curlshape values // TODO CLEAN use getCurlShapeValues
-    el->getShape()->getLocalVectorCurls(mesh, e, p, curlshape);
+    fp1el->getShape()->getLocalVectorCurls(mesh, e, p, curlshape);
     phys_curlshape.zero();
     for (int i = 0; i < nd; i++)
       for (int j = 0; j < dim; j++)
@@ -66,14 +67,16 @@ static void computeResidualBLF(apf::Mesh* mesh, apf::MeshEntity* e,
     V.zero();
     mth::Vector<double> c (dim);
     c(0) = curl[0]; c(1) = curl[1]; c(2) = curl[2];
+    cout << "curl " << c << endl;
     mth::multiply(phys_curlshape, c, V);
+    cout << "V " << V << endl;
     V *= w;
 
     curlcurl_vec += V;
   }
 
   // 2. Compute Vector Mass Integration
-  int_order = 2 * fs->getOrder();
+  int_order = 2 * fp1s->getOrder();
   np = apf::countIntPoints(me, int_order); // int points required
 
   mass_vec.zero();
@@ -87,9 +90,9 @@ static void computeResidualBLF(apf::Mesh* mesh, apf::MeshEntity* e,
 
     w = weight * jdet;
 
-    apf::getVector(el, p, vvalue);
+    apf::getVector(fel, p, vvalue);
 
-    apf::getVectorShapeValues(el, p, vectorshape);
+    apf::getVectorShapeValues(fp1el, p, vectorshape);
     vectorShape.zero();
     for (int i = 0; i < nd; i++)
       for (int j = 0; j < dim; j++)
@@ -111,6 +114,10 @@ static void computeResidualBLF(apf::Mesh* mesh, apf::MeshEntity* e,
   blf += curlcurl_vec;
   blf += mass_vec;
 
+  apf::destroyMeshElement(me);
+  apf::destroyElement(fp1el);
+  apf::destroyElement(fel);
+
   // TODO Take care of Negative Dofs
 }
 
@@ -198,11 +205,11 @@ static void computeLambdaVector(
       theta_vector.zero();
       apf::NewArray<apf::Vector3> triVectorShapes (nfdofs);
       apf::getVectorShapeValues(fel, p, triVectorShapes);
-      int which, rotate; bool flip; // negative ND dofs
+      //int which, rotate; bool flip; // negative ND dofs
       for (int i = 0; i < nedges; i++) {
-        apf::getAlignment(mesh, face, edges[i], which, flip, rotate);
         apf::Vector3 v = triVectorShapes[i];
-        if (flip) { v = v * -1.; }
+        /*apf::getAlignment(mesh, face, edges[i], which, flip, rotate);
+        if (flip) { v = v * -1.; }*/
 
         v = v * theta_coeffs[i];
         theta_vector += v;
@@ -216,8 +223,10 @@ static void computeLambdaVector(
       if (up.n == 2) {
         if (e == firstTet)
           fnormal2 = computeFaceOutwardNormal(mesh, secondTet, face, p);
-        else
+        else {
           fnormal2 = computeFaceOutwardNormal(mesh, firstTet, face, p);
+          theta_vector = theta_vector * -1.;
+        }
         std::cout << "normal1 " << fnormal1 << std::endl; // REMOVE
         std::cout << "normal2 " << fnormal2 << std::endl; // REMOVE
       }
@@ -430,8 +439,11 @@ static double computeL2Error(apf::Mesh* mesh, apf::MeshEntity* e,
 
 		mth::Vector<double> err_func;
 		mth::multiply(vectorShapeT, error_dofs, err_func);
+		cout << "vshape   " << vectorShape << endl;
+		cout << "err_func " << err_func << endl;
 
 		error += w * (err_func * err_func);
+	  cout << "error squared " << error << endl;
 	}
 	if (error < 0.0)
 		error = -error;
@@ -476,23 +488,33 @@ apf::Field* emEstimateError(apf::Field* ef, apf::Field* correctedFlux)
     assembleElementMatrix( apf::getMesh(ef), el, efp1, A);
     // TODO Take care of negative dofs in lhs
     cout << "LHS matrix assembled" << endl;
+    cout << A << endl;
 
     // 2(b). Compute Bilinear Form Vector
     mth::Vector<double> blf;
-    computeResidualBLF(apf::getMesh(efp1), el, efp1, blf);
+    computeResidualBLF(apf::getMesh(efp1), el, ef, efp1, blf);
     // TODO Take care of negative dofs in blf
     cout << "RHS bilinear form vector assembled" << endl;
+    cout << blf << endl;
 
     // 2(c). Compute Linear Form Vector
     mth::Vector<double> lf;
     assembleDomainLFElementVector(apf::getMesh(efp1), el, efp1, lf);
     // TODO Take care of negative dofs in lf
     cout << "RHS linear form vector assembled" << endl;
+    cout << lf << endl;
 
     // 2(d). Compute Lamda Vector
     mth::Vector<double> lambda;
     computeLambdaVector(
         apf::getMesh(ef), el, ef, efp1, tri_nedelec_field, correctedFlux, lambda);
+    cout << "lambda vector " << lambda << endl;
+    double lambda_sum = 0.;
+    for(size_t i = 0; i < lambda.size(); i++) {
+      lambda_sum += lambda(i);
+    }
+    cout << "lambda_sum " << lambda_sum << endl;
+
     // TODO Take care of negative dofs in lambda
     cout << "RHS lambda vector assembled" << endl;
 
@@ -534,10 +556,11 @@ apf::Field* emEstimateError(apf::Field* ef, apf::Field* correctedFlux)
       error_dofs(index) = Xnew(i);
     }
     cout << "Recover the solution" << endl;
+    cout << error_dofs << endl;
 
 		// 2(j). Compute L2 Norm Error
 		double l2_error = computeL2Error(apf::getMesh(ef), el, efp1, error_dofs);
-		cout << "Compute L2 element error" << endl;
+		cout << "Compute L2 element error" << l2_error << endl;
 
 		apf::setScalar(error_field, el, 0, l2_error);
     cout << "Write the L2 error to error_field" << endl;

From d55b482e2cc41b9bb7855e6532e35a3e957e87c6 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Fri, 29 May 2020 18:29:16 -0400
Subject: [PATCH 176/555] Fixes bug in mth::Transpose

---
 mth/mth_def.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mth/mth_def.h b/mth/mth_def.h
index b05c31945..393c2f442 100644
--- a/mth/mth_def.h
+++ b/mth/mth_def.h
@@ -57,12 +57,12 @@ template <class T, unsigned M, unsigned N>
 void transpose(Matrix<T,M,N> const& a,
     Matrix<T,N,M>& b)
 {
-  unsigned m = a.rows();
+  unsigned m = a.cols();
   unsigned n = a.rows();
   b.resize(m, n);
   for (unsigned i=0; i < m; ++i)
   for (unsigned j=0; j < n; ++j)
-    b(j,i) = a(i,j);
+    b(i,j) = a(j,i);
 }
 
 template <class T>

From 083246fa3679a0defc2ede44c3a1ef6f949367fa Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Fri, 29 May 2020 18:29:58 -0400
Subject: [PATCH 177/555] Working code of error estimator.

TODOs:
1) Clean up.
2) Check the effect of right hand ordering.
3) Check the effect of negating gs when assembling RHS on faces.
4) Add Natural BCs in flux and lambda vector computation.
---
 em/emEstimateError.cc       |  12 ++++
 em/emFluxCorrection.cc      |  40 ++++++++++-
 em/emResidualFunctionals.cc | 139 +++++++++++++++++++++++++++++++++++-
 3 files changed, 188 insertions(+), 3 deletions(-)

diff --git a/em/emEstimateError.cc b/em/emEstimateError.cc
index bbe16827e..9394c0a80 100644
--- a/em/emEstimateError.cc
+++ b/em/emEstimateError.cc
@@ -433,13 +433,16 @@ static double computeL2Error(apf::Mesh* mesh, apf::MeshEntity* e,
     for (int j = 0; j < nd; j++)
       for (int k = 0; k < dim; k++)
         vectorShape(j,k) = vectorshape[j][k];
+    cout << "vector shapes" << vectorShape << endl;
 
     mth::Matrix<double> vectorShapeT (dim, nd);
     mth::transpose(vectorShape, vectorShapeT);
+    cout << "transpose vector shapes" << vectorShapeT << endl;
 
 		mth::Vector<double> err_func;
 		mth::multiply(vectorShapeT, error_dofs, err_func);
 		cout << "vshape   " << vectorShape << endl;
+		cout << "error dofs " << error_dofs << endl;
 		cout << "err_func " << err_func << endl;
 
 		error += w * (err_func * err_func);
@@ -529,6 +532,15 @@ apf::Field* emEstimateError(apf::Field* ef, apf::Field* correctedFlux)
     getEssentialElementNDDofs(
         apf::getMesh(efp1), el, efp1, ess_dofs, uness_dofs);
     cout << "Get list of Essential Dofs" << endl;
+    for(size_t i = 0; i < ess_dofs.size(); i++) {
+      cout << ess_dofs[i] << " ";
+    }
+    cout << endl;
+    cout << "Get list of UnEssential Dofs" << endl;
+    for(size_t i = 0; i < uness_dofs.size(); i++) {
+      cout << uness_dofs[i] << " ";
+    }
+    cout << endl;
 
     // 2(g). eliminate Dirichlet (Essential) Boundary Conditions
 		mth::Vector<double> X, Bnew;
diff --git a/em/emFluxCorrection.cc b/em/emFluxCorrection.cc
index 31b454c8e..1f359f4ed 100644
--- a/em/emFluxCorrection.cc
+++ b/em/emFluxCorrection.cc
@@ -54,6 +54,20 @@ static void assembleFaceMassMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
     for (int j = 0; j < nd; j++)
       for (int k = 0; k < sdim; k++)
         vectorShapes(j,k) = vectorshapes[j][k];
+    cout << "Face Tri Shapes" << endl;
+    cout << vectorShapes << endl;
+
+    /*apf::Downward edges;
+    int nedges = mesh->getDownward(e, 1, edges);
+    int which, rotate; bool flip; // negative ND dofs
+    for (int i = 0; i < nedges; i++) {
+      apf::getAlignment(mesh, e, edges[i], which, flip, rotate);
+        if (flip) {
+          for (int j = 0; j < sdim; j++) {
+            vectorShapes(i, j) = -1. * vectorShapes(i,j);
+          }
+        }
+    }*/
 
     mth::Matrix<double> vectorShapesT (sdim, nd);
     mth::transpose(vectorShapes, vectorShapesT);
@@ -82,10 +96,23 @@ apf::Field* computeFluxCorrection(apf::Field* ef, apf::Field* g)
   apf::Field* correctedFluxField =  createPackedField(
       mesh, "corrected_flux_field", 3, apf::getConstant(2));
 
+  int faceNo = 0;
   apf::MeshEntity* face;
   apf::MeshIterator* it = mesh->begin(2);
   while ((face = mesh->iterate(it))) {
 
+    cout << "FaceNo " << faceNo++ << endl;
+    apf::Downward face_vertices;
+    mesh->getDownward(face, 0, face_vertices);
+    apf::Vector3 pt;
+    mesh->getPoint(face_vertices[0], 0, pt);
+    cout << "  v0 " << pt << endl;
+    mesh->getPoint(face_vertices[1], 0, pt);
+    cout << "  v1 " << pt << endl;
+    mesh->getPoint(face_vertices[2], 0, pt);
+    cout << "  v2 " << pt << endl;
+    cout << endl;
+
     // 1. assemble RHS vector
     double components[3];
     apf::getComponents(g, face, 0, components);
@@ -103,10 +130,18 @@ apf::Field* computeFluxCorrection(apf::Field* ef, apf::Field* g)
     apf::NewArray<double> facegs;
     el->getElementDofs(facegs);
 
-    mth::Vector<double> rhs(facegs.size()); // TODO clean
-    for (size_t i = 0; i < facegs.size(); i++) {
+    mth::Vector<double> rhs(3); // TODO clean
+    for (size_t i = 0; i < 3; i++) {
       rhs(i) = facegs[i];
+      //rhs(i) = components[i];
+    }
+    cout << "Face Edges Orientations ";
+    for (int i = 0; i < ne; i++) {
+      int which, rotate; bool flip;
+      apf::getAlignment(mesh, face, edges[i], which, flip, rotate);
+      cout << flip << " ";
     }
+    cout << endl;
     cout << "final gs with negation" << endl; // REMOVE
     cout << rhs << endl; // REMOVE
 
@@ -129,6 +164,7 @@ apf::Field* computeFluxCorrection(apf::Field* ef, apf::Field* g)
     std::cout << rhs << std::endl;
     std::cout << "theta coeffs" << std::endl;
     std::cout << theta << std::endl;
+    cout << "====================" << endl;
 
     // set solution vector on face field
     //theta.toArray(components); TODO clean
diff --git a/em/emResidualFunctionals.cc b/em/emResidualFunctionals.cc
index 3743deb51..eba640b57 100644
--- a/em/emResidualFunctionals.cc
+++ b/em/emResidualFunctionals.cc
@@ -348,7 +348,9 @@ static double getLocalEdgeBLF(EdgePatch* ep, apf::MeshEntity* tet)
   if (flip)
     blf_integrals(ei) = -1*blf_integrals(ei);
 
-  cout << "local blf " << blf_integrals(ei) << endl;
+  cout << "local blf " << blf_integrals(ei);
+  if (flip) { cout << ". local blf before negation " << -1*blf_integrals(ei);}
+  cout << endl;
   return blf_integrals(ei);
 
   /* 0. findIn edge in downward edges of tet
@@ -548,6 +550,20 @@ static apf::MeshEntity* getTetOppVert(
   return 0;
 }
 
+static apf::MeshEntity* getFaceOppVert(
+    apf::Mesh* m, apf::MeshEntity* f, apf::MeshEntity* e)
+{
+  apf::Downward evs;
+  int env = m->getDownward(e, 0, evs);
+  apf::Downward fvs;
+  int fnv = m->getDownward(f, 0, fvs);
+  PCU_ALWAYS_ASSERT(env == 2 && fnv == 3);
+  for (int i = 0; i < fnv; i++) {
+    if (apf::findIn(evs, env, fvs[i]) == -1)
+      return fvs[i];
+  }
+  return 0;
+}
 static apf::Vector3 computeFaceNormal(apf::Mesh* m,
     apf::MeshEntity* f, apf::Vector3 const& p)
 {
@@ -743,6 +759,12 @@ static void assembleEdgePatchRHS(EdgePatch* p)
       p->b(nf+i) = blfIntegral - lfIntegral - fluxIntegral;
     else
       p->b(i) = blfIntegral - lfIntegral - fluxIntegral;
+
+    //DEBUG
+    apf::Downward tetedges;
+    int ntedges = p->mesh->getDownward(tet, 1, tetedges);
+    int ei = apf::findIn(tetedges, ntedges, p->entity);
+    cout << "ei in tet " << ei << endl;
   }
   std::cout << "Blf - lf integrals = " << testblflf << std::endl;
   std::cout << "Flux Integral Sum  = " << testflux << std::endl;
@@ -829,6 +851,120 @@ static void getOrderedTetsandFaces(apf::Mesh* mesh, apf::MeshEntity* edge,
   }
 }
 
+void getClockwiseTetsandFaces(EdgePatch* p)
+{
+  if (p->isOnBdry) {
+    if (p->tets.size() > 1) {
+      apf::MeshEntity* secondFace = p->faces[1];
+      apf::MeshEntity* firstTet = p->tets[0];
+      apf::MeshEntity* secondTet = p->tets[1];
+
+      apf::Downward firstTetFaces, secondTetFaces;
+      int n_firstTetFaces = p->mesh->getDownward(firstTet, 2, firstTetFaces);
+      int n_secondTetFaces = p->mesh->getDownward(secondTet, 2, secondTetFaces);
+      int fi1 = apf::findIn(firstTetFaces, n_firstTetFaces, secondFace);
+      int fi2 = apf::findIn(secondTetFaces, n_secondTetFaces, secondFace);
+      PCU_ALWAYS_ASSERT(fi1 != -1 && fi2 != -1);
+
+      // first tet opp vertex crd
+      apf::MeshEntity* firstTetOppVert = getTetOppVert(
+        p->mesh, firstTet, secondFace);
+      apf::Vector3 firstTetOppVertCrd;
+      p->mesh->getPoint(firstTetOppVert, 0, firstTetOppVertCrd);
+
+      // second tet opp vertex crd
+      apf::MeshEntity* secondTetOppVert = getTetOppVert(
+        p->mesh, secondTet, secondFace);
+      apf::Vector3 secondTetOppVertCrd;
+      p->mesh->getPoint(secondTetOppVert, 0, secondTetOppVertCrd);
+
+      // normal to the face
+      apf::Downward edge_vertices;
+      p->mesh->getDownward(p->entity, 0, edge_vertices);
+      apf::Vector3 p0, p1, p2;
+      p->mesh->getPoint(edge_vertices[0], 0, p0);
+      p->mesh->getPoint(edge_vertices[1], 0, p1);
+      apf::MeshEntity* faceOppVert = getFaceOppVert(
+        p->mesh, secondFace, p->entity);
+      p->mesh->getPoint(faceOppVert, 0, p2);
+
+      apf::Vector3 normal = apf::cross(p1-p0, p2-p0);
+
+      // direction vectors from p0 to opp tet verts
+      apf::Vector3 vFirst = firstTetOppVertCrd - p0;
+      apf::Vector3 vLast = secondTetOppVertCrd - p0;
+
+      if ((vFirst * normal > 0)  && (vLast * normal < 0)) {
+        // reverse list of tets and faces
+        std::reverse(p->tets.begin(), p->tets.end());
+        std::reverse(p->faces.begin(), p->faces.begin());
+      }
+      else if ((vFirst * normal < 0)  && (vLast * normal > 0)) {
+        cout << "already clockwise" << endl;
+      }
+      else if ((vFirst * normal < 0)  && (vLast * normal < 0)) {
+        cout << "failed clockwise. ABORT" << endl;
+      }
+      else if ((vFirst * normal > 0)  && (vLast * normal > 0)) {
+        cout << "failed clockwise. ABORT" << endl;
+      }
+    }
+  }
+  else {
+    apf::MeshEntity* firstFace = p->faces[0];
+    apf::MeshEntity* firstTet = p->tets[0];
+    apf::MeshEntity* lastTet = p->tets[p->tets.size()-1];
+
+    apf::Downward firstTetFaces, lastTetFaces;
+    int n_firstTetFaces = p->mesh->getDownward(firstTet, 2, firstTetFaces);
+    int n_lastTetFaces = p->mesh->getDownward(lastTet, 2, lastTetFaces);
+    int fi1 = apf::findIn(firstTetFaces, n_firstTetFaces, firstFace);
+    int filast = apf::findIn(lastTetFaces, n_lastTetFaces, firstFace);
+    PCU_ALWAYS_ASSERT(fi1 != -1 && filast != -1);
+
+    // first tet opp vertex crd
+    apf::MeshEntity* firstTetOppVert = getTetOppVert(
+        p->mesh, firstTet, firstFace);
+    apf::Vector3 firstTetOppVertCrd;
+    p->mesh->getPoint(firstTetOppVert, 0, firstTetOppVertCrd);
+
+    // last tet opp vertex crd
+    apf::MeshEntity* lastTetOppVert = getTetOppVert(
+        p->mesh, lastTet, firstFace);
+    apf::Vector3 lastTetOppVertCrd;
+    p->mesh->getPoint(lastTetOppVert, 0, lastTetOppVertCrd);
+
+    // normal to the face
+    apf::Downward edge_vertices;
+    p->mesh->getDownward(p->entity, 0, edge_vertices);
+    apf::Vector3 p0, p1, p2;
+    p->mesh->getPoint(edge_vertices[0], 0, p0);
+    p->mesh->getPoint(edge_vertices[1], 0, p1);
+    apf::MeshEntity* faceOppVert = getFaceOppVert(
+        p->mesh, firstFace, p->entity);
+    p->mesh->getPoint(faceOppVert, 0, p2);
+
+    apf::Vector3 normal = apf::cross(p1-p0, p2-p0);
+
+    // direction vectors from p0 to opp tet verts
+    apf::Vector3 vFirst = firstTetOppVertCrd - p0;
+    apf::Vector3 vLast = lastTetOppVertCrd - p0;
+
+    if ((vFirst * normal > 0)  && (vLast * normal < 0)) {
+      // reverse list of tets and faces
+      std::reverse(p->tets.begin(), p->tets.end());
+      std::reverse(p->faces.begin(), p->faces.begin());
+    }
+    else if ((vFirst * normal < 0)  && (vLast * normal > 0)) {
+    }
+    else if ((vFirst * normal < 0)  && (vLast * normal < 0)) {
+      cout << "failed clockwise. ABORT" << endl;
+    }
+    else if ((vFirst * normal > 0)  && (vLast * normal > 0)) {
+      cout << "failed clockwise. ABORT" << endl;
+    }
+  }
+}
 static void runErm(EdgePatch* p)
 {
   // DEBUG
@@ -843,6 +979,7 @@ static void runErm(EdgePatch* p)
   } // REMOVE DEBUG
 
   getOrderedTetsandFaces(p->mesh, p->entity, p->tets, p->faces);
+  //getClockwiseTetsandFaces(p);
   assembleEdgePatchLHS(p);
   assembleEdgePatchRHS(p);
   mth::solveFromQR(p->qr.Q, p->qr.R, p->b, p->x);

From 373aa474d32b8776cde825a2fc3d2c08fc00e623 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Mon, 1 Jun 2020 13:45:37 -0400
Subject: [PATCH 178/555] Clean up and adds TODOs for future (emResiduals)

---
 em/emResidualFunctionals.cc | 477 ++++++++++++------------------------
 1 file changed, 157 insertions(+), 320 deletions(-)

diff --git a/em/emResidualFunctionals.cc b/em/emResidualFunctionals.cc
index eba640b57..d85a72881 100644
--- a/em/emResidualFunctionals.cc
+++ b/em/emResidualFunctionals.cc
@@ -26,9 +26,11 @@ struct Equilibration {
   int dim;
   /* polynomial order of Nedelec space */
   int order;
-  /* input scalar field containing Nedelec dofs for electric field */
+  /* input scalar field containing Nedelec dofs for solution electric field */
   apf::Field* ef;
-  /* output field containing correction values */
+  /* output field containing correction values.
+   * currently 3 scalar values are stored on each face
+   * in order corresponding to downward edges of the face */
   apf::Field* g;
 };
 
@@ -42,11 +44,11 @@ static void setupEquilibration(Equilibration* eq, apf::Field* f, apf::Field* g)
 
   double zeros[3] = {0., 0., 0.};
   apf::MeshEntity* face;
-  apf::MeshIterator* it = apf::getMesh(f)->begin(2);
-  while ((face = apf::getMesh(f)->iterate(it))) {
+  apf::MeshIterator* it = eq->mesh->begin(2);
+  while ((face = eq->mesh->iterate(it))) {
     apf::setComponents(eq->g, face, 0, zeros);
   }
-  apf::getMesh(f)->end(it);
+  eq->mesh->end(it);
 }
 
 struct QRDecomp {
@@ -59,7 +61,7 @@ typedef std::vector<apf::MeshEntity*> EntityVector;
 struct EdgePatch {
   apf::Mesh* mesh;
   Equilibration* equilibration;
-  /* the entity around which the patch
+  /* the edge entity around which the patch
      is centered. a patch collects entities
      (faces & tets) around this edge entity */
   apf::MeshEntity* entity;
@@ -74,113 +76,101 @@ struct EdgePatch {
   QRDecomp qr;
 };
 
-static void setupEdgePatch(EdgePatch* p, Equilibration* eq)
+static void setupEdgePatch(EdgePatch* ep, Equilibration* eq)
 {
-  p->mesh = eq->mesh;
-  p->equilibration = eq;
-  p->entity = 0;
+  ep->mesh = eq->mesh;
+  ep->equilibration = eq;
+  ep->entity = 0;
 }
 
-static void startEdgePatch(EdgePatch* p, apf::MeshEntity* e)
+static void startEdgePatch(EdgePatch* ep, apf::MeshEntity* e)
 {
-  p->tets.clear();
-  p->faces.clear();
-  p->entity = e;
-  p->isOnBdry = crv::isBoundaryEntity(p->mesh, p->entity);
+  ep->tets.clear();
+  ep->faces.clear();
+  ep->entity = e;
+  ep->isOnBdry = crv::isBoundaryEntity(ep->mesh, ep->entity);
 }
 
-static void addEntityToPatch(EdgePatch* p, apf::MeshEntity* e)
+static void addEntityToPatch(EdgePatch* ep, apf::MeshEntity* e)
 {
-  if(p->mesh->getType(e) == apf::Mesh::TRIANGLE)
-    p->faces.push_back(e);
-  if(p->mesh->getType(e) == apf::Mesh::TET)
-    p->tets.push_back(e);
+  if(ep->mesh->getType(e) == apf::Mesh::TRIANGLE)
+    ep->faces.push_back(e);
+  if(ep->mesh->getType(e) == apf::Mesh::TET)
+    ep->tets.push_back(e);
 }
 
-static void addEntitiesToPatch(EdgePatch* p, apf::DynamicArray<apf::MeshEntity*>& es)
+static void addEntitiesToPatch(
+    EdgePatch* ep, apf::DynamicArray<apf::MeshEntity*>& es)
 {
   for (std::size_t i=0; i < es.getSize(); ++i)
-    addEntityToPatch(p, es[i]);
+    addEntityToPatch(ep, es[i]);
 }
 
-static bool getInitialEdgePatch(EdgePatch* p, apf::CavityOp* o)
+static bool getInitialEdgePatch(EdgePatch* ep, apf::CavityOp* o)
 {
-  if ( ! o->requestLocality(&p->entity,1))
+  if ( ! o->requestLocality(&ep->entity,1))
     return false;
   apf::DynamicArray<apf::MeshEntity*> adjacent;
-  p->mesh->getAdjacent(p->entity, 3, adjacent);
-  addEntitiesToPatch(p, adjacent);
-  // REMOVE ALL
-  int ne = adjacent.size();
-  std::cout << " Unordered Tets, ne: " << ne << std::endl; // REMOVE
-  std::cout << " Center of Unordered tets" << std::endl;
-  for (int i = 0; i < ne; i++) {
-    apf::MeshEntity* tet = adjacent[i];;
-    apf::Vector3 center = apf::getLinearCentroid(p->mesh, tet);
-    std::cout << i << ":  " << center << std::endl;
-  } //////////
-
-
-
-  p->mesh->getAdjacent(p->entity, 2, adjacent);
-  addEntitiesToPatch(p, adjacent);
+  ep->mesh->getAdjacent(ep->entity, 3, adjacent);
+  addEntitiesToPatch(ep, adjacent);
 
-  // REMOVE ALL
-  int nf = adjacent.size();
-  std::cout << " Unordered Faces, nf: " << nf << std::endl; // REMOVE
-  std::cout << " Center of Unordered Faces" << std::endl;
-  for (int i = 0; i < nf; i++) {
-    apf::MeshEntity* face = adjacent[i];
-    apf::Vector3 center = apf::getLinearCentroid(p->mesh, face);
-    std::cout << i << ":  " << center << std::endl;
-  }
-  std::cout << "----------------" << std::endl;
+  ep->mesh->getAdjacent(ep->entity, 2, adjacent);
+  addEntitiesToPatch(ep, adjacent);
 
   return true;
 }
 
-static bool buildEdgePatch(EdgePatch* p, apf::CavityOp* o)
+static bool buildEdgePatch(EdgePatch* ep, apf::CavityOp* o)
 {
-  if (!getInitialEdgePatch(p, o)) return false;
+  if (!getInitialEdgePatch(ep, o)) return false;
   return true;
 }
 
-static void assembleEdgePatchLHS(EdgePatch* p)
+/*
+ * Reference: Leszek Demkowicz, Computing with hp-adaptive
+ * finite elements, vol2, equation (11.118, 11.119, 11.122).
+ */
+static void assembleEdgePatchLHS(EdgePatch* ep)
 {
-  int ne = p->tets.size();
-  int nf = p->faces.size();
-  if( crv::isBoundaryEntity(p->mesh, p->entity) ) {
-    p->T.resize(ne+nf, ne+nf);
-    p->T.zero();
+  int ne = ep->tets.size();
+  int nf = ep->faces.size();
+  if( crv::isBoundaryEntity(ep->mesh, ep->entity) ) {
+    ep->T.resize(ne+nf, ne+nf);
+    ep->T.zero();
     for (int i = 0; i < nf; i++)
-      p->T(i,i) = 2.;
+      ep->T(i,i) = 2.;
     for (int i = 0; i < ne-1; i++) {
-      p->T(i+nf,i) = 1.; p->T(i+nf,i+1) = -1.;
-      p->T(i,i+nf) = 1.; p->T(i+1,i+nf) = -1.;
+      ep->T(i+nf,i) = 1.; ep->T(i+nf,i+1) = -1.;
+      ep->T(i,i+nf) = 1.; ep->T(i+1,i+nf) = -1.;
     }
-    p->T(ne+nf-1, ne-1) = 1.; p->T(ne+nf-1, ne) = 1.;
-    p->T(ne-1, ne+nf-1) = 1.; p->T(ne, ne+nf-1) = 1.;
+    ep->T(ne+nf-1, ne-1) = 1.; ep->T(ne+nf-1, ne) = 1.;
+    ep->T(ne-1, ne+nf-1) = 1.; ep->T(ne, ne+nf-1) = 1.;
   }
-  else if( ! crv::isBoundaryEntity(p->mesh, p->entity) ) {
-    p->A.resize(ne, nf);
-    p->A.zero();
+  else if( ! crv::isBoundaryEntity(ep->mesh, ep->entity) ) {
+    ep->A.resize(ne, nf);
+    ep->A.zero();
     for (int i = 0; i < ne-1; i++) {
-      p->A(i,i) = 1.;  p->A(i,i+1) = -1.;
+      ep->A(i,i) = 1.;  ep->A(i,i+1) = -1.;
     }
-    p->A(ne-1,0) = -1.; p->A(ne-1,ne-1) = 1.;
+    ep->A(ne-1,0) = -1.; ep->A(ne-1,ne-1) = 1.;
     // A is singular so do (A*At) + 1.0
     // to pick a particular solution
-    p->At.resize(nf,ne);
-    mth::transpose(p->A, p->At);
-    mth::multiply(p->A, p->At, p->T);
+    ep->At.resize(nf,ne);
+    mth::transpose(ep->A, ep->At);
+    mth::multiply(ep->A, ep->At, ep->T);
 
     for (int i = 0; i < ne; i++)
       for (int j = 0; j < ne; j++)
-        p->T(i,j) += 1.;
+        ep->T(i,j) += 1.;
   }
-  mth::decomposeQR(p->T, p->qr.Q, p->qr.R);
+  mth::decomposeQR(ep->T, ep->qr.Q, ep->qr.R);
 }
 
+/*
+ * Performs Curl Curl integration using curl vector Nedelec shapes
+ * TODO Add 2D curl curl integration
+ * TODO CLEANUP.
+ */
 void assembleCurlCurlElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
     apf::Field* f, mth::Matrix<double>& elmat)
 {
@@ -214,10 +204,10 @@ void assembleCurlCurlElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
     if (dim == 3) { // TODO this is Piola transformation. Put it in apfNedelec
       el->getShape()->getLocalVectorCurls(mesh, e, p, curlshape);
       phys_curlshape.zero();
-      for (int i = 0; i < nd; i++)
-        for (int j = 0; j < dim; j++)
-          for (int k = 0; k < dim; k++)
-            phys_curlshape(i,j) += curlshape[i][k] * J[k][j];
+      for (int j = 0; j < nd; j++)
+        for (int k = 0; k < dim; k++)
+          for (int l = 0; l < dim; l++)
+            phys_curlshape(j,k) += curlshape[j][l] * J[l][k];
     }
     else {
       /*
@@ -241,6 +231,9 @@ void assembleCurlCurlElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
   apf::destroyMeshElement(me);
 }
 
+/*
+ * Performs Vector Vector Mass integration using vector Nedelec shapes
+ */
 void assembleVectorMassElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
     apf::Field* f, mth::Matrix<double>& elmat)
 {
@@ -257,7 +250,7 @@ void assembleVectorMassElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
   apf::MeshElement* me = apf::createMeshElement(mesh, e);
   apf::Element* el = apf::createElement(f, me);
   int int_order = 2 * fs->getOrder();
-  int np = apf::countIntPoints(me, int_order); // int points required
+  int np = apf::countIntPoints(me, int_order);
 
   elmat.zero();
   apf::Vector3 p;
@@ -270,7 +263,7 @@ void assembleVectorMassElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
     w = weight * jdet;
 
     apf::getVectorShapeValues(el, p, vectorshapes);
-    mth::Matrix<double> vectorShapes (nd, dim); // TODO opt clean
+    mth::Matrix<double> vectorShapes (nd, dim);
     for (int j = 0; j < nd; j++)
       for (int k = 0; k < dim; k++)
         vectorShapes(j,k) = vectorshapes[j][k];
@@ -301,14 +294,16 @@ void assembleElementMatrix(apf::Mesh* mesh, apf::MeshEntity*e,
   elmat += mass_elmat;
 }
 
-// computes local bilinear form integral restricted to
-// an edge of a tet element
+/*
+ * computes local bilinear form integral restricted to
+ * an edge of a tet element
+ */
 static double getLocalEdgeBLF(EdgePatch* ep, apf::MeshEntity* tet)
 {
+  PCU_ALWAYS_ASSERT(ep->mesh->getType(tet) == apf::Mesh::TET);
   // findIn edge in downward edges of tet
   apf::Downward e;
   int ne = ep->mesh->getDownward(tet, 1, e);
-  PCU_ALWAYS_ASSERT(ne == 6);
   int ei = apf::findIn(e, ne, ep->entity);
   // get Element Dofs
   apf::MeshElement* me = apf::createMeshElement(ep->mesh, tet);
@@ -318,9 +313,8 @@ static double getLocalEdgeBLF(EdgePatch* ep, apf::MeshEntity* tet)
   apf::NewArray<double> d (nd);
   el->getElementDofs(d);
   mth::Vector<double> dofs (nd);
-  for (int i = 0; i < nd; i++) // TODO cleanup
+  for (int i = 0; i < nd; i++)
     dofs(i) = d[i];
-  cout << "Element Dofs "<< dofs << endl;
   // assemble curl curl element matrix
   mth::Matrix<double> curl_elmat;
   assembleCurlCurlElementMatrix(ep->mesh, tet,
@@ -347,107 +341,13 @@ static double getLocalEdgeBLF(EdgePatch* ep, apf::MeshEntity* tet)
   apf::getAlignment(ep->mesh, tet, ep->entity, which, flip, rotate);
   if (flip)
     blf_integrals(ei) = -1*blf_integrals(ei);
-
-  cout << "local blf " << blf_integrals(ei);
-  if (flip) { cout << ". local blf before negation " << -1*blf_integrals(ei);}
-  cout << endl;
   return blf_integrals(ei);
-
-  /* 0. findIn edge in downward edges of tet
-  apf::Downward e;
-  int ne = ep->mesh->getDownward(tet, 1, e);
-  PCU_ALWAYS_ASSERT(ne == 6);
-  int ei = apf::findIn(e, ne, ep->entity);
-
-
-  apf::FieldShape* fs = ep->equilibration->ef->getShape();
-  int type = ep->mesh->getType(tet);
-  int dim = apf::getDimension(ep->mesh, tet);
-
-
-  apf::MeshElement* me = apf::createMeshElement(ep->mesh, tet);
-  apf::Element* el = apf::createElement(ep->equilibration->ef, me);
-  int nd = apf::countElementNodes(el->getFieldShape(), type);
-  int int_order = 2 * fs->getOrder();
-  int np = apf::countIntPoints(me, int_order);
-  double w;
-
-
-  apf::NewArray<apf::Vector3> curlshapes(nd);
-  apf::NewArray<apf::Vector3> vectorshapes(nd);
-  mth::Matrix<double> phys_curlshapes(nd, dim);  // TODO clean
-
-
-  // 1. Curl Curl Integration
-  double curlcurl = 0.0;
-
-  apf::Vector3 p, curl;
-  for (int i = 0; i < np; i++) {
-    apf::getIntPoint(me, int_order, i, p);
-    double weight = apf::getIntWeight(me, int_order, i);
-    apf::Matrix3x3 J;
-    apf::getJacobian(me, p, J);
-    //double jdet = apf::getJacobianDeterminant(J, dim);
-    w = weight; // TODO check why do not need division by jdet
-
-    // get curl vector
-    apf::getCurl(el, p, curl);
-
-    // get curlshape values // TODO CLEAN use getCurlShapeValues
-    el->getShape()->getLocalVectorCurls(ep->mesh, tet, p, curlshapes);
-    phys_curlshapes.zero();
-    for (int i = 0; i < nd; i++)
-      for (int j = 0; j < dim; j++)
-        for (int k = 0; k < dim; k++)
-          phys_curlshapes(i,j) += curlshapes[i][k] * J[k][j];
-
-    // pick the ei curlshape // TODO Clean apf::Vector
-    apf::Vector3 cshape;
-    cshape[0] = phys_curlshapes(ei, 0);
-    cshape[1] = phys_curlshapes(ei, 1);
-    cshape[2] = phys_curlshapes(ei, 2);
-
-    // multiply
-    curlcurl += (curl * cshape) * w;
-  }
-
-  // 2. Vector Mass Integration
-  int_order = 2 * fs->getOrder();
-  np = apf::countIntPoints(me, int_order); // int points required
-
-  double vectormass = 0.0;
-  apf::Vector3 vvalue;
-  for (int i = 0; i < np; i++) {
-    apf::getIntPoint(me, int_order, i, p);
-    double weight = apf::getIntWeight(me, int_order, i);
-    apf::Matrix3x3 J;
-    apf::getJacobian(me, p, J);
-    double jdet = apf::getJacobianDeterminant(J, dim);
-
-    w = weight * jdet;
-
-    apf::getVector(el, p, vvalue);
-
-    apf::getVectorShapeValues(el, p, vectorshapes);
-    apf::Vector3 vshape = vectorshapes[ei];
-
-    vectormass += (vvalue * vshape) * w;
-  }
-
-  double blf_integral = curlcurl + vectormass;
-
-  int which, rotate; bool flip;
-  apf::getAlignment(ep->mesh, tet, ep->entity, which, flip, rotate);
-  if (flip)
-    blf_integral = -1*blf_integral;
-  cout << "local blf " << blf_integral << endl;
-  return blf_integral;*/
 }
 
 
 // TODO QUESTION redo this to allow user access from outside
-void pumiUserFunction(const apf::Vector3& x, mth::Vector<double>& f,
-    apf::MeshEntity* e, apf::Mesh* mesh)
+void pumiUserFunction(apf::Mesh* mesh, apf::MeshEntity* e,
+    const apf::Vector3& x, mth::Vector<double>& f)
 {
   double freq = 1.;
   double kappa = freq * M_PI;
@@ -496,45 +396,49 @@ void assembleDomainLFElementVector(apf::Mesh* mesh, apf::MeshEntity* e,
     apf::getVectorShapeValues(el, p, vectorshapes);
     apf::Vector3 global;
     apf::mapLocalToGlobal(me, p, global);
-    pumiUserFunction(global, val, e, apf::getMesh(f)); // eval vector function
+    pumiUserFunction(mesh, e, global, val);
     val *= w;
 
     mth::Matrix<double> vectorShapes (nd, dim);
-    for (int i = 0; i < nd; i++)
-      for (int j = 0; j < dim; j++)
-        vectorShapes(i,j) = vectorshapes[i][j];
+    for (int j = 0; j < nd; j++)
+      for (int k = 0; k < dim; k++)
+        vectorShapes(j,k) = vectorshapes[j][k];
     mth::Vector<double> V (nd);
     V.zero();
     mth::multiply(vectorShapes, val, V);
     elvect += V;
   }
+
   apf::destroyElement(el);
   apf::destroyMeshElement(me);
 }
 
-// computes local linear form integral restricted to
-// an edge of a tet element
-static double getLocalEdgeLF(EdgePatch* p, apf::MeshEntity* tet)
+/*
+ * computes local linear form integral restricted to
+ * an edge of a tet element
+ */
+static double getLocalEdgeLF(EdgePatch* ep, apf::MeshEntity* tet)
 {
+  PCU_ALWAYS_ASSERT(ep->mesh->getType(tet) == apf::Mesh::TET);
   // findIn edge in downward edges of tet
   apf::Downward e;
-  int ne = p->mesh->getDownward(tet, 1, e);
-  PCU_ALWAYS_ASSERT(ne == 6);
-  int ei = apf::findIn(e, ne, p->entity);
+  int ne = ep->mesh->getDownward(tet, 1, e);
+  int ei = apf::findIn(e, ne, ep->entity);
   // assemble Domain LF Vector
   mth::Vector<double> elvect;
-  assembleDomainLFElementVector(p->mesh, tet,
-      p->equilibration->ef, elvect);
+  assembleDomainLFElementVector(ep->mesh, tet,
+      ep->equilibration->ef, elvect);
   int which, rotate; bool flip;
-  apf::getAlignment(p->mesh, tet, p->entity, which, flip, rotate);
+  apf::getAlignment(ep->mesh, tet, ep->entity, which, flip, rotate);
   if (flip)
     elvect(ei) = -1*elvect(ei);
-  cout << "local lf " << elvect(ei) << endl;
   return elvect(ei);
 }
 
-// Given a tet and one of its faces, the vertex of the tet
-// opposite to the face is returned.
+/*
+ * Given a tet and one of its faces, the vertex of the tet
+ * opposite to the given face is returned.
+ */
 static apf::MeshEntity* getTetOppVert(
     apf::Mesh* m, apf::MeshEntity* t, apf::MeshEntity* f)
 {
@@ -550,6 +454,10 @@ static apf::MeshEntity* getTetOppVert(
   return 0;
 }
 
+/*
+ * Given a face and one of its edges, the vertex of the face
+ * opposite to the given edge is returned.
+ */
 static apf::MeshEntity* getFaceOppVert(
     apf::Mesh* m, apf::MeshEntity* f, apf::MeshEntity* e)
 {
@@ -564,10 +472,12 @@ static apf::MeshEntity* getFaceOppVert(
   }
   return 0;
 }
-static apf::Vector3 computeFaceNormal(apf::Mesh* m,
-    apf::MeshEntity* f, apf::Vector3 const& p)
+
+static apf::Vector3 computeFaceNormal(
+    apf::Mesh* m, apf::MeshEntity* f, apf::Vector3 const& p)
 {
-  // Compute face normal using face Jacobian
+  // Compute face normal using face Jacobian,
+  // so it can also be used for curved mesh elements
   apf::MeshElement* me = apf::createMeshElement(m, f);
   apf::Matrix3x3 J;
   apf::getJacobian(me, p, J);
@@ -584,37 +494,37 @@ apf::Vector3 computeFaceOutwardNormal(apf::Mesh* m,
 {
   apf::Vector3 n = computeFaceNormal(m, f, p);
 
-  // Orient the normal outwards from the tet
+  // orient the normal outwards from the tet
   apf::MeshEntity* oppVert = getTetOppVert(m, t, f);
   apf::Vector3 vxi = apf::Vector3(0.,0.,0.);
 
-  // get global coordinates of the vertex
   apf::Vector3 txi;
   m->getPoint(oppVert, 0, txi);
 
-  // get global coordinates of the point on the face
   apf::MeshElement* fme = apf::createMeshElement(m, f);
   apf::Vector3 pxi;
   apf::mapLocalToGlobal(fme, p, pxi);
   apf::destroyMeshElement(fme);
 
   apf::Vector3 pxiTotxi = txi - pxi;
-  //std::cout << "dot product " << pxiTotxi*n << std::endl;
   if (pxiTotxi*n > 0) {
     n = n*-1.;
   }
   return n;
 }
 
-// computes local flux integral restricted to
-// an edge of a tet element
+/*
+ * computes local flux integral restricted to
+ * an edge of a tet element
+ */
 static double getLocalFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
 {
+  PCU_ALWAYS_ASSERT(ep->mesh->getType(tet) == apf::Mesh::TET);
+
   double fluxIntegral = 0.0;
   // 1. get faces of the tet in the patch
   apf::Downward f;
   int nf = ep->mesh->getDownward(tet, 2, f);
-  PCU_ALWAYS_ASSERT(nf == 4);
   std::vector<apf::MeshEntity*> patchFaces;
   for (int i = 0; i < nf; i++) {
     if(std::find(ep->faces.begin(), ep->faces.end(), f[i]) != ep->faces.end())
@@ -651,11 +561,10 @@ static double getLocalFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
     apf::MeshElement* fme = apf::createMeshElement(ep->mesh, currentFace);
     apf::Element* fel = apf::createElement(ep->equilibration->ef, fme);
     int np = apf::countIntPoints(fme, int_order);
-    std::cout << "np " << np << std::endl; // REMOVE
 
     // loop over integration points
     apf::Vector3 p, tet1xi, tet2xi, curl1, curl2, curl,
-      fnormal1, fnormal2, tk, vshape;
+      fn1, fn2, tk, vshape;
     for (int n = 0; n < np; n++) {
       apf::getIntPoint(fme, int_order, n, p);
       double weight = apf::getIntWeight(fme, int_order, n);
@@ -666,16 +575,14 @@ static double getLocalFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
 
       // compute face outward normals wrt tets
       if (tet == firstTet)
-        fnormal1 = computeFaceOutwardNormal(ep->mesh, firstTet, currentFace, p);
+        fn1 = computeFaceOutwardNormal(ep->mesh, firstTet, currentFace, p);
       else
-        fnormal1 = computeFaceOutwardNormal(ep->mesh, secondTet, currentFace, p);
+        fn1 = computeFaceOutwardNormal(ep->mesh, secondTet, currentFace, p);
       if (up.n == 2) {
         if (tet == firstTet)
-          fnormal2 = computeFaceOutwardNormal(ep->mesh, secondTet, currentFace, p);
+          fn2 = computeFaceOutwardNormal(ep->mesh, secondTet, currentFace, p);
         else
-          fnormal2 = computeFaceOutwardNormal(ep->mesh, firstTet, currentFace, p);
-        std::cout << "normal1 " << fnormal1 << std::endl; // REMOVE
-        std::cout << "normal2 " << fnormal2 << std::endl; // REMOVE
+          fn2 = computeFaceOutwardNormal(ep->mesh, firstTet, currentFace, p);
       }
 
       curl.zero();
@@ -684,7 +591,7 @@ static double getLocalFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
       apf::MeshElement* me1 = apf::createMeshElement(ep->mesh, firstTet);
       apf::Element* el1 = apf::createElement(ep->equilibration->ef, me1);
       apf::getCurl(el1, tet1xi, curl1);
-      apf::Vector3 temp1 = apf::cross(fnormal1, curl1);
+      apf::Vector3 temp1 = apf::cross(fn1, curl1);
       curl += temp1;
       apf::destroyElement(el1);
       apf::destroyMeshElement(me1);
@@ -695,16 +602,15 @@ static double getLocalFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
         apf::MeshElement* me2 = apf::createMeshElement(ep->mesh, secondTet);
         apf::Element* el2 = apf::createElement(ep->equilibration->ef, me2);
         apf::getCurl(el2, tet2xi, curl2);
-        apf::Vector3 temp2 = apf::cross(fnormal2, curl2);
+        apf::Vector3 temp2 = apf::cross(fn2, curl2);
         curl += (temp2 * -1.);
-        curl = curl * 1./2.; //
+        curl = curl * 1./2.;
         apf::destroyElement(el2);
         apf::destroyMeshElement(me2);
       }
 
       // compute tk (inter-element averaged flux)
       tk = curl;
-      std::cout << "tk " << tk << std::endl; // REMOVE
 
       // compute vector shape
       int type = apf::Mesh::TRIANGLE;
@@ -713,6 +619,7 @@ static double getLocalFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
       apf::getVectorShapeValues(fel, p, vectorshapes);
       vshape = vectorshapes[ei];
 
+      // TODO why?
       int which, rotate; bool flip;
       apf::getAlignment(
           ep->mesh, currentFace, ep->entity, which, flip, rotate);
@@ -720,14 +627,13 @@ static double getLocalFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
       vshape = vshape * -1.;
       }
 
-
       // compute integral
       fluxFaceIntegral += (tk * vshape) * weight * jdet;
     }
     apf::destroyElement(fel);
     apf::destroyMeshElement(fme);
+
     fluxIntegral += fluxFaceIntegral;
-    std::cout << "flux Face integral " << fluxFaceIntegral << std::endl; // REMOVE
   }
   return fluxIntegral;
 }
@@ -742,8 +648,6 @@ static void assembleEdgePatchRHS(EdgePatch* p)
     p->b.resize(p->tets.size());
     p->b.zero();
   }
-  double testblflf = 0.0; // REMOVE
-  double testflux = 0.0; // REMOVE
   int ne = p->tets.size();
   int nf = p->faces.size();
   for (int i = 0; i < ne; i++) {
@@ -751,27 +655,13 @@ static void assembleEdgePatchRHS(EdgePatch* p)
     double blfIntegral = getLocalEdgeBLF(p, tet);
     double lfIntegral = getLocalEdgeLF(p, tet);
     double fluxIntegral = getLocalFluxIntegral(p, tet);
-    testblflf += (blfIntegral - lfIntegral);
-    testflux += fluxIntegral;
-    cout << "Blf Integral" << blfIntegral << endl;
-    cout << "Lf Integral" << lfIntegral << endl;
     if(p->isOnBdry)
       p->b(nf+i) = blfIntegral - lfIntegral - fluxIntegral;
     else
       p->b(i) = blfIntegral - lfIntegral - fluxIntegral;
-
-    //DEBUG
-    apf::Downward tetedges;
-    int ntedges = p->mesh->getDownward(tet, 1, tetedges);
-    int ei = apf::findIn(tetedges, ntedges, p->entity);
-    cout << "ei in tet " << ei << endl;
   }
-  std::cout << "Blf - lf integrals = " << testblflf << std::endl;
-  std::cout << "Flux Integral Sum  = " << testflux << std::endl;
 }
 
-// The following two functions help order tets and faces in a cavity in a
-// clockwise (or ccw) direction.
 static apf::MeshEntity* getTetOppFaceSharingEdge(
     apf::Mesh* m, apf::MeshEntity* t, apf::MeshEntity* f, apf::MeshEntity* e)
 {
@@ -786,12 +676,17 @@ static apf::MeshEntity* getTetOppFaceSharingEdge(
   }
   return 0;
 }
+
+/*
+ * Orders tets and faces in an edge cavity in a
+ * clockwise (or ccw) direction around the edge.
+ */
 static void getOrderedTetsandFaces(apf::Mesh* mesh, apf::MeshEntity* edge,
      EntityVector& tets, EntityVector& faces)
 {
   tets.clear();
   faces.clear();
-  if( ! crv::isBoundaryEntity(mesh, edge) ) { // interior edge patches
+  if( ! crv::isBoundaryEntity(mesh, edge) ) {
     apf::MeshEntity* currentFace = mesh->getUpward(edge, 0);
     apf::Up up;
     mesh->getUp(currentFace, up);
@@ -817,7 +712,7 @@ static void getOrderedTetsandFaces(apf::Mesh* mesh, apf::MeshEntity* edge,
         nextTet = up.e[1];
     }
   }
-  else { // boundary edge patches
+  else {
     apf::Up up;
     mesh->getUp(edge, up);
     apf::MeshEntity* firstFace;
@@ -965,91 +860,37 @@ void getClockwiseTetsandFaces(EdgePatch* p)
     }
   }
 }
-static void runErm(EdgePatch* p)
+
+static void runErm(EdgePatch* ep)
 {
-  // DEBUG
-  cout << "PUMI edge vertex coordinates" << endl;
-  apf::Downward ev;
-  int nev = p->mesh->getDownward(p->entity, 0, ev);
-  PCU_ALWAYS_ASSERT(nev == 2);
-  for (int i = 0; i < nev; i++) {
-    apf::Vector3 pt;
-    p->mesh->getPoint(ev[i], 0, pt);
-    cout << pt << endl;
-  } // REMOVE DEBUG
-
-  getOrderedTetsandFaces(p->mesh, p->entity, p->tets, p->faces);
-  //getClockwiseTetsandFaces(p);
-  assembleEdgePatchLHS(p);
-  assembleEdgePatchRHS(p);
-  mth::solveFromQR(p->qr.Q, p->qr.R, p->b, p->x);
-
-  if (!p->isOnBdry) { // solve At*mu = g for g
-    mth::Vector<double> temp(p->tets.size());
-    mth::multiply(p->At, p->x, temp);
-    for (size_t i = 0; i < p->tets.size(); i++) {
-      p->x(i) = temp(i);
+  getOrderedTetsandFaces(ep->mesh, ep->entity, ep->tets, ep->faces);
+  //getClockwiseTetsandFaces(ep);
+  assembleEdgePatchLHS(ep);
+  assembleEdgePatchRHS(ep);
+  mth::solveFromQR(ep->qr.Q, ep->qr.R, ep->b, ep->x);
+
+  if (!ep->isOnBdry) { // solve At*mu = g for g
+    mth::Vector<double> temp(ep->tets.size());
+    mth::multiply(ep->At, ep->x, temp);
+    for (size_t i = 0; i < ep->tets.size(); i++) {
+      ep->x(i) = temp(i);
     }
   }
 
-  bool debug = true; // REMOVE DEBUG
-  if (debug) {
-    if (p->isOnBdry)
-      cout << "Boundary Patch" << endl;
-    else
-      cout << "Interior Patch" << endl;
-    cout << "ne " << p->tets.size() << " nf " << p->faces.size() << endl;
-    cout << "LHS Matrix" << endl;
-    cout << p->T << endl;
-
-    cout << "RHS Vector" << endl;
-    cout << p->b << endl;
-    double rhs_sum = 0.0;
-    for(unsigned int i = 0; i < p->b.size(); i++)
-      rhs_sum += p->b(i);
-    cout << "rhs_sum " << rhs_sum << endl;
-    if ( (abs(rhs_sum) > 1e-8) && (!p->isOnBdry) )
-      cout << "nonzero! error" << endl;
-  }
-
-  int nf = p->faces.size();
+  int nf = ep->faces.size();
   for(int i = 0; i < nf; i++) {
-    // get face entitiy
-    apf::MeshEntity* face = p->faces[i];
-    // get ei of edge in face downward edges
+    apf::MeshEntity* face = ep->faces[i];
+
     apf::Downward e;
-    int ned = p->mesh->getDownward(face, 1, e);
-    int ei = apf::findIn(e, ned, p->entity);
+    int ned = ep->mesh->getDownward(face, 1, e);
+    int ei = apf::findIn(e, ned, ep->entity);
     PCU_ALWAYS_ASSERT(ned == 3 && ei != -1);
-    // save the g value at that index on that face in the g field
-    double components[3];
-    apf::getComponents(p->equilibration->g, face, 0, components);
-    components[ei] = p->x(i);
-    apf::setComponents(p->equilibration->g, face, 0, components);
-  }
 
-  // REMOVE ALL BELOW
-  int ne = p->tets.size();
-  //int nf = p->faces.size();
-  std::cout << " Reordered Tets, ne: " << ne << " nf " << nf << std::endl; // REMOVE
-  std::cout << " Center of Reordered tets" << std::endl;
-  for (int i = 0; i < ne; i++) {
-    apf::MeshEntity* tet = p->tets[i];
-    apf::Vector3 center = apf::getLinearCentroid(p->mesh, tet);
-    std::cout << i << ":  " << center << std::endl;
-  }
-  std::cout << " Center of Reordered Faces" << std::endl;
-  for (int i = 0; i < nf; i++) {
-    apf::MeshEntity* face = p->faces[i];
-    apf::Vector3 center = apf::getLinearCentroid(p->mesh, face);
-    std::cout << i << ":  " << center << std::endl;
+    double components[3];
+    apf::getComponents(ep->equilibration->g, face, 0, components);
+    components[ei] = ep->x(i);
+    apf::setComponents(ep->equilibration->g, face, 0, components);
   }
-  std::cout << "----------------" << std::endl;
-
-  std::cout << "x" << std::endl; // REMOVE
-  std::cout << p->x << std::endl; // REMOVE
-
-  cout << "======================================================" << endl;
 }
 
 
@@ -1078,7 +919,8 @@ class EdgePatchOp : public apf::CavityOp
 
 apf::Field* equilibrateResiduals(apf::Field* f)
 {
-  apf::Field* g = createPackedField( apf::getMesh(f), "g", 3, apf::getConstant(2) );
+  apf::Field* g = createPackedField(
+      apf::getMesh(f), "g", 3, apf::getConstant(2) );
   Equilibration equilibration;
   setupEquilibration(&equilibration, f, g);
   EdgePatchOp op(&equilibration);
@@ -1087,9 +929,4 @@ apf::Field* equilibrateResiduals(apf::Field* f)
 }
 
 
-
-
-
-
-
 }

From e59daab70eda9587f1ebdf9318691fa6b7696e70 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Mon, 1 Jun 2020 15:55:23 -0400
Subject: [PATCH 179/555] Clean ups. Removes print statements for debugging

---
 em/emFluxCorrection.cc      | 139 ++++--------------------------------
 em/emResidualFunctionals.cc |   8 ++-
 2 files changed, 20 insertions(+), 127 deletions(-)

diff --git a/em/emFluxCorrection.cc b/em/emFluxCorrection.cc
index 1f359f4ed..ad5fa58a3 100644
--- a/em/emFluxCorrection.cc
+++ b/em/emFluxCorrection.cc
@@ -20,165 +20,56 @@ struct QRDecomp {
   mth::Matrix<double> R;
 };
 
-static void assembleFaceMassMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
-    apf::Field* f, mth::Matrix<double>& elmat)
-{
-  apf::FieldShape* fs = f->getShape();
-  int type = mesh->getType(e);
-  int nd = apf::countElementNodes(fs, type);
-  int dim = apf::getDimension(mesh, e);
-  int sdim = mesh->getDimension();
-  PCU_ALWAYS_ASSERT(type == apf::Mesh::TRIANGLE && dim == 2);
-  double w;
-
-  apf::NewArray<apf::Vector3> vectorshapes(nd);
-  elmat.resize(nd,nd);
-
-  apf::MeshElement* me = apf::createMeshElement(mesh, e);
-  apf::Element* el = apf::createElement(f, me);
-  int int_order = 2 * fs->getOrder();
-  int np = apf::countIntPoints(me, int_order);
-
-  elmat.zero();
-  apf::Vector3 p;
-  for (int i = 0; i < np; i++) {
-    apf::getIntPoint(me, int_order, i, p);
-    double weight = apf::getIntWeight(me, int_order, i);
-    apf::Matrix3x3 J;
-    apf::getJacobian(me, p, J);
-    double jdet = apf::getJacobianDeterminant(J, dim);
-    w = weight * jdet;
-
-    apf::getVectorShapeValues(el, p, vectorshapes);
-    mth::Matrix<double> vectorShapes (nd, sdim);
-    for (int j = 0; j < nd; j++)
-      for (int k = 0; k < sdim; k++)
-        vectorShapes(j,k) = vectorshapes[j][k];
-    cout << "Face Tri Shapes" << endl;
-    cout << vectorShapes << endl;
-
-    /*apf::Downward edges;
-    int nedges = mesh->getDownward(e, 1, edges);
-    int which, rotate; bool flip; // negative ND dofs
-    for (int i = 0; i < nedges; i++) {
-      apf::getAlignment(mesh, e, edges[i], which, flip, rotate);
-        if (flip) {
-          for (int j = 0; j < sdim; j++) {
-            vectorShapes(i, j) = -1. * vectorShapes(i,j);
-          }
-        }
-    }*/
-
-    mth::Matrix<double> vectorShapesT (sdim, nd);
-    mth::transpose(vectorShapes, vectorShapesT);
-    mth::Matrix<double> M (nd,nd);
-    M.zero();
-    mth::multiply(vectorShapes, vectorShapesT, M);
-    M *= w;
-    elmat += M;
-  }
-  apf::destroyElement(el);
-  apf::destroyMeshElement(me);
-}
-
-
-// This function takes the NedelecField and computes
-// correction vectors on faces and stores them in a field
+/*
+ * This function takes the NedelecField and g parameters field
+ * and returns the THETA field. It computes the 3 scalar parameters
+ * per face for the flux correction vectors.
+ */
 apf::Field* computeFluxCorrection(apf::Field* ef, apf::Field* g)
 {
   apf::Mesh* mesh = apf::getMesh(ef);
-  int order = ef->getShape()->getOrder();
-
-  apf::Field* triNedelecField = apf::createField(
-      mesh, "tri_nedelec_field", apf::SCALAR, apf::getNedelec(order));
-  apf::zeroField(triNedelecField);
-
-  apf::Field* correctedFluxField =  createPackedField(
-      mesh, "corrected_flux_field", 3, apf::getConstant(2));
+  apf::Field* THETA_Field =  createPackedField(
+      mesh, "theta_field", 3, apf::getConstant(2));
 
-  int faceNo = 0;
   apf::MeshEntity* face;
   apf::MeshIterator* it = mesh->begin(2);
   while ((face = mesh->iterate(it))) {
 
-    cout << "FaceNo " << faceNo++ << endl;
-    apf::Downward face_vertices;
-    mesh->getDownward(face, 0, face_vertices);
-    apf::Vector3 pt;
-    mesh->getPoint(face_vertices[0], 0, pt);
-    cout << "  v0 " << pt << endl;
-    mesh->getPoint(face_vertices[1], 0, pt);
-    cout << "  v1 " << pt << endl;
-    mesh->getPoint(face_vertices[2], 0, pt);
-    cout << "  v2 " << pt << endl;
-    cout << endl;
-
     // 1. assemble RHS vector
     double components[3];
     apf::getComponents(g, face, 0, components);
-    cout << "initial gs without negation" << endl; // REMOVE
-    cout << components[0] << " " << components[1] << " " << components[2] << endl; // REMOVE
 
+    mth::Vector<double> rhs(3);
     apf::Downward edges;
     int ne = mesh->getDownward(face, 1, edges);
-    for (int i = 0; i < ne; i++) {
-      apf::setScalar(triNedelecField, edges[i], 0, components[i]);
-    }
-
-    apf::MeshElement* me = apf::createMeshElement(mesh, face);
-    apf::Element* el = apf::createElement(triNedelecField, me);
-    apf::NewArray<double> facegs;
-    el->getElementDofs(facegs);
-
-    mth::Vector<double> rhs(3); // TODO clean
-    for (size_t i = 0; i < 3; i++) {
-      rhs(i) = facegs[i];
-      //rhs(i) = components[i];
-    }
-    cout << "Face Edges Orientations ";
     for (int i = 0; i < ne; i++) {
       int which, rotate; bool flip;
       apf::getAlignment(mesh, face, edges[i], which, flip, rotate);
-      cout << flip << " ";
+      if (flip)
+        rhs(i) = -1. * components[i];
+      else
+        rhs(i) = components[i];
     }
-    cout << endl;
-    cout << "final gs with negation" << endl; // REMOVE
-    cout << rhs << endl; // REMOVE
-
-    apf::destroyElement(el);
-    apf::destroyMeshElement(me);
 
     // 2. assemble LHS face mass matrix
     mth::Matrix<double> M;
-    assembleFaceMassMatrix(
-        mesh, face, triNedelecField, M);
+    assembleVectorMassElementMatrix(mesh, face, ef, M);
 
     // 3. solve the system
     QRDecomp qr;
     mth::decomposeQR(M, qr.Q, qr.R);
     mth::Vector<double> theta;
     mth::solveFromQR(qr.Q, qr.R, rhs, theta);
-    std::cout << "LHS Face Mass Matrix" << std::endl; // REMOVE DEBUG
-    std::cout << M << std::endl;
-    std::cout << "RHS Vector" << std::endl;
-    std::cout << rhs << std::endl;
-    std::cout << "theta coeffs" << std::endl;
-    std::cout << theta << std::endl;
-    cout << "====================" << endl;
 
     // set solution vector on face field
-    //theta.toArray(components); TODO clean
     components[0] = theta(0);
     components[1] = theta(1);
     components[2] = theta(2);
-    apf::setComponents(
-        correctedFluxField, face, 0, components);
+    apf::setComponents(THETA_Field, face, 0, components);
   }
   mesh->end(it);
 
-  apf::destroyField(triNedelecField);
-
-  return correctedFluxField;
+  return THETA_Field;
 }
 
 }
diff --git a/em/emResidualFunctionals.cc b/em/emResidualFunctionals.cc
index d85a72881..adaf55fe9 100644
--- a/em/emResidualFunctionals.cc
+++ b/em/emResidualFunctionals.cc
@@ -242,6 +242,7 @@ void assembleVectorMassElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
   PCU_ALWAYS_ASSERT(type == apf::Mesh::TET || type == apf::Mesh::TRIANGLE);
   int nd = apf::countElementNodes(fs, type);
   int dim = apf::getDimension(mesh, e);
+  int sdim = mesh->getDimension();
   double w;
 
   apf::NewArray<apf::Vector3> vectorshapes(nd);
@@ -263,12 +264,12 @@ void assembleVectorMassElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
     w = weight * jdet;
 
     apf::getVectorShapeValues(el, p, vectorshapes);
-    mth::Matrix<double> vectorShapes (nd, dim);
+    mth::Matrix<double> vectorShapes (nd, sdim);
     for (int j = 0; j < nd; j++)
-      for (int k = 0; k < dim; k++)
+      for (int k = 0; k < sdim; k++)
         vectorShapes(j,k) = vectorshapes[j][k];
 
-    mth::Matrix<double> vectorShapesT (dim, nd);
+    mth::Matrix<double> vectorShapesT (sdim, nd);
     mth::transpose(vectorShapes, vectorShapesT);
 
     mth::Matrix<double> M (nd,nd);
@@ -277,6 +278,7 @@ void assembleVectorMassElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
     M *= w;
     elmat += M;
   }
+
   apf::destroyElement(el);
   apf::destroyMeshElement(me);
 }

From 47809cb119bd420ffbcce8a24087f7aaeed5e7f0 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Mon, 1 Jun 2020 17:09:46 -0400
Subject: [PATCH 180/555] Clean ups

---
 em/em.h               |  20 +++++---
 em/emEstimateError.cc | 114 ++++++------------------------------------
 2 files changed, 29 insertions(+), 105 deletions(-)

diff --git a/em/em.h b/em/em.h
index 3e60c1ef3..36bcc9c3d 100644
--- a/em/em.h
+++ b/em/em.h
@@ -27,19 +27,25 @@
 
 namespace em {
 
-// Takes the solution electric field and corrected flux field and solves
-// local element level BVPs to estimate the error.
-// Returns a per-element scalar error field.
-apf::Field* emEstimateError(apf::Field* ef, apf::Field* correctedFlux);
+/* Takes the solution electric field and corrected flux field and solves
+ * local element level BVPs to estimate the error.
+ * Returns a per-element scalar error field.
+ */
+apf::Field* computeErrorField(apf::Field* ef, apf::Field* THETA_Field);
 
-// Takes the solution electric field and equilibrated field (of face vectors)
-// and computes the 'correction' to the flux vectors on each face.
+/*
+ * Takes the solution electric field and equilibrated field (of face vectors)
+ * and computes the 'correction' to the flux vectors on each face.
+ */
 apf::Field* computeFluxCorrection(apf::Field* ef, apf::Field* g);
 
-// Takes the solution electric field and computes edge equilibrations.
+/*
+ * Takes the solution electric field and computes edge equilibrations.
+ */
 apf::Field* equilibrateResiduals(apf::Field* f);
 
 
+
 void assembleCurlCurlElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
     apf::Field* f, mth::Matrix<double>& elmat);
 
diff --git a/em/emEstimateError.cc b/em/emEstimateError.cc
index 9394c0a80..2d618950e 100644
--- a/em/emEstimateError.cc
+++ b/em/emEstimateError.cc
@@ -14,7 +14,6 @@
 #include "em.h"
 using namespace std;
 namespace em {
-//TODO destroy all fields created inside functions to prevent memory leaks.
 
 static void computeResidualBLF(apf::Mesh* mesh, apf::MeshEntity* e,
   apf::Field* f, apf::Field* fp1, mth::Vector<double>& blf)
@@ -38,7 +37,7 @@ static void computeResidualBLF(apf::Mesh* mesh, apf::MeshEntity* e,
   apf::Element* fp1el = apf::createElement(fp1, me);
   apf::Element* fel = apf::createElement(f, me);
   int int_order = 2 * fp1s->getOrder();
-  int np = apf::countIntPoints(me, int_order); // int points required
+  int np = apf::countIntPoints(me, int_order);
 
   // 1. Compute Curl Curl Integration
   curlcurl_vec.zero();
@@ -67,9 +66,7 @@ static void computeResidualBLF(apf::Mesh* mesh, apf::MeshEntity* e,
     V.zero();
     mth::Vector<double> c (dim);
     c(0) = curl[0]; c(1) = curl[1]; c(2) = curl[2];
-    cout << "curl " << c << endl;
     mth::multiply(phys_curlshape, c, V);
-    cout << "V " << V << endl;
     V *= w;
 
     curlcurl_vec += V;
@@ -117,31 +114,23 @@ static void computeResidualBLF(apf::Mesh* mesh, apf::MeshEntity* e,
   apf::destroyMeshElement(me);
   apf::destroyElement(fp1el);
   apf::destroyElement(fel);
-
-  // TODO Take care of Negative Dofs
 }
 
-// @ apf::Field* f --> input p-order tet ND field
-// @ apf::Field* fp1 --> p+1 order tet ND field created for local BVPs
-// @ apf::Field* THETA --> constant face field containing theta coeffs
 static void computeLambdaVector(
     apf::Mesh* mesh,
     apf::MeshEntity* e,
     apf::Field* f,
     apf::Field* fp1,
-    apf::Field* tri_nedelec_field,
-    apf::Field* THETA,
+    apf::Field* THETA_Field,
     mth::Vector<double>& lambda)
 {
   apf::FieldShape* fp1s = fp1->getShape();
+  int order = fp1s->getOrder();
   int etype = mesh->getType(e);
   PCU_ALWAYS_ASSERT(etype == apf::Mesh::TET);
   int nedofs = apf::countElementNodes(fp1s, etype);
   lambda.resize(nedofs);
 
-  // create a 2D nedelec field
-  int order = fp1s->getOrder();
-  apf::FieldShape* tri_nedelec_fieldShape = tri_nedelec_field->getShape();
 
   // get the downward faces of the element
   apf::Downward faces;
@@ -171,22 +160,20 @@ static void computeLambdaVector(
 
     // 3. get theta coeffs on the face
     double components[3];
-    apf::getComponents(THETA, face, 0, components);
-    //mth::Vector<double> theta_coeffs(components); // TODO clean
-    //mth::Vector<double> theta_coeffs(*components); // TODO clean
-    mth::Vector<double> theta_coeffs(3); // TODO clean
+    apf::getComponents(THETA_Field, face, 0, components);
+    mth::Vector<double> theta_coeffs(3);
     theta_coeffs(0) = components[0];
     theta_coeffs(1) = components[1];
     theta_coeffs(2) = components[2];
 
     int ftype = mesh->getType(face);
     PCU_ALWAYS_ASSERT(ftype == apf::Mesh::TRIANGLE);
-    int nfdofs = apf::countElementNodes(tri_nedelec_fieldShape, ftype);
+    int nfdofs = apf::countElementNodes(f->getShape(), ftype);
     apf::NewArray<apf::Vector3> vectorshape(nfdofs);
 
     apf::MeshElement* fme = apf::createMeshElement(mesh, face);
-    apf::Element* fel = apf::createElement(tri_nedelec_field, fme);
-    int np = apf::countIntPoints(fme, 2*order); // int points required
+    apf::Element* fel = apf::createElement(f, fme);
+    int np = apf::countIntPoints(fme, 2*order);
 
     // 4. Compute integral on the face
     apf::Vector3 p, tet1xi, tet2xi, curl1, curl2, curl,
@@ -205,12 +192,8 @@ static void computeLambdaVector(
       theta_vector.zero();
       apf::NewArray<apf::Vector3> triVectorShapes (nfdofs);
       apf::getVectorShapeValues(fel, p, triVectorShapes);
-      //int which, rotate; bool flip; // negative ND dofs
       for (int i = 0; i < nedges; i++) {
         apf::Vector3 v = triVectorShapes[i];
-        /*apf::getAlignment(mesh, face, edges[i], which, flip, rotate);
-        if (flip) { v = v * -1.; }*/
-
         v = v * theta_coeffs[i];
         theta_vector += v;
       }
@@ -227,8 +210,6 @@ static void computeLambdaVector(
           fnormal2 = computeFaceOutwardNormal(mesh, firstTet, face, p);
           theta_vector = theta_vector * -1.;
         }
-        std::cout << "normal1 " << fnormal1 << std::endl; // REMOVE
-        std::cout << "normal2 " << fnormal2 << std::endl; // REMOVE
       }
 
       curl.zero();
@@ -250,16 +231,13 @@ static void computeLambdaVector(
         apf::getCurl(el2, tet2xi, curl2);
         apf::Vector3 temp2 = apf::cross(fnormal2, curl2);
         curl += (temp2 * -1.);
-        curl = curl * 1./2.; //
+        curl = curl * 1./2.;
         apf::destroyElement(el2);
         apf::destroyMeshElement(me2);
       }
 
       // compute tk (inter-element averaged flux)
       tk = curl;
-      std::cout << "tk           " << tk << std::endl; // REMOVE
-      std::cout << "theta_coeffs " << theta_coeffs << std::endl; // REMOVE
-      std::cout << "theta_vector " << theta_vector << std::endl; // REMOVE
 
       // compute p+1 order 3D vector shapes
       apf::NewArray<apf::Vector3> tetVectorShapes (nedofs);
@@ -267,7 +245,7 @@ static void computeLambdaVector(
       apf::Element* el = apf::createElement(fp1, me);
       apf::Vector3 tetxi = apf::boundaryToElementXi(mesh, face, e, p);
       apf::getVectorShapeValues(el, tetxi, tetVectorShapes);
-      // TODO take care of negative ND dofs
+
       apf::destroyElement(el);
       apf::destroyMeshElement(me);
 
@@ -345,7 +323,7 @@ static void getEssentialElementNDDofs(apf::Mesh* mesh, apf::MeshEntity* e,
 }
 
 /**
- * Inputs: Matrix A, Vector X, Vector B, essential dofs, not essential dofs.
+ * Inputs: Matrix A, Vector X, Vector B, essential dofs, other dofs.
  * Output: reduced matrix A, reduced rhs B.
  */
 static void eliminateDBCs(
@@ -428,25 +406,18 @@ static double computeL2Error(apf::Mesh* mesh, apf::MeshEntity* e,
     w = weight * jdet;
 
     apf::getVectorShapeValues(el, p, vectorshape);
-		// TODO take care of negative dof values
     mth::Matrix<double> vectorShape (nd, dim);
     for (int j = 0; j < nd; j++)
       for (int k = 0; k < dim; k++)
         vectorShape(j,k) = vectorshape[j][k];
-    cout << "vector shapes" << vectorShape << endl;
 
     mth::Matrix<double> vectorShapeT (dim, nd);
     mth::transpose(vectorShape, vectorShapeT);
-    cout << "transpose vector shapes" << vectorShapeT << endl;
 
 		mth::Vector<double> err_func;
 		mth::multiply(vectorShapeT, error_dofs, err_func);
-		cout << "vshape   " << vectorShape << endl;
-		cout << "error dofs " << error_dofs << endl;
-		cout << "err_func " << err_func << endl;
 
 		error += w * (err_func * err_func);
-	  cout << "error squared " << error << endl;
 	}
 	if (error < 0.0)
 		error = -error;
@@ -454,93 +425,51 @@ static double computeL2Error(apf::Mesh* mesh, apf::MeshEntity* e,
 	return sqrt(error);
 }
 
-// @ apf::Field* ef --> input p order ND electric field
-// @ apf::Field* ef --> input theta corrected flux field
-// *** p+1 order field is created inside this function
-apf::Field* emEstimateError(apf::Field* ef, apf::Field* correctedFlux)
+apf::Field* computeErrorField(apf::Field* ef, apf::Field* THETA_Field)
 {
 
 	// 1. Create per-element SCALAR error field
   apf::Field* error_field = apf::createIPField(
-		apf::getMesh(ef), "residual_error_field", apf::SCALAR, 1); // TODO maybe use getConstant here
-
-  cout << "ELEMENT ERROR FIELD CREATED" << endl; // REMOVE
+		apf::getMesh(ef), "residual_error_field", apf::SCALAR, 1);
 
   // 2. Create p+1 order tet ND field
-  // and p order triangle ND field
   int order = ef->getShape()->getOrder();
   int orderp1 = order+1;
   apf::Field* efp1 = apf::createField(apf::getMesh(ef),
 	 "orderp1_nedelec_field", apf::SCALAR, apf::getNedelec(orderp1));
   apf::zeroField(efp1);
 
-  apf::Field* tri_nedelec_field = apf::createField(apf::getMesh(ef),
-      "face_nedelec_field", apf::SCALAR, apf::getNedelec(order));
-  apf::zeroField(tri_nedelec_field);;
-
-  cout << "P+1 ORDER FIELD CREATED" << endl; // REMOVE
-
   // 2. iterate over all elements of the mesh
   apf::MeshEntity* el;
   apf::MeshIterator* it = apf::getMesh(ef)->begin(3);
-  int elemNo = 0; // remove
   while ((el = apf::getMesh(ef)->iterate(it))) {
-    cout << "elemNo " << elemNo << endl;
+
     // 2(a). Assemble LHS element matrix
     mth::Matrix<double> A;
     assembleElementMatrix( apf::getMesh(ef), el, efp1, A);
-    // TODO Take care of negative dofs in lhs
-    cout << "LHS matrix assembled" << endl;
-    cout << A << endl;
 
     // 2(b). Compute Bilinear Form Vector
     mth::Vector<double> blf;
     computeResidualBLF(apf::getMesh(efp1), el, ef, efp1, blf);
-    // TODO Take care of negative dofs in blf
-    cout << "RHS bilinear form vector assembled" << endl;
-    cout << blf << endl;
 
     // 2(c). Compute Linear Form Vector
     mth::Vector<double> lf;
     assembleDomainLFElementVector(apf::getMesh(efp1), el, efp1, lf);
-    // TODO Take care of negative dofs in lf
-    cout << "RHS linear form vector assembled" << endl;
-    cout << lf << endl;
 
-    // 2(d). Compute Lamda Vector
+    // 2(d). Compute Lambda Vector
     mth::Vector<double> lambda;
     computeLambdaVector(
-        apf::getMesh(ef), el, ef, efp1, tri_nedelec_field, correctedFlux, lambda);
-    cout << "lambda vector " << lambda << endl;
-    double lambda_sum = 0.;
-    for(size_t i = 0; i < lambda.size(); i++) {
-      lambda_sum += lambda(i);
-    }
-    cout << "lambda_sum " << lambda_sum << endl;
-
-    // TODO Take care of negative dofs in lambda
-    cout << "RHS lambda vector assembled" << endl;
+        apf::getMesh(ef), el, ef, efp1, THETA_Field, lambda);
 
     // 2(e). Assemble RHS element vector = blf - lf - lambda
     mth::Vector<double> B(blf.size());
     B.zero();
     B += blf; B -= lf; B -= lambda;
-    cout << "RHS Vector assembled" << endl;
 
     // 2(f). Get List of Essential Dofs
     apf::NewArray<int> ess_dofs, uness_dofs;
     getEssentialElementNDDofs(
         apf::getMesh(efp1), el, efp1, ess_dofs, uness_dofs);
-    cout << "Get list of Essential Dofs" << endl;
-    for(size_t i = 0; i < ess_dofs.size(); i++) {
-      cout << ess_dofs[i] << " ";
-    }
-    cout << endl;
-    cout << "Get list of UnEssential Dofs" << endl;
-    for(size_t i = 0; i < uness_dofs.size(); i++) {
-      cout << uness_dofs[i] << " ";
-    }
-    cout << endl;
 
     // 2(g). eliminate Dirichlet (Essential) Boundary Conditions
 		mth::Vector<double> X, Bnew;
@@ -548,14 +477,12 @@ apf::Field* emEstimateError(apf::Field* ef, apf::Field* correctedFlux)
 		X.resize(B.size());
 		X.zero(); // initialize X with exact DBC (e = 0.0)
 		eliminateDBCs(A, X, B, ess_dofs, uness_dofs, Anew, Bnew);
-		cout << "eliminate DBCs" << endl;
 
 		// 2(h). Solve the reduced system
 	  mth::Matrix<double> Q, R;
 		mth::decomposeQR(Anew, Q, R);
 		mth::Vector<double> Xnew;
 		mth::solveFromQR(Q, R, Bnew, Xnew);
-		cout << "Solve the reduced system" << endl;
 
 		// 2(i). Recover the solution
 		mth::Vector<double> error_dofs(B.size());
@@ -567,22 +494,13 @@ apf::Field* emEstimateError(apf::Field* ef, apf::Field* correctedFlux)
       int index = uness_dofs[i];
       error_dofs(index) = Xnew(i);
     }
-    cout << "Recover the solution" << endl;
-    cout << error_dofs << endl;
 
 		// 2(j). Compute L2 Norm Error
 		double l2_error = computeL2Error(apf::getMesh(ef), el, efp1, error_dofs);
-		cout << "Compute L2 element error" << l2_error << endl;
-
 		apf::setScalar(error_field, el, 0, l2_error);
-    cout << "Write the L2 error to error_field" << endl;
-
-    elemNo++;
   }
   apf::getMesh(ef)->end(it);
-  cout << "End loop over elements" << endl;
 	apf::destroyField(efp1);
-  apf::destroyField(tri_nedelec_field);
 
 	return error_field;
 }

From 766a07cab487b243c2dac0417abc505844e67285 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Mon, 1 Jun 2020 17:56:48 -0400
Subject: [PATCH 181/555] Adds estimateError() to package error estimator

---
 em/em.h | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/em/em.h b/em/em.h
index 36bcc9c3d..f71c8c2cc 100644
--- a/em/em.h
+++ b/em/em.h
@@ -27,11 +27,10 @@
 
 namespace em {
 
-/* Takes the solution electric field and corrected flux field and solves
- * local element level BVPs to estimate the error.
- * Returns a per-element scalar error field.
+/*
+ * Takes the solution electric field and computes edge equilibrations.
  */
-apf::Field* computeErrorField(apf::Field* ef, apf::Field* THETA_Field);
+apf::Field* equilibrateResiduals(apf::Field* f);
 
 /*
  * Takes the solution electric field and equilibrated field (of face vectors)
@@ -39,11 +38,13 @@ apf::Field* computeErrorField(apf::Field* ef, apf::Field* THETA_Field);
  */
 apf::Field* computeFluxCorrection(apf::Field* ef, apf::Field* g);
 
-/*
- * Takes the solution electric field and computes edge equilibrations.
+/* Takes the solution electric field and corrected flux field and solves
+ * local element level BVPs to estimate the error.
+ * Returns a per-element scalar error field.
  */
-apf::Field* equilibrateResiduals(apf::Field* f);
+apf::Field* computeErrorField(apf::Field* ef, apf::Field* THETA_Field);
 
+apf::Field* estimateError(apf::Field* f);
 
 
 void assembleCurlCurlElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e,

From 6e85f364fcab27cbb5c116ed93d699ad36f03372 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Mon, 1 Jun 2020 17:57:53 -0400
Subject: [PATCH 182/555] Adds estimateError to package error esitmator in 1

---
 em/emEstimateError.cc | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/em/emEstimateError.cc b/em/emEstimateError.cc
index 2d618950e..51358655f 100644
--- a/em/emEstimateError.cc
+++ b/em/emEstimateError.cc
@@ -505,4 +505,16 @@ apf::Field* computeErrorField(apf::Field* ef, apf::Field* THETA_Field)
 	return error_field;
 }
 
+apf::Field* estimateError(apf::Field* f)
+{
+  apf::Field* g = em::equilibrateResiduals(f);
+  apf::Field* THETA = em::computeFluxCorrection(f, g);
+  apf::destroyField(g);
+
+  apf::Field* error_field = em::computeErrorField(f, THETA);
+  apf::destroyField(THETA);
+
+  return error_field;
+}
+
 }

From 51d2eeaf6f4465078190cc12cdc8ce9ca0a13e44 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Tue, 2 Jun 2020 18:16:00 -0400
Subject: [PATCH 183/555] Adding size field computations for error estimator

---
 em/CMakeLists.txt     |  3 ++-
 em/em.h               |  8 ++++++++
 em/emEstimateError.cc | 23 +++++++++++++++++++++++
 3 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/em/CMakeLists.txt b/em/CMakeLists.txt
index 229ae59f9..8628e8f06 100644
--- a/em/CMakeLists.txt
+++ b/em/CMakeLists.txt
@@ -7,7 +7,8 @@ endif()
 set(SOURCES
   emResidualFunctionals.cc
   emFluxCorrection.cc
-  emEstimateError.cc)
+  emEstimateError.cc
+  emSizeField.cc)
 
 # Package headers
 set(HEADERS
diff --git a/em/em.h b/em/em.h
index f71c8c2cc..20b5a5303 100644
--- a/em/em.h
+++ b/em/em.h
@@ -27,6 +27,14 @@
 
 namespace em {
 
+/*
+ * Computes nodal size field // TODO currently only element size field
+ */
+apf::Field* getTargetEMSizeField(
+    apf::Field* ef,
+    apf::Field* error_field,
+    double alpha,
+    double beta);
 /*
  * Takes the solution electric field and computes edge equilibrations.
  */
diff --git a/em/emEstimateError.cc b/em/emEstimateError.cc
index 51358655f..d47bef20c 100644
--- a/em/emEstimateError.cc
+++ b/em/emEstimateError.cc
@@ -517,4 +517,27 @@ apf::Field* estimateError(apf::Field* f)
   return error_field;
 }
 
+
+/*class SizeFieldOp : public apf::CavityOp
+{
+public:
+  SizeFieldOpOp(apf::Field* error_field):
+    apf::CavityOp(error_field->getMesh())
+  {}
+  virtual Outcome setEntity(apf::MeshEntity* e)
+  {
+    entity = e;
+    if (apf::hasEntity(error_field, entity))
+      return SKIP;
+    if ( !requestLocality(&entity,1))
+      return REQUEST;
+    return OK;
+  }
+  virtual void apply()
+  {
+
+  }
+  apf::MeshEntity* e;
+};*/
+
 }

From 4d90076c820f63e67935ac6c17de5b1ddf982c40 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Wed, 3 Jun 2020 14:14:55 -0400
Subject: [PATCH 184/555] Adds time to estimate error

---
 em/emEstimateError.cc | 28 +++++-----------------------
 1 file changed, 5 insertions(+), 23 deletions(-)

diff --git a/em/emEstimateError.cc b/em/emEstimateError.cc
index d47bef20c..cf0523326 100644
--- a/em/emEstimateError.cc
+++ b/em/emEstimateError.cc
@@ -507,6 +507,7 @@ apf::Field* computeErrorField(apf::Field* ef, apf::Field* THETA_Field)
 
 apf::Field* estimateError(apf::Field* f)
 {
+  double t0 = PCU_Time();
   apf::Field* g = em::equilibrateResiduals(f);
   apf::Field* THETA = em::computeFluxCorrection(f, g);
   apf::destroyField(g);
@@ -514,30 +515,11 @@ apf::Field* estimateError(apf::Field* f)
   apf::Field* error_field = em::computeErrorField(f, THETA);
   apf::destroyField(THETA);
 
+  double t1 = PCU_Time();
+  if (!PCU_Comm_Self())
+    lion_eprint(1,"EM: Error estimated in %f seconds\n",t1-t0);
+
   return error_field;
 }
 
-
-/*class SizeFieldOp : public apf::CavityOp
-{
-public:
-  SizeFieldOpOp(apf::Field* error_field):
-    apf::CavityOp(error_field->getMesh())
-  {}
-  virtual Outcome setEntity(apf::MeshEntity* e)
-  {
-    entity = e;
-    if (apf::hasEntity(error_field, entity))
-      return SKIP;
-    if ( !requestLocality(&entity,1))
-      return REQUEST;
-    return OK;
-  }
-  virtual void apply()
-  {
-
-  }
-  apf::MeshEntity* e;
-};*/
-
 }

From 9b83439742a4a9274f8a6df183e1865df8780987 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Wed, 3 Jun 2020 14:16:00 -0400
Subject: [PATCH 185/555] Adds target SizeField computation

---
 em/em.h           |   5 +-
 em/emSizeField.cc | 179 ++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 182 insertions(+), 2 deletions(-)
 create mode 100644 em/emSizeField.cc

diff --git a/em/em.h b/em/em.h
index 20b5a5303..f5cae5d29 100644
--- a/em/em.h
+++ b/em/em.h
@@ -33,8 +33,9 @@ namespace em {
 apf::Field* getTargetEMSizeField(
     apf::Field* ef,
     apf::Field* error_field,
-    double alpha,
-    double beta);
+    int n,
+    double alpha = 0.25,
+    double beta = 2.0);
 /*
  * Takes the solution electric field and computes edge equilibrations.
  */
diff --git a/em/emSizeField.cc b/em/emSizeField.cc
new file mode 100644
index 000000000..1ae6d92b2
--- /dev/null
+++ b/em/emSizeField.cc
@@ -0,0 +1,179 @@
+/*
+ * Copyright (C) 2011 Scientific Computation Research Center
+ *
+ * This work is open source software, licensed under the terms of the
+ * BSD license as described in the LICENSE file in the top-level directory.
+ */
+#include <iostream>
+#include <cstdlib>
+
+#include "apfElement.h"
+#include "crv.h"
+#include "crvShape.h"
+#include <apfCavityOp.h>
+
+#include "em.h"
+using namespace std;
+namespace em {
+
+
+struct Sizefield {
+  apf::Mesh* mesh;
+  /* the polynomial order of the solution electric Nedelec field */
+  int order;
+  /* the input solution electric field obtained after the FEM solve */
+  apf::Field* ef;
+  /* the per element error field obtained after executing the implicit
+     residual error estimator */
+  apf::Field* errorField;
+  double size_factor;
+  /* target error = sum of all element erros / total number of elements */
+  double target_error;
+  double alpha;
+  double beta;
+  /* a temporary field storing desired sizes at elements */
+  apf::Field* element_size;
+  /* the resulting size field, recovered from the element_size field
+     (using a local average recovery method much weaker than SPR) */
+  apf::Field* size;
+};
+
+static void setupSizefield(
+    Sizefield* sz,
+    apf::Field* ef,
+    apf::Field* error_field,
+    int n,
+    double alpha,
+    double beta)
+{
+  sz->mesh = ef->getMesh();
+  sz->order = ef->getShape()->getOrder();
+  sz->errorField = error_field;
+  sz->size_factor = 0;
+
+  apf::MeshEntity* entity;
+  int d = sz->mesh->getDimension();
+  apf::MeshIterator* it = sz->mesh->begin(d);
+  double total_error = 0.;
+  while ((entity = sz->mesh->iterate(it))) {
+    total_error += apf::getScalar(sz->errorField, entity, 0);
+  }
+  sz->mesh->end(it);
+  sz->target_error = total_error / (sz->mesh->count(d)* n);
+
+  sz->alpha = alpha;
+  sz->beta = beta;
+  sz->element_size = 0;
+  sz->size = 0;
+}
+
+static double getCurrentSize(apf::Mesh* m, apf::MeshEntity* e)
+{
+  /* right now minimum edge length is the formula... */
+  apf::Downward edges;
+  int ne = m->getDownward(e,1,edges);
+  double h = std::numeric_limits<double>::max();
+  for (int i=0; i < ne; ++i)
+    h = std::min(h, measure(m, edges[i]));
+  return h;
+}
+
+static double getDesiredSize(Sizefield* sz, apf::MeshEntity* entity)
+{
+  double element_error = apf::getScalar(sz->errorField, entity, 0);
+  double h = getCurrentSize(sz->mesh, entity);
+
+  double p = sz->order;
+  int d = sz->mesh->getDimension();
+  sz->size_factor = pow(sz->target_error/element_error, (2. / (2.*p + d)));
+
+  double h_new = h * element_error * sz->size_factor;
+  if (h_new < sz->alpha*h) h_new = sz->alpha*h;
+  if (h_new > sz->beta*h) h_new = sz->beta*h;
+  return h_new;
+}
+
+static void getElementSizeField(Sizefield* sz)
+{
+  apf::Field* eSize = apf::createStepField(
+      sz->mesh, "esize", apf::SCALAR);
+  int d = sz->mesh->getDimension();
+  apf::MeshEntity* entity;
+  apf::MeshIterator* elements = sz->mesh->begin(d);
+  while ((entity = sz->mesh->iterate(elements))) {
+    double h = getDesiredSize(sz, entity);
+    apf::setScalar(eSize, entity, 0, h);
+    cout << "h " << h << endl;
+  }
+  sz->mesh->end(elements);
+  sz->element_size = eSize;
+}
+
+
+void averageToVertex(apf::Field* ef, apf::Field* vf, apf::MeshEntity* ent)
+{
+  apf::Mesh* m = apf::getMesh(ef);
+  apf::Adjacent elements;
+  m->getAdjacent(ent, m->getDimension(), elements);
+  double s=0;
+  for (std::size_t i=0; i < elements.getSize(); ++i)
+    s += apf::getScalar(ef, elements[i], 0);
+  s /= elements.getSize();
+  apf::setScalar(vf, ent, 0, s);
+}
+
+class AverageOp : public apf::CavityOp
+{
+public:
+  AverageOp(Sizefield* sz):
+    apf::CavityOp(sz->mesh),
+    sizefield(sz),
+    entity(0)
+  {
+  }
+  virtual Outcome setEntity(apf::MeshEntity* e)
+  {
+    entity = e;
+    if (apf::hasEntity(sizefield->size, entity))
+      return SKIP;
+    if ( !requestLocality(&entity,1))
+      return REQUEST;
+    return OK;
+  }
+  virtual void apply()
+  {
+    averageToVertex(sizefield->element_size,
+        sizefield->size, entity);
+  }
+  Sizefield* sizefield;
+  apf::MeshEntity* entity;
+};
+
+void averageSizeField(Sizefield* sz)
+{
+  sz->size = apf::createLagrangeField(sz->mesh, "size", apf::SCALAR, 1);
+  AverageOp op(sz);
+  op.applyToDimension(0);
+}
+
+apf::Field* getTargetEMSizeField(
+    apf::Field* ef,
+    apf::Field* error_field,
+    int n,
+    double alpha /*= 0.25*/,
+    double beta /*= 2.0*/)
+{
+  double t0 = PCU_Time();
+  Sizefield sz;
+  setupSizefield(&sz, ef, error_field, n, alpha, beta);
+  getElementSizeField(&sz);
+  averageSizeField(&sz);
+  apf::destroyField(sz.element_size);
+  double t1 = PCU_Time();
+  if (!PCU_Comm_Self())
+    lion_eprint(1,"EM: SizeField computed in %f seconds\n",t1-t0);
+  return sz.size;
+}
+
+
+}

From 0335ec5079485e2ff4670011b7cb41ca2d380e37 Mon Sep 17 00:00:00 2001
From: Morteza HS <hakimi1364@gmail.com>
Date: Sat, 6 Jun 2020 19:40:48 -0400
Subject: [PATCH 186/555] Adds registerSelf to Nedelec class

---
 apf/apfNedelec.cc | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index 574c800d5..87536a91e 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -683,10 +683,15 @@ static void getTi(
 template<int P>
 class Nedelec: public FieldShape {
   public:
-    const char* getName() const { return "Nedelec"; }
-    bool isVectorShape() {return true;}
     Nedelec()
-    {}
+    {
+      std::stringstream ss;
+      ss << "Nedelec_" << P;
+      name = ss.str();
+      registerSelf(name.c_str());
+    }
+    const char* getName() const { return name.c_str(); }
+    bool isVectorShape() {return true;}
     class Vertex : public apf::EntityShape
     {
     public:
@@ -1277,6 +1282,8 @@ class Nedelec: public FieldShape {
       else
         t = Vector3(0, 0, 0);
     }
+  private:
+    std::string name;
 };
 
 apf::FieldShape* getNedelec(int order)

From ba574b5a3c27242a3f15fc9761e06f23be465857 Mon Sep 17 00:00:00 2001
From: Morteza HS <hakimi1364@gmail.com>
Date: Sat, 6 Jun 2020 19:51:41 -0400
Subject: [PATCH 187/555] Adds getNedelec(1) to getShapeByName

---
 apf/apfShape.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/apf/apfShape.cc b/apf/apfShape.cc
index fe4f38b7d..657f408bc 100644
--- a/apf/apfShape.cc
+++ b/apf/apfShape.cc
@@ -78,6 +78,7 @@ FieldShape* getShapeByName(const char* name)
   getIPShape(2,1);
   getVoronoiShape(2,1);
   getIPFitShape(2,1);
+  getNedelec(1);
   std::string s(name);
   if (registry.count(s))
     return registry[s];

From 033a349ecb69967c3d623f569b2e74e8f7620ca1 Mon Sep 17 00:00:00 2001
From: Morteza HS <hakimi1364@gmail.com>
Date: Sat, 13 Jun 2020 14:54:26 -0400
Subject: [PATCH 188/555] Adds a projectNedelec field (no nodal field)

---
 apf/apfNedelec.cc | 119 ++++++++++++++++++++++++++++++++++++++++++++++
 apf/apfShape.h    |   3 ++
 2 files changed, 122 insertions(+)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index 87536a91e..c241821c4 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -1307,4 +1307,123 @@ apf::FieldShape* getNedelec(int order)
   return nedelecShapes[order-1];
 }
 
+void projectNedelecField(Field* to, Field* from)
+{
+	// checks on the from field
+	// checks on the to field
+	apf::FieldShape* tShape = getShape(to);
+	std::string      tName  = tShape->getName();
+	int              tOrder = tShape->getOrder();
+	PCU_ALWAYS_ASSERT_VERBOSE(tName == std::string("Linear"),
+			"The to field needs to be 1st order Lagrange!");
+
+	Mesh* m = getMesh(from);
+	// auxiliary count fields
+	Field* count = createField(m, "counter", SCALAR, getLagrange(1));
+	double xis[4][3] = {{0., 0., 0.},
+		                  {1., 0., 0.},
+		                  {0., 1., 0.},
+		                  {0., 0., 1.}};
+  // zero out the fields
+	zeroField(to);
+	zeroField(count);
+
+	MeshEntity* e;
+	MeshIterator* it = m->begin(m->getDimension());
+	while( (e = m->iterate(it)) ) {
+		MeshElement* me = createMeshElement(m, e);
+		Element* el = createElement(from, me);
+		MeshEntity* dvs[4];
+		m->getDownward(e, 0, dvs);
+		for (int i=0; i<4; i++) {
+			Vector3 atXi;
+			getVector(el, Vector3(xis[i]), atXi);
+		  Vector3 currentVal;	
+			getVector(to, dvs[i], 0, currentVal);
+			double currentCount = getScalar(count, dvs[i], 0);
+			currentVal += atXi;
+			currentCount += 1.;
+			setVector(to, dvs[i], 0, currentVal);
+			setScalar(count, dvs[i], 0, currentCount);
+		}
+		destroyElement(el);
+		destroyMeshElement(me);
+	}
+	m->end(it);
+
+	it = m->begin(0);
+	while( (e = m->iterate(it)) ) {
+		Vector3 sum;
+		double cnt;
+		getVector(to, e, 0, sum);
+		setVector(to, e, 0, sum/getScalar(count, e, 0));
+	}
+	m->end(it);
+	m->removeField(count);
+	destroyField(count);
+}
+
+/* void projectNedelecCurlField(Field* to, Field* from) */
+/* { */
+/* 	// checks on the from field */
+/* 	apf::FieldShape* fShape = getShape(from); */
+/* 	std::string      fName  = fShape->getName(); */
+/* 	PCU_ALWAYS_ASSERT_VERBOSE(fName == std::string("Nedelec"), */
+/* 			"The from field needs to be Nedelec!"); */
+
+/* 	// checks on the to field */
+/* 	apf::FieldShape* tShape = getShape(to); */
+/* 	std::string      tName  = tShape->getName(); */
+/* 	int              tOrder = tShape->getOrder(); */
+/* 	PCU_ALWAYS_ASSERT_VERBOSE(tName == std::string("Lagrange"), */
+/* 			"The to field needs to be Lagrange!"); */
+/* 	PCU_ALWAYS_ASSERT_VERBOSE(tOrder == 1, */
+/* 			"The to field needs to be 1st order!"); */
+
+/* 	/1* Mesh* m = getMesh(from); *1/ */
+/* 	/1* // auxiliary count fields *1/ */
+/* 	/1* Field* count = createField(m, "counter", SCALAR, getLagrange(1)); *1/ */
+/* 	/1* double xis[4][3] = {{0., 0., 0.}, *1/ */
+/* 	/1* 	                  {1., 0., 0.}, *1/ */
+/* 	/1* 	                  {0., 1., 0.}, *1/ */
+/* 	/1* 	                  {0., 0., 1.}}; *1/ */
+/*   /1* // zero out the fields *1/ */
+/* 	/1* zeroField(to); *1/ */
+/* 	/1* zeroField(count); *1/ */
+
+/* 	/1* MeshEntity* e; *1/ */
+/* 	/1* MeshIterator* it = m->begin(m->getDimension()); *1/ */
+/* 	/1* while( (e = m->iterate(it)) ) { *1/ */
+/* 	/1* 	MeshElement* me = createMeshElement(m, e); *1/ */
+/* 	/1* 	Element* el = createElement(from, me); *1/ */
+/* 	/1* 	MeshEntity* dvs[4]; *1/ */
+/* 	/1* 	m->getDownward(e, 0, dvs); *1/ */
+/* 	/1* 	for (int i=0; i<4; i++) { *1/ */
+/* 	/1* 		Vector3 atXi; *1/ */
+/* 	/1* 		getVector(el, Vector3(xis[i]), atXi); *1/ */
+/* 	/1* 	  Vector3 currentVal; *1/ */	
+/* 	/1* 		getVector(to, dvs[i], 0, currentVal); *1/ */
+/* 	/1* 		double currentCount = getScalar(count, dvs[i], 0); *1/ */
+/* 	/1* 		currentVal += atXi; *1/ */
+/* 	/1* 		currentCount += 1.; *1/ */
+/* 	/1* 		setVector(to, dvs[i], 0, currentVal); *1/ */
+/* 	/1* 		setScalar(count, dvs[i], 0, currentCount); *1/ */
+/* 	/1* 	} *1/ */
+/* 	/1* 	destroyElement(el); *1/ */
+/* 	/1* 	destroyMeshElement(me); *1/ */
+/* 	/1* } *1/ */
+/* 	/1* m->end(it); *1/ */
+
+/* 	/1* it = m->begin(0); *1/ */
+/* 	/1* while( (e = m->iterate(it)) ) { *1/ */
+/* 	/1* 	Vector3 sum; *1/ */
+/* 	/1* 	double cnt; *1/ */
+/* 	/1* 	getVector(to, e, 0, sum); *1/ */
+/* 	/1* 	getScalar(count, e, 0); *1/ */
+/* 	/1* 	setVector(to, e, 0, sum/getScalar(count, e, 0)); *1/ */
+/* 	/1* } *1/ */
+/* 	/1* m->end(it); *1/ */
+/* 	/1* destroyField(count); *1/ */
+/* } */
+
 };
diff --git a/apf/apfShape.h b/apf/apfShape.h
index 37e259408..df6429e40 100644
--- a/apf/apfShape.h
+++ b/apf/apfShape.h
@@ -170,6 +170,9 @@ FieldShape* getNedelec(int order);
 /** \brief Project a hierarchic field */
 void projectHierarchicField(Field* to, Field* from);
 
+/** \brief Project a Nedelec field */
+void projectNedelecField(Field* to, Field* from);
+
 FieldShape* getShapeByName(const char* name);
 
 /** \brief count the number of nodes affecting an element type

From 9e3c6a1b61e9549b09beda181d836ece120a7ee5 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Sat, 13 Jun 2020 15:00:12 -0400
Subject: [PATCH 189/555] Fixes indentations and removes commented code

---
 apf/apfNedelec.cc | 165 ++++++++++++++--------------------------------
 1 file changed, 51 insertions(+), 114 deletions(-)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index c241821c4..25c673af7 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -1309,121 +1309,58 @@ apf::FieldShape* getNedelec(int order)
 
 void projectNedelecField(Field* to, Field* from)
 {
-	// checks on the from field
-	// checks on the to field
-	apf::FieldShape* tShape = getShape(to);
-	std::string      tName  = tShape->getName();
-	int              tOrder = tShape->getOrder();
-	PCU_ALWAYS_ASSERT_VERBOSE(tName == std::string("Linear"),
-			"The to field needs to be 1st order Lagrange!");
-
-	Mesh* m = getMesh(from);
-	// auxiliary count fields
-	Field* count = createField(m, "counter", SCALAR, getLagrange(1));
-	double xis[4][3] = {{0., 0., 0.},
-		                  {1., 0., 0.},
-		                  {0., 1., 0.},
-		                  {0., 0., 1.}};
+  // checks on the from field
+  // checks on the to field
+  apf::FieldShape* tShape = getShape(to);
+  std::string      tName  = tShape->getName();
+  int              tOrder = tShape->getOrder();
+  PCU_ALWAYS_ASSERT_VERBOSE(tName == std::string("Linear"),
+  		"The to field needs to be 1st order Lagrange!");
+
+  Mesh* m = getMesh(from);
+  // auxiliary count fields
+  Field* count = createField(m, "counter", SCALAR, getLagrange(1));
+  double xis[4][3] = {{0., 0., 0.},
+  	              {1., 0., 0.},
+  	              {0., 1., 0.},
+  	              {0., 0., 1.}};
   // zero out the fields
-	zeroField(to);
-	zeroField(count);
-
-	MeshEntity* e;
-	MeshIterator* it = m->begin(m->getDimension());
-	while( (e = m->iterate(it)) ) {
-		MeshElement* me = createMeshElement(m, e);
-		Element* el = createElement(from, me);
-		MeshEntity* dvs[4];
-		m->getDownward(e, 0, dvs);
-		for (int i=0; i<4; i++) {
-			Vector3 atXi;
-			getVector(el, Vector3(xis[i]), atXi);
-		  Vector3 currentVal;	
-			getVector(to, dvs[i], 0, currentVal);
-			double currentCount = getScalar(count, dvs[i], 0);
-			currentVal += atXi;
-			currentCount += 1.;
-			setVector(to, dvs[i], 0, currentVal);
-			setScalar(count, dvs[i], 0, currentCount);
-		}
-		destroyElement(el);
-		destroyMeshElement(me);
-	}
-	m->end(it);
-
-	it = m->begin(0);
-	while( (e = m->iterate(it)) ) {
-		Vector3 sum;
-		double cnt;
-		getVector(to, e, 0, sum);
-		setVector(to, e, 0, sum/getScalar(count, e, 0));
-	}
-	m->end(it);
-	m->removeField(count);
-	destroyField(count);
+  zeroField(to);
+  zeroField(count);
+
+  MeshEntity* e;
+  MeshIterator* it = m->begin(m->getDimension());
+  while( (e = m->iterate(it)) ) {
+    MeshElement* me = createMeshElement(m, e);
+    Element* el = createElement(from, me);
+    MeshEntity* dvs[4];
+    m->getDownward(e, 0, dvs);
+    for (int i=0; i<4; i++) {
+      Vector3 atXi;
+      getVector(el, Vector3(xis[i]), atXi);
+      Vector3 currentVal;
+      getVector(to, dvs[i], 0, currentVal);
+      double currentCount = getScalar(count, dvs[i], 0);
+      currentVal += atXi;
+      currentCount += 1.;
+      setVector(to, dvs[i], 0, currentVal);
+      setScalar(count, dvs[i], 0, currentCount);
+    }
+    destroyElement(el);
+    destroyMeshElement(me);
+  }
+  m->end(it);
+
+  it = m->begin(0);
+  while( (e = m->iterate(it)) ) {
+    Vector3 sum;
+    double cnt;
+    getVector(to, e, 0, sum);
+    setVector(to, e, 0, sum/getScalar(count, e, 0));
+  }
+  m->end(it);
+  m->removeField(count);
+  destroyField(count);
 }
 
-/* void projectNedelecCurlField(Field* to, Field* from) */
-/* { */
-/* 	// checks on the from field */
-/* 	apf::FieldShape* fShape = getShape(from); */
-/* 	std::string      fName  = fShape->getName(); */
-/* 	PCU_ALWAYS_ASSERT_VERBOSE(fName == std::string("Nedelec"), */
-/* 			"The from field needs to be Nedelec!"); */
-
-/* 	// checks on the to field */
-/* 	apf::FieldShape* tShape = getShape(to); */
-/* 	std::string      tName  = tShape->getName(); */
-/* 	int              tOrder = tShape->getOrder(); */
-/* 	PCU_ALWAYS_ASSERT_VERBOSE(tName == std::string("Lagrange"), */
-/* 			"The to field needs to be Lagrange!"); */
-/* 	PCU_ALWAYS_ASSERT_VERBOSE(tOrder == 1, */
-/* 			"The to field needs to be 1st order!"); */
-
-/* 	/1* Mesh* m = getMesh(from); *1/ */
-/* 	/1* // auxiliary count fields *1/ */
-/* 	/1* Field* count = createField(m, "counter", SCALAR, getLagrange(1)); *1/ */
-/* 	/1* double xis[4][3] = {{0., 0., 0.}, *1/ */
-/* 	/1* 	                  {1., 0., 0.}, *1/ */
-/* 	/1* 	                  {0., 1., 0.}, *1/ */
-/* 	/1* 	                  {0., 0., 1.}}; *1/ */
-/*   /1* // zero out the fields *1/ */
-/* 	/1* zeroField(to); *1/ */
-/* 	/1* zeroField(count); *1/ */
-
-/* 	/1* MeshEntity* e; *1/ */
-/* 	/1* MeshIterator* it = m->begin(m->getDimension()); *1/ */
-/* 	/1* while( (e = m->iterate(it)) ) { *1/ */
-/* 	/1* 	MeshElement* me = createMeshElement(m, e); *1/ */
-/* 	/1* 	Element* el = createElement(from, me); *1/ */
-/* 	/1* 	MeshEntity* dvs[4]; *1/ */
-/* 	/1* 	m->getDownward(e, 0, dvs); *1/ */
-/* 	/1* 	for (int i=0; i<4; i++) { *1/ */
-/* 	/1* 		Vector3 atXi; *1/ */
-/* 	/1* 		getVector(el, Vector3(xis[i]), atXi); *1/ */
-/* 	/1* 	  Vector3 currentVal; *1/ */	
-/* 	/1* 		getVector(to, dvs[i], 0, currentVal); *1/ */
-/* 	/1* 		double currentCount = getScalar(count, dvs[i], 0); *1/ */
-/* 	/1* 		currentVal += atXi; *1/ */
-/* 	/1* 		currentCount += 1.; *1/ */
-/* 	/1* 		setVector(to, dvs[i], 0, currentVal); *1/ */
-/* 	/1* 		setScalar(count, dvs[i], 0, currentCount); *1/ */
-/* 	/1* 	} *1/ */
-/* 	/1* 	destroyElement(el); *1/ */
-/* 	/1* 	destroyMeshElement(me); *1/ */
-/* 	/1* } *1/ */
-/* 	/1* m->end(it); *1/ */
-
-/* 	/1* it = m->begin(0); *1/ */
-/* 	/1* while( (e = m->iterate(it)) ) { *1/ */
-/* 	/1* 	Vector3 sum; *1/ */
-/* 	/1* 	double cnt; *1/ */
-/* 	/1* 	getVector(to, e, 0, sum); *1/ */
-/* 	/1* 	getScalar(count, e, 0); *1/ */
-/* 	/1* 	setVector(to, e, 0, sum/getScalar(count, e, 0)); *1/ */
-/* 	/1* } *1/ */
-/* 	/1* m->end(it); *1/ */
-/* 	/1* destroyField(count); *1/ */
-/* } */
-
 };

From d794da03e0e33e399aeccefd47ca9fa5c760e97e Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Sat, 13 Jun 2020 15:08:21 -0400
Subject: [PATCH 190/555] Fixes unused variable compile errors

---
 apf/apfNedelec.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index 25c673af7..7e7b09708 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -1314,7 +1314,7 @@ void projectNedelecField(Field* to, Field* from)
   apf::FieldShape* tShape = getShape(to);
   std::string      tName  = tShape->getName();
   int              tOrder = tShape->getOrder();
-  PCU_ALWAYS_ASSERT_VERBOSE(tName == std::string("Linear"),
+  PCU_ALWAYS_ASSERT_VERBOSE((tName == std::string("Linear")) && (tOrder == 1),
   		"The to field needs to be 1st order Lagrange!");
 
   Mesh* m = getMesh(from);
@@ -1354,7 +1354,6 @@ void projectNedelecField(Field* to, Field* from)
   it = m->begin(0);
   while( (e = m->iterate(it)) ) {
     Vector3 sum;
-    double cnt;
     getVector(to, e, 0, sum);
     setVector(to, e, 0, sum/getScalar(count, e, 0));
   }

From 0fee4ff58af31e970ac41e085f22229d1a489254 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Wed, 17 Jun 2020 12:25:06 -0400
Subject: [PATCH 191/555] clean up. fixes error in sizefield computation

---
 em/emEstimateError.cc       |  6 ++----
 em/emResidualFunctionals.cc | 14 ++++++++------
 em/emSizeField.cc           |  2 +-
 3 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/em/emEstimateError.cc b/em/emEstimateError.cc
index cf0523326..87ceaecd5 100644
--- a/em/emEstimateError.cc
+++ b/em/emEstimateError.cc
@@ -47,10 +47,8 @@ static void computeResidualBLF(apf::Mesh* mesh, apf::MeshEntity* e,
     double weight = apf::getIntWeight(me, int_order, i);
     apf::Matrix3x3 J;
     apf::getJacobian(me, p, J);
-    //double jdet = apf::getJacobianDeterminant(J, dim);
-    w = weight; // TODO check why do not need division by jdet
+    w = weight;
 
-    // get curl vector
     apf::getCurl(fel, p, curl);
 
     // get curlshape values // TODO CLEAN use getCurlShapeValues
@@ -74,7 +72,7 @@ static void computeResidualBLF(apf::Mesh* mesh, apf::MeshEntity* e,
 
   // 2. Compute Vector Mass Integration
   int_order = 2 * fp1s->getOrder();
-  np = apf::countIntPoints(me, int_order); // int points required
+  np = apf::countIntPoints(me, int_order);
 
   mass_vec.zero();
   apf::Vector3 vvalue;
diff --git a/em/emResidualFunctionals.cc b/em/emResidualFunctionals.cc
index adaf55fe9..e4b22ff98 100644
--- a/em/emResidualFunctionals.cc
+++ b/em/emResidualFunctionals.cc
@@ -168,22 +168,21 @@ static void assembleEdgePatchLHS(EdgePatch* ep)
 
 /*
  * Performs Curl Curl integration using curl vector Nedelec shapes
- * TODO Add 2D curl curl integration
- * TODO CLEANUP.
+ * TODO Add 2D curl curl integration. not needed right now.
  */
 void assembleCurlCurlElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
     apf::Field* f, mth::Matrix<double>& elmat)
 {
   apf::FieldShape* fs = f->getShape();
   int type = mesh->getType(e);
-  PCU_ALWAYS_ASSERT(type == apf::Mesh::TET); // TODO add Triangle
+  PCU_ALWAYS_ASSERT(type == apf::Mesh::TET);
   int nd = apf::countElementNodes(fs, type);
   int dim = apf::getDimension(mesh, e);
   int dimc = (dim == 3) ? 3 : 1;
   double w;
 
   apf::NewArray<apf::Vector3> curlshape(nd);
-  mth::Matrix<double> phys_curlshape(nd, dimc); // TODO remove once curl Piola is in place in apfNedelec
+  mth::Matrix<double> phys_curlshape(nd, dimc);
   elmat.resize(nd,nd);
 
   apf::MeshElement* me = apf::createMeshElement(mesh, e);
@@ -201,7 +200,7 @@ void assembleCurlCurlElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
     double jdet = apf::getJacobianDeterminant(J, dim);
     w = weight / jdet;
 
-    if (dim == 3) { // TODO this is Piola transformation. Put it in apfNedelec
+    if (dim == 3) {
       el->getShape()->getLocalVectorCurls(mesh, e, p, curlshape);
       phys_curlshape.zero();
       for (int j = 0; j < nd; j++)
@@ -211,7 +210,7 @@ void assembleCurlCurlElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
     }
     else {
       /*
-      TODO 2D TODO in apfNedelec.cc
+      TODO 2D in apfNedelec.cc
       el->getShape()->getLocalVectorCurls(mesh, e, p, curlshape);
       phys_curlshape.zero();
       for (int i = 0; i < nd; i++)
@@ -358,6 +357,9 @@ void pumiUserFunction(apf::Mesh* mesh, apf::MeshEntity* e,
       f(0) = (1. + kappa * kappa) * sin(kappa * x[1]);
       f(1) = (1. + kappa * kappa) * sin(kappa * x[2]);
       f(2) = (1. + kappa * kappa) * sin(kappa * x[0]);
+      /*f(0) = 0.;
+      f(1) = 0.;
+      f(2) = 0.;*/
   }
   else {
      f(0) = (1. + kappa * kappa) * sin(kappa * x[1]);
diff --git a/em/emSizeField.cc b/em/emSizeField.cc
index 1ae6d92b2..fbffadcae 100644
--- a/em/emSizeField.cc
+++ b/em/emSizeField.cc
@@ -87,7 +87,7 @@ static double getDesiredSize(Sizefield* sz, apf::MeshEntity* entity)
   int d = sz->mesh->getDimension();
   sz->size_factor = pow(sz->target_error/element_error, (2. / (2.*p + d)));
 
-  double h_new = h * element_error * sz->size_factor;
+  double h_new = h * sz->size_factor;
   if (h_new < sz->alpha*h) h_new = sz->alpha*h;
   if (h_new > sz->beta*h) h_new = sz->beta*h;
   return h_new;

From 3c41d345134b3db11fca2489806b41195e5f332c Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Wed, 17 Jun 2020 13:33:41 -0400
Subject: [PATCH 192/555] clean ups

---
 em/emResidualFunctionals.cc | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/em/emResidualFunctionals.cc b/em/emResidualFunctionals.cc
index e4b22ff98..eae8d9c32 100644
--- a/em/emResidualFunctionals.cc
+++ b/em/emResidualFunctionals.cc
@@ -210,7 +210,7 @@ void assembleCurlCurlElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
     }
     else {
       /*
-      TODO 2D in apfNedelec.cc
+      2D in apfNedelec.cc
       el->getShape()->getLocalVectorCurls(mesh, e, p, curlshape);
       phys_curlshape.zero();
       for (int i = 0; i < nd; i++)
@@ -623,14 +623,6 @@ static double getLocalFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
       apf::getVectorShapeValues(fel, p, vectorshapes);
       vshape = vectorshapes[ei];
 
-      // TODO why?
-      int which, rotate; bool flip;
-      apf::getAlignment(
-          ep->mesh, currentFace, ep->entity, which, flip, rotate);
-      if (flip) {
-      vshape = vshape * -1.;
-      }
-
       // compute integral
       fluxFaceIntegral += (tk * vshape) * weight * jdet;
     }

From aef7745041f6115eeaf5c30ba7478c7fde2be3b4 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Thu, 18 Jun 2020 10:29:05 -0400
Subject: [PATCH 193/555] Adds some updates l2 shapes + cleanup

---
 apf/apfL2Shapes.cc           | 62 +++++++++++++++++++++++++++---------
 apf/apfShape.cc              |  2 ++
 apf/apfShape.h               |  2 +-
 apf/apfVtkPieceWiseFields.cc |  3 +-
 test/L2Shapes.cc             |  2 +-
 5 files changed, 52 insertions(+), 19 deletions(-)

diff --git a/apf/apfL2Shapes.cc b/apf/apfL2Shapes.cc
index e3a9c7b9f..d9f35fff0 100644
--- a/apf/apfL2Shapes.cc
+++ b/apf/apfL2Shapes.cc
@@ -179,10 +179,15 @@ static void getTi(
 template<int P>
 class L2ShapeTri: public FieldShape {
   public:
-    const char* getName() const { return "L2ShapeTri"; }
-    bool isVectorShape() {return false;}
     L2ShapeTri()
-    {}
+    {
+      std::stringstream ss;
+      ss << "L2ShapeTri" << P;
+      name = ss.str();
+      registerSelf(name.c_str());
+    }
+    const char* getName() const { return name.c_str(); }
+    bool isVectorShape() {return false;}
     class Triangle : public apf::EntityShape
     {
     public:
@@ -256,9 +261,9 @@ class L2ShapeTri: public FieldShape {
     // Faces: no need to count the nodes associated with bounding edges
     // Tets: no need to count the nodes associated with bounding edges/faces
     bool hasNodesIn(int dimension)
-    { 
-    	if (dimension == Mesh::typeDimension[Mesh::TRIANGLE])
-    		return true;
+    {
+      if (dimension == Mesh::typeDimension[Mesh::TRIANGLE])
+      	return true;
       return false;
     }
     int countNodesOn(int type)
@@ -286,15 +291,22 @@ class L2ShapeTri: public FieldShape {
       	}
       }
     }
+  private:
+    std::string name;
 };
 
 template<int P>
 class L2ShapeTet: public FieldShape {
   public:
-    const char* getName() const { return "L2ShapeTet"; }
-    bool isVectorShape() {return false;}
     L2ShapeTet()
-    {}
+    {
+      std::stringstream ss;
+      ss << "L2ShapeTet_" << P;
+      name = ss.str();
+      registerSelf(name.c_str());
+    }
+    const char* getName() const { return name.c_str(); }
+    bool isVectorShape() {return false;}
     class Tetrahedron : public apf::EntityShape
     {
     public:
@@ -354,7 +366,7 @@ class L2ShapeTet: public FieldShape {
       }
       int countNodes() const {return countTetNodes(P);}
       void getVectorValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
-	  apf::Vector3 const& xi, apf::NewArray<apf::Vector3>& shapes) const
+	  apf::Vector3 const& /*xi*/, apf::NewArray<apf::Vector3>& /*shapes*/) const
       {
       	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getVectorValues not implemented for \
       	    L2ShapeTet. Try getValues. Aborting()!");
@@ -404,6 +416,8 @@ class L2ShapeTet: public FieldShape {
       	}
       }
     }
+  private:
+    std::string name;
 };
 
 
@@ -424,8 +438,17 @@ static apf::FieldShape* getL2ShapeTri(int order)
   static L2ShapeTri<8>  l2_8;
   static L2ShapeTri<9>  l2_9;
   static L2ShapeTri<10> l2_10;
-  static FieldShape* const l2Shapes[11] =
-  {&l2_0, &l2_1, &l2_2, &l2_3, &l2_4, &l2_5, &l2_6, &l2_7, &l2_8, &l2_9, &l2_10};
+  static FieldShape* const l2Shapes[11] = {&l2_0,
+                                           &l2_1,
+                                           &l2_2,
+                                           &l2_3,
+                                           &l2_4,
+                                           &l2_5,
+                                           &l2_6,
+                                           &l2_7,
+                                           &l2_8,
+                                           &l2_9,
+                                           &l2_10};
   return l2Shapes[order];
 }
 
@@ -446,13 +469,22 @@ static apf::FieldShape* getL2ShapeTet(int order)
   static L2ShapeTet<8>  l2_8;
   static L2ShapeTet<9>  l2_9;
   static L2ShapeTet<10> l2_10;
-  static FieldShape* const l2Shapes[11] =
-  {&l2_0, &l2_1, &l2_2, &l2_3, &l2_4, &l2_5, &l2_6, &l2_7, &l2_8, &l2_9, &l2_10};
+  static FieldShape* const l2Shapes[11] = {&l2_0,
+                                           &l2_1,
+                                           &l2_2,
+                                           &l2_3,
+                                           &l2_4,
+                                           &l2_5,
+                                           &l2_6,
+                                           &l2_7,
+                                           &l2_8,
+                                           &l2_9,
+                                           &l2_10};
   return l2Shapes[order];
 }
 
 
-apf::FieldShape* getL2Shapes(int order, int type)
+apf::FieldShape* getL2Shape(int order, int type)
 {
   if (type == Mesh::TRIANGLE)
     return getL2ShapeTri(order);
diff --git a/apf/apfShape.cc b/apf/apfShape.cc
index 657f408bc..5a4973e4a 100644
--- a/apf/apfShape.cc
+++ b/apf/apfShape.cc
@@ -79,6 +79,8 @@ FieldShape* getShapeByName(const char* name)
   getVoronoiShape(2,1);
   getIPFitShape(2,1);
   getNedelec(1);
+  getL2Shape(0, apf::Mesh::TRIANGLE);
+  getL2Shape(0, apf::Mesh::TET);
   std::string s(name);
   if (registry.count(s))
     return registry[s];
diff --git a/apf/apfShape.h b/apf/apfShape.h
index e0b7af427..9aac0f33a 100644
--- a/apf/apfShape.h
+++ b/apf/apfShape.h
@@ -170,7 +170,7 @@ FieldShape* getNedelec(int order);
 /** \brief Get the L2 shapes of a given order and entity type
   \details
  */
-FieldShape* getL2Shapes(int order, int type);
+FieldShape* getL2Shape(int order, int type);
 
 /** \brief Project a hierarchic field */
 void projectHierarchicField(Field* to, Field* from);
diff --git a/apf/apfVtkPieceWiseFields.cc b/apf/apfVtkPieceWiseFields.cc
index 63eaac9e0..028f2274f 100644
--- a/apf/apfVtkPieceWiseFields.cc
+++ b/apf/apfVtkPieceWiseFields.cc
@@ -77,9 +77,8 @@ static std::string getFileNameAndPathVtu(const char* prefix,
     std::string fileName,
     int id)
 {
-  int dirNum = id/1024;
   std::stringstream ss;
-  ss << prefix << '/' << prefix << '_' << fileName;
+  ss << prefix << '/' << prefix << '_' << id << '_' << fileName;
   return ss.str();
 }
 
diff --git a/test/L2Shapes.cc b/test/L2Shapes.cc
index 18cc7bf93..7c38b3869 100644
--- a/test/L2Shapes.cc
+++ b/test/L2Shapes.cc
@@ -86,7 +86,7 @@ void testL2(
   // Loop over all nodes and set scalar dofs.
   int dim = m->getDimension();
   apf::Field* l2Field = apf::createField(
-      m, "l2_test", apf::VECTOR, apf::getL2Shapes(ndOrder, apf::Mesh::simplexTypes[dim]));
+      m, "l2_test", apf::VECTOR, apf::getL2Shape(ndOrder, apf::Mesh::simplexTypes[dim]));
   apf::MeshEntity* ent;
   apf::MeshIterator* it;
 

From d180677026f1b50b5bc0cb8129215b535b1a5ef9 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Thu, 18 Jun 2020 10:43:17 -0400
Subject: [PATCH 194/555] Cleans the L2Shape tests

---
 test/L2Shapes.cc | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/test/L2Shapes.cc b/test/L2Shapes.cc
index 7c38b3869..82c41f440 100644
--- a/test/L2Shapes.cc
+++ b/test/L2Shapes.cc
@@ -30,7 +30,7 @@ int main(int argc, char** argv)
   MPI_Init(&argc,&argv);
   PCU_Comm_Init();
 
-  lion_set_verbosity(1);
+  lion_set_verbosity(0);
 
   if (argc != 3) {
     if(0==PCU_Comm_Self())
@@ -50,7 +50,7 @@ int main(int argc, char** argv)
     testL2(
     	m, /* mesh */
 	apf::Vector3(1./7., 1./11., 1./3.), /* test point */
-	i, /* order of Nedelec field */
+	i, /* order of l2 field */
 	i); /* order of test field */
   }
 
@@ -135,8 +135,8 @@ void testL2(
     apf::Vector3 eFieldValue;
     apf::getVector(el, testXi, eFieldValue);
 
-    double err = ((eFieldValue - eFieldExact) * (eFieldValue - eFieldExact))
-      / (eFieldExact * eFieldExact); // normalization factor
+    double err = ((eFieldValue - eFieldExact) * (eFieldValue - eFieldExact));
+    err /= (eFieldExact * eFieldExact); // normalization factor
     L2ErrorE += err;
     apf::destroyMeshElement(me);
     apf::destroyElement(el);
@@ -145,6 +145,7 @@ void testL2(
   m->end(it);
 
   // check for field interpolation
+  lion_oprint(1, "L2ErrorE is %e\n", L2ErrorE);
   PCU_ALWAYS_ASSERT_VERBOSE(L2ErrorE < 1.e-16,
       "Fields were not interpolated correctly!");
 

From 78d0c75add89070be33effe5bee263e090f223f8 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Thu, 18 Jun 2020 15:49:30 -0400
Subject: [PATCH 195/555] Adds project to nodal (Lagrange 1) for L2 Fields

---
 apf/apfL2Shapes.cc | 71 ++++++++++++++++++++++++++++++++++++++++++++++
 apf/apfShape.h     |  3 ++
 2 files changed, 74 insertions(+)

diff --git a/apf/apfL2Shapes.cc b/apf/apfL2Shapes.cc
index d9f35fff0..0ce82344f 100644
--- a/apf/apfL2Shapes.cc
+++ b/apf/apfL2Shapes.cc
@@ -495,4 +495,75 @@ apf::FieldShape* getL2Shape(int order, int type)
     	"L2Shapes are only implemented for tris and tets");
 }
 
+void projectL2Field(Field* to, Field* from)
+{
+  // checks on the from field
+  // checks on the to field
+  apf::FieldShape* tShape = getShape(to);
+  std::string      tName  = tShape->getName();
+  int              tOrder = tShape->getOrder();
+  PCU_ALWAYS_ASSERT_VERBOSE((tName == std::string("Linear")) && (tOrder == 1),
+                "The to field needs to be 1st order Lagrange!");
+
+  Mesh* m = getMesh(from);
+  // auxiliary count fields
+  Field* count = createField(m, "counter", SCALAR, getLagrange(1));
+  double xis[4][3] = {{0., 0., 0.},
+                      {1., 0., 0.},
+                      {0., 1., 0.},
+                      {0., 0., 1.}};
+  // zero out the fields
+  zeroField(to);
+  zeroField(count);
+
+  int nc = countComponents(to);
+  NewArray<double> atXi(nc);
+  NewArray<double> currentVal(nc);
+  NewArray<double> sum(nc);
+
+  MeshEntity* e;
+  MeshIterator* it = m->begin(m->getDimension());
+  while( (e = m->iterate(it)) ) {
+    MeshElement* me = createMeshElement(m, e);
+    Element* el = createElement(from, me);
+    MeshEntity* dvs[4];
+    m->getDownward(e, 0, dvs);
+    for (int i=0; i<4; i++) {
+      getComponents(el, Vector3(xis[i]), &(atXi[0]));
+      getComponents(to, dvs[i], 0, &(currentVal[0]));
+      for (int j = 0; j < nc; j++) {
+        currentVal[j] += atXi[j];
+      }
+      double currentCount = getScalar(count, dvs[i], 0);
+      currentCount += 1.;
+      setComponents(to, dvs[i], 0, &(currentVal[0]));
+      setScalar(count, dvs[i], 0, currentCount);
+    }
+    destroyElement(el);
+    destroyMeshElement(me);
+  }
+  m->end(it);
+
+  // take care of entities on part boundary
+  accumulate(to);
+  accumulate(count);
+
+  it = m->begin(0);
+  while( (e = m->iterate(it)) ) {
+    getComponents(to, e, 0, &(sum[0]));
+    int cnt = getScalar(count, e, 0);
+    for (int i = 0; i < nc; i++) {
+      sum[i] /= cnt;
+    }
+    setComponents(to, e, 0, &(sum[0]));
+  }
+  m->end(it);
+
+  // take care of entities on part boundary
+  synchronize(to);
+
+  m->removeField(count);
+  destroyField(count);
+}
+
 };
diff --git a/apf/apfShape.h b/apf/apfShape.h
index 9aac0f33a..927e0df18 100644
--- a/apf/apfShape.h
+++ b/apf/apfShape.h
@@ -178,6 +178,9 @@ void projectHierarchicField(Field* to, Field* from);
 /** \brief Project a Nedelec field */
 void projectNedelecField(Field* to, Field* from);
 
+/** \brief Project a L2 field */
+void projectL2Field(Field* to, Field* from);
+
 FieldShape* getShapeByName(const char* name);
 
 /** \brief count the number of nodes affecting an element type

From b01b5efe1404af3a614f09f499f499904af86d25 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Thu, 18 Jun 2020 15:50:36 -0400
Subject: [PATCH 196/555] Fixes projectNedelec to work in parallel

---
 apf/apfNedelec.cc | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index 4fdfc5f53..6d2d690c8 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -1182,6 +1182,10 @@ void projectNedelecField(Field* to, Field* from)
   }
   m->end(it);
 
+  // take care of entities on part boundary
+  accumulate(to);
+  accumulate(count);
+
   it = m->begin(0);
   while( (e = m->iterate(it)) ) {
     Vector3 sum;
@@ -1189,6 +1193,10 @@ void projectNedelecField(Field* to, Field* from)
     setVector(to, e, 0, sum/getScalar(count, e, 0));
   }
   m->end(it);
+
+  // take care of entities on part boundary
+  synchronize(to);
+
   m->removeField(count);
   destroyField(count);
 }

From ac65529c62980902f0947f3ee6ee5128d8aa8a54 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Thu, 18 Jun 2020 16:17:16 -0400
Subject: [PATCH 197/555] Updates to L2Shape test

Also tests project to nodal and writing a mesh including and l2 field into native.
---
 test/L2Shapes.cc | 88 ++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 81 insertions(+), 7 deletions(-)

diff --git a/test/L2Shapes.cc b/test/L2Shapes.cc
index 82c41f440..f8554b1d8 100644
--- a/test/L2Shapes.cc
+++ b/test/L2Shapes.cc
@@ -25,12 +25,16 @@ void testL2(
     const apf::Vector3& testXi,
     int ndOrder, int exactOrder);
 
+void testL2writeNative(
+    apf::Mesh2* m,
+    int ndOrder, int exactOrder);
+
 int main(int argc, char** argv)
 {
   MPI_Init(&argc,&argv);
   PCU_Comm_Init();
 
-  lion_set_verbosity(0);
+  lion_set_verbosity(1);
 
   if (argc != 3) {
     if(0==PCU_Comm_Self())
@@ -54,6 +58,8 @@ int main(int argc, char** argv)
 	i); /* order of test field */
   }
 
+  testL2writeNative(m, 3, 3);
+
   apf::destroyMesh(m);
   PCU_Comm_Free();
   MPI_Finalize();
@@ -64,7 +70,7 @@ void E_exact(const apf::Vector3& x, apf::Vector3& value, int p)
   // Polynomial coefficients for each component of exact vector field
   double a[7] = { 1.0, -1.0,  2., -2., -1.0,  1.0, -1.0};
   double b[7] = {-2.0,  1.0, -2.,  2., -1.0, -1.0,  1.0};
-  double c[7] = { 3.0,  0.0, -1.,  0., -1.0,  1.0,  0.0};
+  double c[7] = { 3.0,  0.0, -1.,  1., -1.0,  1.0,  0.0};
 
   value[0] = 0.0;
   value[1] = 0.0;
@@ -76,8 +82,6 @@ void E_exact(const apf::Vector3& x, apf::Vector3& value, int p)
   }
 }
 
-
-
 void testL2(
     apf::Mesh2* m,
     const apf::Vector3& testXi,
@@ -92,11 +96,13 @@ void testL2(
 
   for (int d = 0; d <= dim; d++) {
     if (!l2Field->getShape()->countNodesOn(apf::Mesh::simplexTypes[d])) {
-      lion_oprint(1, "no nodes in dimension %d\n", d);
+      if(0==PCU_Comm_Self())
+        lion_oprint(1, "no nodes in dimension %d\n", d);
       continue;
     }
     else
-      lion_oprint(1, "computing dofs for dimension %d\n", d);
+      if(0==PCU_Comm_Self())
+        lion_oprint(1, "computing dofs for dimension %d\n", d);
     it = m->begin(d);
     int count = 0;
     while( (ent = m->iterate(it)) ) {
@@ -145,9 +151,77 @@ void testL2(
   m->end(it);
 
   // check for field interpolation
-  lion_oprint(1, "L2ErrorE is %e\n", L2ErrorE);
+  if(0==PCU_Comm_Self())
+    lion_oprint(1, "L2ErrorE is %e\n", L2ErrorE);
   PCU_ALWAYS_ASSERT_VERBOSE(L2ErrorE < 1.e-16,
       "Fields were not interpolated correctly!");
 
+  m->removeField(l2Field);
+  apf::destroyField(l2Field);
+}
+
+void testL2writeNative(
+    apf::Mesh2* m,
+    int ndOrder, int exactOrder)
+{
+  // Loop over all nodes and set scalar dofs.
+  int dim = m->getDimension();
+  apf::Field* l2Field = apf::createField(
+      m, "l2_test", apf::VECTOR, apf::getL2Shape(ndOrder, apf::Mesh::simplexTypes[dim]));
+  apf::MeshEntity* ent;
+  apf::MeshIterator* it;
+
+  for (int d = 0; d <= dim; d++) {
+    if (!l2Field->getShape()->countNodesOn(apf::Mesh::simplexTypes[d])) {
+      if(0==PCU_Comm_Self())
+        lion_oprint(1, "no nodes in dimension %d\n", d);
+      continue;
+    }
+    else
+      if(0==PCU_Comm_Self())
+        lion_oprint(1, "computing dofs for dimension %d\n", d);
+    it = m->begin(d);
+    int count = 0;
+    while( (ent = m->iterate(it)) ) {
+      int type = m->getType(ent);
+      int non = l2Field->getShape()->countNodesOn(type);
+      apf::MeshElement* me = apf::createMeshElement(m, ent);
+      for (int i = 0; i < non; i++)
+      {
+        apf::Vector3 xi, p, value;
+        l2Field->getShape()->getNodeXi(type, i, xi);
+        apf::mapLocalToGlobal(me, xi, p);
+        E_exact(p, value, exactOrder);
+        apf::setVector(l2Field, ent, i, value);
+      }
+      apf::destroyMeshElement(me);
+      count++;
+    }
+    m->end(it);
+  }
+
+  // project to nodal field
+  apf::Field* projectedL2Field = apf::createField(m, "projected_l2_test", apf::VECTOR,
+      apf::getLagrange(1));
+  apf::projectL2Field(projectedL2Field, l2Field);
+
+  // write to native
+  // 1- write the mesh to native
+  // 2- read the mesh back in and make sure fields are on the mesh
+  // 3- clean up the newly loaded mesh
+  m->writeNative("L2Shape_test_mesh.smb");
+  apf::Mesh2* m2 = apf::loadMdsMesh(".null", "./L2Shape_test_mesh.smb");
+  int fCount = 0;
+  for (int i = 0; i < m2->countFields(); i++) {
+    if(0==PCU_Comm_Self())
+      lion_oprint(1, "field %d's name and shape are %s and %s\n", i,
+	  m2->getField(i)->getName(), m2->getField(i)->getShape()->getName());
+    fCount++;
+  }
+  PCU_ALWAYS_ASSERT_VERBOSE(fCount == 2, "Expecting 2 fields on m2 at this point");
+
+  apf::destroyMesh(m2);
+  // clean up the fields
   apf::destroyField(l2Field);
+  apf::destroyField(projectedL2Field);
 }

From 27de4bbde29f5aaa264b05f33339d2714f60246e Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Thu, 18 Jun 2020 16:34:16 -0400
Subject: [PATCH 198/555] Adds a parallel regression test for l2 shapes

---
 test/testing.cmake | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/test/testing.cmake b/test/testing.cmake
index f33be609f..6fd8120e7 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -517,10 +517,14 @@ mpi_test(nedelec 1
   ./nedelecShapes
   "${MDIR}/cube.dmg"
   "${MDIR}/cube.smb")
-mpi_test(l2_shape_tet 1
+mpi_test(l2_shape_tet_serial 1
   ./L2Shapes
   "${MDIR}/cube.dmg"
   "${MDIR}/cube.smb")
+mpi_test(l2_shape_tet_parallel 4
+  ./L2Shapes
+  "${MDIR}/cube.dmg"
+  "${MDIR}/4p/.smb")
 mpi_test(pumiLoadMesh-1p 1
   ./pumiLoadMesh
   ${MDIR}/cube.dmg

From ba3c837b14e2ecac91b035084458af54e7211c42 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Thu, 18 Jun 2020 19:53:38 -0400
Subject: [PATCH 199/555] Fixes the failing test on Travis

---
 test/testing.cmake | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/test/testing.cmake b/test/testing.cmake
index 6fd8120e7..5841957fd 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -517,14 +517,16 @@ mpi_test(nedelec 1
   ./nedelecShapes
   "${MDIR}/cube.dmg"
   "${MDIR}/cube.smb")
+set(MDIR ${MESHES}/cube/pumi670)
 mpi_test(l2_shape_tet_serial 1
   ./L2Shapes
-  "${MDIR}/cube.dmg"
+  ".null"
   "${MDIR}/cube.smb")
 mpi_test(l2_shape_tet_parallel 4
   ./L2Shapes
-  "${MDIR}/cube.dmg"
-  "${MDIR}/4p/.smb")
+  ".null"
+  "${MDIR}/4/cube.smb")
+set(MDIR ${MESHES}/cube/pumi24)
 mpi_test(pumiLoadMesh-1p 1
   ./pumiLoadMesh
   ${MDIR}/cube.dmg

From a8ec79098d238e257428df75ead2a25f0bc30c89 Mon Sep 17 00:00:00 2001
From: Morteza HS <hakimi1364@gmail.com>
Date: Fri, 19 Jun 2020 10:32:53 -0400
Subject: [PATCH 200/555] Removes commented code

---
 apf/apfVtkPieceWiseFields.cc | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/apf/apfVtkPieceWiseFields.cc b/apf/apfVtkPieceWiseFields.cc
index 028f2274f..35016f775 100644
--- a/apf/apfVtkPieceWiseFields.cc
+++ b/apf/apfVtkPieceWiseFields.cc
@@ -52,18 +52,6 @@ static void safe_mkdir(const char* path)
  * before writing it to file, the others are to maintain the
  * templated design of writeCornerCoords and others */
 
-/* static float workaround(double v) */
-/* { */
-/*   return static_cast<float>(v); */
-/* } */
-/* static int workaround(int v) */
-/* { */
-/*   return v; */
-/* } */
-/* static long workaround(long v) */
-/* { */
-/*   return v; */
-/* } */
 
 static std::string getPieceFileName(int id)
 {
@@ -72,7 +60,6 @@ static std::string getPieceFileName(int id)
   return ss.str();
 }
 
-
 static std::string getFileNameAndPathVtu(const char* prefix,
     std::string fileName,
     int id)

From 317b20b1a57a558049e129412890bd9dec2bc4cf Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Fri, 19 Jun 2020 10:57:47 -0400
Subject: [PATCH 201/555] Fixes warnings from Travis

---
 em/emEstimateError.cc       | 10 +++++++---
 em/emResidualFunctionals.cc | 10 +++++++---
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/em/emEstimateError.cc b/em/emEstimateError.cc
index 87ceaecd5..ed859b56f 100644
--- a/em/emEstimateError.cc
+++ b/em/emEstimateError.cc
@@ -148,7 +148,7 @@ static void computeLambdaVector(
       PCU_ALWAYS_ASSERT( up.n == 2);
 
     apf::MeshEntity* firstTet = up.e[0];
-    apf::MeshEntity* secondTet;
+    apf::MeshEntity* secondTet = nullptr;
     if (up.n == 2)
       secondTet  = up.e[1];
 
@@ -197,10 +197,14 @@ static void computeLambdaVector(
       }
 
       // compute face outward normals wrt tets
-      if (e == firstTet)
+      if (e == firstTet) {
         fnormal1 = computeFaceOutwardNormal(mesh, firstTet, face, p);
-      else
+        fnormal2 = apf::Vector3(0.,0.,0.);
+      }
+      else {
         fnormal1 = computeFaceOutwardNormal(mesh, secondTet, face, p);
+        fnormal2 = apf::Vector3(0.,0.,0.);
+      }
       if (up.n == 2) {
         if (e == firstTet)
           fnormal2 = computeFaceOutwardNormal(mesh, secondTet, face, p);
diff --git a/em/emResidualFunctionals.cc b/em/emResidualFunctionals.cc
index eae8d9c32..c0cd637e3 100644
--- a/em/emResidualFunctionals.cc
+++ b/em/emResidualFunctionals.cc
@@ -549,7 +549,7 @@ static double getLocalFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
       PCU_ALWAYS_ASSERT( up.n == 2);
 
     apf::MeshEntity* firstTet = up.e[0];
-    apf::MeshEntity* secondTet;
+    apf::MeshEntity* secondTet = nullptr;
     if (up.n == 2)
       secondTet  = up.e[1];
 
@@ -578,10 +578,14 @@ static double getLocalFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
           fJ, apf::getDimension(ep->mesh, currentFace));
 
       // compute face outward normals wrt tets
-      if (tet == firstTet)
+      if (tet == firstTet) {
         fn1 = computeFaceOutwardNormal(ep->mesh, firstTet, currentFace, p);
-      else
+        fn2 = apf::Vector3(0.,0.,0.);
+      }
+      else {
         fn1 = computeFaceOutwardNormal(ep->mesh, secondTet, currentFace, p);
+        fn2 = apf::Vector3(0.,0.,0.);
+      }
       if (up.n == 2) {
         if (tet == firstTet)
           fn2 = computeFaceOutwardNormal(ep->mesh, secondTet, currentFace, p);

From e43073746adb6245b49529451d66f3535e364720 Mon Sep 17 00:00:00 2001
From: Morteza HS <hakimi1364@gmail.com>
Date: Mon, 29 Jun 2020 05:58:27 -0400
Subject: [PATCH 202/555] Updates the pkg_tribits.cmake

Adds Nedelec and L2 shape related source files to pkg_tribits.cmake
---
 apf/CMakeLists.txt    | 4 ++--
 apf/pkg_tribits.cmake | 5 +++++
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/apf/CMakeLists.txt b/apf/CMakeLists.txt
index 94d4d6e5e..47b943f46 100644
--- a/apf/CMakeLists.txt
+++ b/apf/CMakeLists.txt
@@ -25,9 +25,9 @@ set(SOURCES
   apfShape.cc
   apfIPShape.cc
   apfHierarchic.cc
-	apfPolyBasis1D.cc
+  apfPolyBasis1D.cc
   apfNedelec.cc
-	apfL2Shapes.cc
+  apfL2Shapes.cc
   apfVector.cc
   apfVectorElement.cc
   apfVectorField.cc
diff --git a/apf/pkg_tribits.cmake b/apf/pkg_tribits.cmake
index a0c6cff0c..37ad6a0ad 100644
--- a/apf/pkg_tribits.cmake
+++ b/apf/pkg_tribits.cmake
@@ -28,9 +28,14 @@ set(APF_SOURCES
   apfShape.cc
   apfIPShape.cc
   apfHierarchic.cc
+  apfPolyBasis1D.cc
+  apfNedelec.cc
+  apfL2Shapes.cc
   apfVector.cc
   apfVectorElement.cc
   apfVectorField.cc
+  apfMixedVectorElement.cc
+  apfMixedVectorField.cc
   apfPackedField.cc
   apfNumbering.cc
   apfMixedNumbering.cc

From 7ed8852a54860da91bd09e7d9267a059a9fa1194 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 15 Jul 2020 10:27:35 -0400
Subject: [PATCH 203/555] replace VLA #314, remove extra semicolons

---
 apf/apfL2Shapes.cc    | 2 +-
 apf/apfNedelec.cc     | 8 +++++---
 apf/apfPolyBasis1D.cc | 2 +-
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/apf/apfL2Shapes.cc b/apf/apfL2Shapes.cc
index 0ce82344f..783f993a2 100644
--- a/apf/apfL2Shapes.cc
+++ b/apf/apfL2Shapes.cc
@@ -566,4 +566,4 @@ void projectL2Field(Field* to, Field* from)
   destroyField(count);
 }
 
-};
+}
diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index 6d2d690c8..9558b7791 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -825,8 +825,8 @@ class Nedelec: public FieldShape {
           }
 
           // get the init ordered list with all face nodes
-          int init_order[non];
-          int final_order[4*non];
+          int* init_order = new int[non];
+          int* final_order = new int[4*non];
           int i = 0;
           for ( int r = size-1; r >= 0; r--)
             for (int c = size-r-1 ; c < size; c++) {
@@ -837,6 +837,8 @@ class Nedelec: public FieldShape {
 
           for (int i = 0; i < 4*non; i++)
             order[i] = final_order[i];
+          delete [] init_order;
+          delete [] final_order;
         }
       }
       void getVectorValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
@@ -1201,4 +1203,4 @@ void projectNedelecField(Field* to, Field* from)
   destroyField(count);
 }
 
-};
+}
diff --git a/apf/apfPolyBasis1D.cc b/apf/apfPolyBasis1D.cc
index 6f0e4b4c1..e6703fdae 100644
--- a/apf/apfPolyBasis1D.cc
+++ b/apf/apfPolyBasis1D.cc
@@ -97,7 +97,7 @@ void getGaussLegendrePoints(int np, double* pts)
 
 void getGaussLobattoPoints(int /*np*/, double* /*pts*/)
 { /* implement Gauss Lobatto points. Later when needed. */
-};
+}
 
 
 void getOpenPoints(

From ab5c82ec702392e1295c54f6f2cfb1f56d7df4d4 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 15 Jul 2020 10:32:05 -0400
Subject: [PATCH 204/555] update pumi-meshes submodule

---
 pumi-meshes | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pumi-meshes b/pumi-meshes
index 6d1601a29..94870b0ce 160000
--- a/pumi-meshes
+++ b/pumi-meshes
@@ -1 +1 @@
-Subproject commit 6d1601a29152e2043b16b2522c1a346b877920a0
+Subproject commit 94870b0ce0f9d953e985ed95c47c5b65246ae361

From 33a64f9fdb7c96ce3aff33ee41e05eba14a6517d Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 15 Jul 2020 10:24:44 -0400
Subject: [PATCH 205/555] cgns and hdf5 required, need dl lib for hdf5

---
 CMakeLists.txt | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 21880f4e1..b79259993 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -131,11 +131,9 @@ if(ENABLE_OMEGA_H)
 endif()
 
 if(ENABLE_CGNS)
-  find_package(CGNS)
+  find_package(CGNS REQUIRED)
   include_directories(SYSTEM ${CGNS_INCLUDE_DIR})
-  #
-  find_package(HDF5) 
-  #
+  find_package(HDF5 REQUIRED COMPONENTS HL C)
   add_definitions(-DHAVE_CGNS)
 endif()
 
@@ -166,6 +164,7 @@ add_library(core INTERFACE)
 target_link_libraries(core INTERFACE ${SCOREC_EXPORTED_TARGETS})
 if(ENABLE_CGNS)
   target_link_libraries(core INTERFACE ${CGNS_LIBRARIES} ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES})
+  target_link_libraries(core INTERFACE ${CMAKE_DL_LIBS}) #HDF5 uses dlopen
 endif()
 scorec_export_library(core)
 

From 28115b33856b6601df7ba785f3b8b3eb4c1c519b Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 15 Jul 2020 10:25:06 -0400
Subject: [PATCH 206/555] zoltan is required for cgns tests

---
 test/testing.cmake | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/testing.cmake b/test/testing.cmake
index 96c419933..53387d923 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -451,7 +451,7 @@ if(ENABLE_ZOLTAN)
   )
 endif()
 
-if(ENABLE_CGNS)
+if(ENABLE_CGNS AND ENABLE_ZOLTAN)
 #
 # sort of an arbitrary choice
 set(numProcs 4)
@@ -536,7 +536,7 @@ mpi_test(cgns_bcs_3 ${numProcs}
   bcs3.smb
   additional)
 
-endif(ENABLE_CGNS)
+endif(ENABLE_CGNS AND ENABLE_ZOLTAN)
 
 mpi_test(construct 4
   ./construct

From 30b1a55da37656bee2f6063c87bd3ed10c6863f5 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Thu, 16 Jul 2020 08:25:44 -0400
Subject: [PATCH 207/555] Exposes a few other apis for the python wrappers

---
 python_wrappers/apf.i | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/python_wrappers/apf.i b/python_wrappers/apf.i
index e32cc2242..7d8d7ddf8 100644
--- a/python_wrappers/apf.i
+++ b/python_wrappers/apf.i
@@ -71,6 +71,8 @@ void gmi_register_null(void);
   void gmi_register_sim(void);
   void gmi_sim_start(void);
   void gmi_sim_stop(void);
+  void gmi_sim_stop(void);
+  gmi_model* gmi_sim_load(const char* nativefile, const char* smdfile);
 #endif
 
 
@@ -171,6 +173,7 @@ void lion_set_verbosity(int lvl);
 namespace apf {
   apf::Mesh2* makeEmptyMdsMesh(gmi_model* model, int dim, bool isMatched);
   apf::Mesh2* loadMdsMesh(const char* modelfile, const char* meshfile);
+  apf::Mesh2* loadMdsMesh(gmi_model* model, const char* meshfile);
   void writeASCIIVtkFiles(const char* prefix, apf::Mesh2* m);
   /* void writeVtkFiles(const char* prefix, apf::Mesh* m, int cellDim = -1); */
   /* void writeVtkFiles(const char* prefix, apf::Mesh* m, */

From 08ba1dc5d077eb946b70df1fcb04f7ee8cf46b6a Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Thu, 16 Jul 2020 08:49:45 -0400
Subject: [PATCH 208/555] Adds is_sim_started to python wrappers

---
 python_wrappers/apf.i         | 1 +
 python_wrappers/sim_helper.cc | 5 +++++
 python_wrappers/sim_helper.h  | 3 +++
 3 files changed, 9 insertions(+)

diff --git a/python_wrappers/apf.i b/python_wrappers/apf.i
index 7d8d7ddf8..7f7f736ac 100644
--- a/python_wrappers/apf.i
+++ b/python_wrappers/apf.i
@@ -61,6 +61,7 @@ void PCU_ALWAYS_ASSERT_VERBOSE(int cond, const char* msg);
 #ifdef HAVE_SIMMETRIX
   void start_sim(const char* logfile = 0);
   void stop_sim();
+  bool is_sim_started();
 #endif
 
 /* GMI RELATED WRAPPERS */
diff --git a/python_wrappers/sim_helper.cc b/python_wrappers/sim_helper.cc
index 384c8de72..e228db922 100644
--- a/python_wrappers/sim_helper.cc
+++ b/python_wrappers/sim_helper.cc
@@ -7,6 +7,7 @@ void start_sim(const char* logfile)
   Sim_readLicenseFile(0);
   if (logfile)
     Sim_logOn(logfile);
+  is_started = true;
 }
 
 void stop_sim()
@@ -16,3 +17,7 @@ void stop_sim()
   MS_exit();
 }
 
+bool is_sim_started()
+{
+  return is_started;
+}
diff --git a/python_wrappers/sim_helper.h b/python_wrappers/sim_helper.h
index b881b5d12..1030042d5 100644
--- a/python_wrappers/sim_helper.h
+++ b/python_wrappers/sim_helper.h
@@ -6,7 +6,10 @@
 #include <MeshSim.h>
 #include <SimModel.h>
 
+static bool is_started = false;
+
 void start_sim(const char* logfile = 0);
 void stop_sim();
+bool is_sim_started();
 
 #endif

From c151e69a1a181cda6a1e9a695e873905868b025e Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 17 Jul 2020 08:49:48 -0400
Subject: [PATCH 209/555] simplify cxx flag logic

---
 CMakeLists.txt | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index b79259993..564f9ea82 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -32,16 +32,10 @@ if(NOT USE_XSDK_DEFAULTS)
   bob_begin_cxx_flags()
   bob_end_cxx_flags()
   set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS}")
-  if(SCOREC_ENABLE_CXX11)
-    if(ENABLE_CGNS)
-      bob_cxx14_flags()
-    else()
-      bob_cxx14_flags()
-    endif()
-  else()
-    if(ENABLE_CGNS)
-      bob_cxx14_flags()
-    endif()
+  if(ENABLE_CGNS) #takes precedence over SCOREC_ENABLE_CXX11
+    bob_cxx14_flags()
+  elseif(SCOREC_ENABLE_CXX11)
+    bob_cxx11_flags()
   endif()
 endif()
 message(STATUS "CMAKE_CXX_FLAGS = ${CMAKE_CXX_FLAGS}")

From 6eb7881ea50a7d942b919dbe9f2ee03cc740c1e4 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 17 Jul 2020 09:00:28 -0400
Subject: [PATCH 210/555] use bob_public_dep for cgns and hdf5

---
 CMakeLists.txt     | 8 +++++---
 apf/CMakeLists.txt | 6 ------
 2 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 564f9ea82..84aabba05 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -125,9 +125,12 @@ if(ENABLE_OMEGA_H)
 endif()
 
 if(ENABLE_CGNS)
-  find_package(CGNS REQUIRED)
+  set(SCOREC_USE_CGNS_DEFAULT ${ENABLE_CGNS})
+  bob_public_dep(CGNS)
+  #CGNS does not provide cmake targets :(
   include_directories(SYSTEM ${CGNS_INCLUDE_DIR})
-  find_package(HDF5 REQUIRED COMPONENTS HL C)
+  set(SCOREC_USE_HDF5_DEFAULT ${ENABLE_CGNS})
+  bob_public_dep(HDF5)
   add_definitions(-DHAVE_CGNS)
 endif()
 
@@ -157,7 +160,6 @@ add_subdirectory(omega_h)
 add_library(core INTERFACE)
 target_link_libraries(core INTERFACE ${SCOREC_EXPORTED_TARGETS})
 if(ENABLE_CGNS)
-  target_link_libraries(core INTERFACE ${CGNS_LIBRARIES} ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES})
   target_link_libraries(core INTERFACE ${CMAKE_DL_LIBS}) #HDF5 uses dlopen
 endif()
 scorec_export_library(core)
diff --git a/apf/CMakeLists.txt b/apf/CMakeLists.txt
index ed2bb2140..d9e0f27cf 100644
--- a/apf/CMakeLists.txt
+++ b/apf/CMakeLists.txt
@@ -98,12 +98,6 @@ target_link_libraries(apf
      mth
    )
 
-
-if(ENABLE_CGNS)
-  message(STATUS ${CGNS_LIBRARIES})
-  target_link_libraries(apf PRIVATE ${CGNS_LIBRARIES} ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES})
-endif(ENABLE_CGNS)
-
 scorec_export_library(apf)
 
 bob_end_subdir()

From abafa7d2feebf00d62132a593e439d6fd569534d Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 17 Jul 2020 10:18:55 -0400
Subject: [PATCH 211/555] fix leak

---
 test/test_integrator.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/test_integrator.cc b/test/test_integrator.cc
index 79c2faae0..682ea2eee 100644
--- a/test/test_integrator.cc
+++ b/test/test_integrator.cc
@@ -44,6 +44,7 @@ int main(int argc, char ** argv) {
     PCU_ALWAYS_ASSERT(mesh->count(i) == countInt->getCount());
   }
 
+  delete countInt;
   mesh->destroyNative();
   apf::destroyMesh(mesh);
   PCU_Comm_Free();

From 9cc3e220304b837365410992ea0065ea4d807aa6 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Mon, 20 Jul 2020 09:14:43 -0400
Subject: [PATCH 212/555] compile with simmodsuite 15.0-200714

GM_translateModel takes an additional arg
---
 cmake/FindSimModSuite.cmake | 2 +-
 test/simTranslate.cc        | 7 +++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/cmake/FindSimModSuite.cmake b/cmake/FindSimModSuite.cmake
index 190f86fb5..64ad49986 100644
--- a/cmake/FindSimModSuite.cmake
+++ b/cmake/FindSimModSuite.cmake
@@ -84,7 +84,7 @@ string(REGEX REPLACE
   "${SIM_VERSION}")
 
 set(MIN_VALID_SIM_VERSION 12.0.190225)
-set(MAX_VALID_SIM_VERSION 15.0.200110)
+set(MAX_VALID_SIM_VERSION 15.0.200714)
 if( ${SKIP_SIMMETRIX_VERSION_CHECK} )
   message(STATUS "Skipping Simmetrix SimModSuite version check."
     " This may result in undefined behavior")
diff --git a/test/simTranslate.cc b/test/simTranslate.cc
index f2425a404..9b99255e2 100644
--- a/test/simTranslate.cc
+++ b/test/simTranslate.cc
@@ -20,6 +20,8 @@
 #include <stdlib.h>
 #include <string>
 
+/* hack to get SIMMODSUITE_MAJOR_VERSION and SIMMODSUITE_MINOR_VERSION */
+#include "../apf_sim/apf_simConfig.h"
 /* cheap hackish way to get SIM_PARASOLID and SIM_ACIS */
 #include "gmi_sim_config.h"
 #include <gmi_sim.h>
@@ -96,7 +98,12 @@ void translateModel(std::string mdlName, pGModel* simmodel, pProgress& progress)
   PList_delete(modelErrors);
 
   // translate the model
+#if SIMMODSUITE_MAJOR_VERSION >= 15 && SIMMODSUITE_MINOR_VERSION >= 200714
+  const int keepAnalyticSurfaces = 1;
+  *simmodel = GM_translateModel(model, NULL, keepAnalyticSurfaces);
+#else 
   *simmodel = GM_translateModel(model, NULL);
+#endif
   GM_release(model);
 }
 

From 492ee541a305e5cd4ec5ed2e2e10090a261899ee Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Mon, 20 Jul 2020 14:23:47 -0400
Subject: [PATCH 213/555] fix leaks

there are others #315
---
 pumi-meshes                     | 2 +-
 pumi/pumi_ghost.cc              | 5 ++++-
 test/fieldReduce.cc             | 3 +++
 test/pumi.cc                    | 2 +-
 test/verify_2nd_order_shapes.cc | 2 ++
 5 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/pumi-meshes b/pumi-meshes
index e3980df41..94870b0ce 160000
--- a/pumi-meshes
+++ b/pumi-meshes
@@ -1 +1 @@
-Subproject commit e3980df41b1c9446c60190a1e3870f8fddb0e589
+Subproject commit 94870b0ce0f9d953e985ed95c47c5b65246ae361
diff --git a/pumi/pumi_ghost.cc b/pumi/pumi_ghost.cc
index af59bfd63..e8dfdacdd 100644
--- a/pumi/pumi_ghost.cc
+++ b/pumi/pumi_ghost.cc
@@ -394,7 +394,10 @@ void ghost_sendEntities(Ghosting* plan, int entDim,
 void pumi_ghost_create(pMesh m, Ghosting* plan)
 // *********************************************************
 {
-  if (PCU_Comm_Peers()==1) return;
+  if (PCU_Comm_Peers()==1) {
+    delete plan;
+    return;
+  }
  
   std::vector<apf::Field*> fields;
   std::vector<apf::Field*> frozen_fields;
diff --git a/test/fieldReduce.cc b/test/fieldReduce.cc
index ffe6095fb..b33fea701 100644
--- a/test/fieldReduce.cc
+++ b/test/fieldReduce.cc
@@ -53,6 +53,7 @@ apf::Field* getTestField(apf::Mesh* m, const char* fname, double addval)
       double val = getValue(coords, addval);
       apf::setScalar(f, e, 0, val);
     }  // end while
+    m->end(it);
   }  // end for
 
   return f;
@@ -133,8 +134,10 @@ bool testReduce(apf::Mesh* m, int casenum)
 
       }  // end if ntimes
     } // end while
+    m->end(it);
   }  // end for
 
+  delete shr;
   return failflag;
 }
 
diff --git a/test/pumi.cc b/test/pumi.cc
index c46696b70..bda9ed4a8 100644
--- a/test/pumi.cc
+++ b/test/pumi.cc
@@ -342,7 +342,7 @@ void TEST_GENT_SETGET_TAG (pGeom g, pGeomEnt ent)
 
   // pumi_gent_set/getPtrTag 
   pumi_gent_setPtrTag (ent, pointer_tag, (void*)(data));
-  void* void_data = (void*)calloc(strlen(data), sizeof(char)); //this memory is leaked
+  void* void_data; // pumi_gent_getPtrTag will point void_data at the stored address
   pumi_gent_getPtrTag (ent, pointer_tag, &void_data);
   PCU_ALWAYS_ASSERT(!strcmp((char*)void_data, data));
 
diff --git a/test/verify_2nd_order_shapes.cc b/test/verify_2nd_order_shapes.cc
index cedb5f713..bc26005eb 100644
--- a/test/verify_2nd_order_shapes.cc
+++ b/test/verify_2nd_order_shapes.cc
@@ -100,6 +100,8 @@ int main(int argc, char** argv) {
       abort();
   }
 
+  apf::destroyElement(elem);
+
   m->destroyNative();
   apf::destroyMesh(m);
 #ifdef HAVE_SIMMETRIX

From a3a017bcc4ab0bc9bdd6ccacdd0e1ab70ed68dc0 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Fri, 24 Jul 2020 09:03:30 -0400
Subject: [PATCH 214/555] Fixes memory leak issue in BezeriRefine

Mentioned in #315
---
 apf/apfMesh.cc | 8 ++++++--
 ma/maShape.cc  | 1 +
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/apf/apfMesh.cc b/apf/apfMesh.cc
index 6502443b2..02ff25118 100644
--- a/apf/apfMesh.cc
+++ b/apf/apfMesh.cc
@@ -238,12 +238,16 @@ bool Mesh::isParamPointInsideModel(ModelEntity* g,
   gmi_ent* e = (gmi_ent*)g;
   gmi_set* adjRegions = gmi_adjacent(getModel(), e, 3);
   // for 2D models return true
-  if (adjRegions->n == 0 || adjRegions->n == 2)
+  if (adjRegions->n == 0 || adjRegions->n == 2) {
+    gmi_free_set(adjRegions);
     return true;
+  }
   // for faces with more than 1 adj model region return true for now
   // TODO: update for future
-  if (adjRegions->n == 2)
+  if (adjRegions->n == 2) {
+    gmi_free_set(adjRegions);
     return true;
+  }
   PCU_ALWAYS_ASSERT(adjRegions->n <= 1);
   gmi_ent* r = (gmi_ent*)adjRegions->e[0];
   gmi_eval(getModel(), (gmi_ent*)g, &param[0], &x[0]);
diff --git a/ma/maShape.cc b/ma/maShape.cc
index 458c9f4eb..54b8936a8 100644
--- a/ma/maShape.cc
+++ b/ma/maShape.cc
@@ -652,6 +652,7 @@ class LargeAngleTriFixer : public Operator
       apf::MeshElement* me = apf::createMeshElement(mesh, tri);
       Vector center(1./3.,1./3.,1./3.);
       sf->getTransform(me,center,Q);
+      apf::destroyMeshElement(me);
 
       // pick the edge opposite to the largest angle (in metric) for swap
       Entity* edges[3];

From 59c3bacf10888c99152894e6e70ad337b08bf01f Mon Sep 17 00:00:00 2001
From: Jacob Merson <mersoj@rpi.edu>
Date: Thu, 30 Jul 2020 08:19:59 -0400
Subject: [PATCH 215/555] Add a cache variable for test meshes directory (#318)

The default of the cache variable is the path to the pumi-meshes
submodule.
---
 CMakeLists.txt | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index bb3ddd839..b28dc12ef 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -46,6 +46,9 @@ message(STATUS "CMAKE_C_FLAGS = ${CMAKE_C_FLAGS}")
 option(IS_TESTING "Build for CTest" OFF)
 message(STATUS "IS_TESTING: ${IS_TESTING}")
 
+set(MESHES "${CMAKE_SOURCE_DIR}/pumi-meshes" CACHE STRING "Directory of test meshes")
+message(STATUS "MESHES: ${MESHES}")
+
 option(BUILD_EXES "Build executables" ON)
 message(STATUS "BUILD_EXES: ${BUILD_EXES}")
 

From 46530b5875e03f2a3e1922e63d95ed8e98b680f9 Mon Sep 17 00:00:00 2001
From: Jacob Merson <mersoj@rpi.edu>
Date: Thu, 30 Jul 2020 08:20:41 -0400
Subject: [PATCH 216/555] remove memory leaks from test_matrix_grad.cc (#317)

In reference to #315
---
 test/test_matrix_grad.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test/test_matrix_grad.cc b/test/test_matrix_grad.cc
index aae3ac861..c479c464b 100644
--- a/test/test_matrix_grad.cc
+++ b/test/test_matrix_grad.cc
@@ -27,6 +27,7 @@ void setMatField(apf::Mesh* mesh, apf::Field* nodal_fld)
     apf::Matrix3x3 F(1+5*coords[0], 0, 2*coords[1], 0, 1+coords[1], 0, 0, 0, 1+3*coords[2]);
     apf::setMatrix(nodal_fld, ent, 0, F);
   }
+  mesh->end(it);
 }
 
 class MatrixDerivIntegrator : public apf::Integrator
@@ -107,9 +108,11 @@ int main(int argc, char* argv[])
   std::cout<<"Setting matrix derivatives"<<std::endl;
   apf::Integrator * set_matrix_deriv = new MatrixDerivIntegrator(order, nodal_matrix_fld, matrix_deriv);
   set_matrix_deriv->process(mesh);
+  delete set_matrix_deriv;
   std::cout<<"Checking values"<<std::endl;
   apf::Integrator * check_matrix_deriv = new CheckMatrixDerivIntegrator(order, matrix_deriv);
   check_matrix_deriv->process(mesh);
+  delete check_matrix_deriv;
   std::cout<<"Done"<<std::endl;
   PCU_Comm_Free();
   MPI_Finalize();

From 377a26deac3c03a51552f2304963e65a81b51a51 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Sat, 8 Aug 2020 14:22:35 -0400
Subject: [PATCH 217/555] Parallel implementation of error estimator. Debugging
 required.

---
 em/CMakeLists.txt           |   1 +
 em/em.h                     |   7 +-
 em/emCorrectedFlux.cc       | 329 ++++++++++++++++++++++++++++++++++++
 em/emEstimateError.cc       | 122 ++++---------
 em/emResidualFunctionals.cc |  79 ++++++++-
 5 files changed, 443 insertions(+), 95 deletions(-)
 create mode 100644 em/emCorrectedFlux.cc

diff --git a/em/CMakeLists.txt b/em/CMakeLists.txt
index 8628e8f06..4d4adeedb 100644
--- a/em/CMakeLists.txt
+++ b/em/CMakeLists.txt
@@ -7,6 +7,7 @@ endif()
 set(SOURCES
   emResidualFunctionals.cc
   emFluxCorrection.cc
+  emCorrectedFlux.cc
   emEstimateError.cc
   emSizeField.cc)
 
diff --git a/em/em.h b/em/em.h
index f5cae5d29..656e69356 100644
--- a/em/em.h
+++ b/em/em.h
@@ -47,11 +47,16 @@ apf::Field* equilibrateResiduals(apf::Field* f);
  */
 apf::Field* computeFluxCorrection(apf::Field* ef, apf::Field* g);
 
+/* Takes the solution electric field and correctiion to the flux vectors on
+ * each face and computes the 'corrected' flux vectors on each face
+ */
+apf::Field* computeCorrectedFlux(apf::Field* ef, apf::Field* theta);
+
 /* Takes the solution electric field and corrected flux field and solves
  * local element level BVPs to estimate the error.
  * Returns a per-element scalar error field.
  */
-apf::Field* computeErrorField(apf::Field* ef, apf::Field* THETA_Field);
+apf::Field* computeErrorField(apf::Field* ef, apf::Field* correctedFlux);
 
 apf::Field* estimateError(apf::Field* f);
 
diff --git a/em/emCorrectedFlux.cc b/em/emCorrectedFlux.cc
new file mode 100644
index 000000000..b196ead69
--- /dev/null
+++ b/em/emCorrectedFlux.cc
@@ -0,0 +1,329 @@
+/*
+ * Copyright (C) 2011 Scientific Computation Research Center
+ *
+ * This work is open source software, licensed under the terms of the
+ * BSD license as described in the LICENSE file in the top-level directory.
+ */
+#include <iostream>
+#include <cstdlib>
+
+#include <apfCavityOp.h>
+#include "apfElement.h"
+#include "crv.h"
+#include "crvShape.h"
+
+#include "em.h"
+using namespace std;
+namespace em {
+
+enum {VISITED};
+
+/* overall information useful during computation of corrected flux */
+struct CorrectFlux
+{
+  apf::Mesh* mesh;
+  /* mesh dimension, so far handling 3 only */
+  int dim;
+  /* polynomial order of nedelec space */
+  int order;
+  /* polynomial order of p+1 nedelec space */
+  int orderp1;
+  /* input nedelec field containing scalar dofs for electric field */
+  apf::Field* ef;
+  /* input theta field for flux correction
+   * (3 scalar dofs on each face, 1 per edge) */
+  apf::Field* theta;
+  /* tags each face once it has been visited */
+  apf::MeshTag* tag;
+  /* output per-element field containing correctedFlux vectors,
+   * which are computed using theta and electric fields */
+  apf::Field* correctedFlux;
+};
+
+static void setupCorrectFlux(
+    CorrectFlux* cf,
+    apf::Field* f,
+    apf::Field* theta,
+    apf::Field* correctedFlux)
+{
+  cf->mesh = apf::getMesh(f);
+  cf->dim = cf->mesh->getDimension();
+  cf->order = f->getShape()->getOrder();
+  cf->orderp1 = cf->order+1;
+  cf->ef = f;
+  cf->theta = theta;
+  cf->tag = cf->mesh->createIntTag("isVisited", 1);
+
+  cf->correctedFlux = correctedFlux;
+  int nc = apf::countComponents(cf->correctedFlux);
+  double zeros[nc];
+  for (int i = 0; i < nc; i++)
+    zeros[i] = 0.;
+  apf::MeshEntity* tet;
+  apf::MeshIterator* it = cf->mesh->begin(3);
+  while ((tet = cf->mesh->iterate(it))) {
+    apf::MeshElement* me = apf::createMeshElement(cf->mesh, tet);
+    int orderp1 = cf->order+1;
+    int np = apf::countIntPoints(me, 2*orderp1-1); // TODO 2*order
+
+    for (int i = 0; i < np; i++) {
+      apf::setComponents(cf->correctedFlux, tet, i, zeros);
+    }
+  }
+  cf->mesh->end(it);
+}
+
+typedef std::vector<apf::MeshEntity*> EntityVector;
+struct FaceCavity
+{
+  apf::Mesh* mesh;
+  apf::MeshEntity* entity;
+  CorrectFlux* correctflux;
+  EntityVector tets;
+};
+
+static void setupFaceCavity(FaceCavity* fc, CorrectFlux* cf)
+{
+  fc->mesh = cf->mesh;
+  fc->correctflux = cf;
+  fc->entity = 0;
+}
+
+static void startFaceCavity(FaceCavity* fc, apf::MeshEntity* f)
+{
+  fc->entity = f;
+  fc->tets.clear();
+}
+
+static void addEntityToCavity(FaceCavity* fc, apf::MeshEntity* e)
+{
+  PCU_ALWAYS_ASSERT(fc->mesh->getType(e) == apf::Mesh::TET);
+  fc->tets.push_back(e);
+}
+
+static void addEntitiesToCavity(
+    FaceCavity* fc, apf::DynamicArray<apf::MeshEntity*>& es)
+{
+  for (std::size_t i=0; i < es.getSize(); ++i)
+    addEntityToCavity(fc, es[i]);
+}
+
+static bool getInitialFaceCavity(FaceCavity* fc, apf::CavityOp* o)
+{
+  if (! o->requestLocality(&fc->entity, 1))
+    return false;
+
+  apf::DynamicArray<apf::MeshEntity*> adjacent;
+  fc->mesh->getAdjacent(fc->entity, 3, adjacent);
+  addEntitiesToCavity(fc, adjacent);
+  return true;
+}
+
+static bool buildFaceCavity(FaceCavity* fc, apf::CavityOp* o)
+{
+  if (!getInitialFaceCavity(fc, o)) return false;
+  return true;
+}
+
+
+static void computeCorrectedFlux(FaceCavity* fc)
+{
+  apf::MeshEntity* face = fc->entity;
+
+  // 1. get upward tets of the face
+  apf::Up up;
+  fc->mesh->getUp(face, up);
+  if (crv::isBoundaryEntity(fc->mesh, face))
+    PCU_ALWAYS_ASSERT(up.n == 1);
+  else
+    PCU_ALWAYS_ASSERT(up.n == 2);
+
+  apf::MeshEntity* firstTet = up.e[0];
+  apf::MeshEntity* secondTet = nullptr;
+  if (up.n == 2)
+    secondTet  = up.e[1];
+
+  // 2. get positions of the face in downward faces of upward tets
+  apf::Downward tet1_faces, tet2_faces;
+  int nf = fc->mesh->getDownward(firstTet, 2, tet1_faces);
+  if (up.n == 2)
+    fc->mesh->getDownward(secondTet, 2, tet2_faces);
+  int tet1_pos = -1;
+  int tet2_pos = -1;
+  tet1_pos = apf::findIn(tet1_faces, nf, face);
+  if (up.n == 2)
+    tet2_pos = apf::findIn(tet2_faces, nf, face);
+
+  // 3. get downward edges of the face
+  apf::Downward edges;
+  int nedges = fc->mesh->getDownward(face, 1, edges);
+
+  // 4. get theta coeffs on the face
+  double components[3];
+  apf::getComponents(fc->correctflux->theta, face, 0, components);
+  mth::Vector<double> theta_coeffs(3);
+  theta_coeffs(0) = components[0];
+  theta_coeffs(1) = components[1];
+  theta_coeffs(2) = components[2];
+
+//////
+  // 4. Evaluate and save corrected flux vector in an auxiliary field
+  int ftype = fc->mesh->getType(face);
+  PCU_ALWAYS_ASSERT(ftype == apf::Mesh::TRIANGLE);
+  int nfdofs = apf::countElementNodes(fc->correctflux->ef->getShape(), ftype);
+  apf::NewArray<apf::Vector3> vectorshapes(nfdofs);
+
+  apf::MeshElement* fme = apf::createMeshElement(fc->mesh, face);
+  apf::Element* fel = apf::createElement(fc->correctflux->ef, fme);
+  int int_order = 2*fc->correctflux->orderp1-1; // TODO 2*order
+  int np = apf::countIntPoints(fme, int_order);
+  int nc = apf::countComponents(fc->correctflux->correctedFlux);
+
+  apf::Vector3 p, tet1xi, tet2xi, curl1, curl2, curl,
+      fnormal1, fnormal2, tk, tk1, tk2, vshape;
+  for (int n = 0; n < np; n++) {
+
+    apf::getIntPoint(fme, int_order, n, p);
+
+    // evaluate theta vector using theta coeffs
+    apf::Vector3 theta_vector, theta_vector1, theta_vector2;
+    theta_vector.zero(); theta_vector1.zero(); theta_vector2.zero();
+    apf::NewArray<apf::Vector3> triVectorShapes (nfdofs);
+    apf::getVectorShapeValues(fel, p, triVectorShapes);
+    for (int i = 0; i < nedges; i++) {
+      apf::Vector3 v = triVectorShapes[i];
+      v = v * theta_coeffs[i];
+      theta_vector += v;
+    }
+
+    // orient face outward normals wrt tets and theta vectors
+    fnormal1 = computeFaceOutwardNormal(fc->mesh, firstTet, face, p);
+    fnormal2 = apf::Vector3(0.,0.,0.);
+    theta_vector1 = theta_vector;
+    if (up.n == 2) {
+      fnormal2 = computeFaceOutwardNormal(fc->mesh, secondTet, face, p);
+      theta_vector2 = theta_vector * -1.;
+    }
+
+    curl.zero();
+    // compute curl1
+    tet1xi = apf::boundaryToElementXi(fc->mesh, face, firstTet, p);
+    apf::MeshElement* me1 = apf::createMeshElement(fc->mesh, firstTet);
+    apf::Element* el1 = apf::createElement(fc->correctflux->ef, me1);
+    apf::getCurl(el1, tet1xi, curl1);
+    apf::Vector3 temp1 = apf::cross(fnormal1, curl1);
+    curl += temp1;
+    apf::destroyElement(el1);
+    apf::destroyMeshElement(me1);
+
+    // compute curl2
+    if (up.n == 2) {
+      tet2xi = apf::boundaryToElementXi(fc->mesh, face, secondTet, p);
+      apf::MeshElement* me2 = apf::createMeshElement(fc->mesh, secondTet);
+      apf::Element* el2 = apf::createElement(fc->correctflux->ef, me2);
+      apf::getCurl(el2, tet2xi, curl2);
+      apf::Vector3 temp2 = apf::cross(fnormal2, curl2);
+      curl += (temp2 * -1.);
+      curl = curl * 1./2.;
+      apf::destroyElement(el2);
+      apf::destroyMeshElement(me2);
+    }
+
+    tk = curl;
+    tk1 = tk;
+    apf::Vector3 theta_plus_tk1 = theta_vector1 + tk1;
+
+    // get and set components in the auxiliary field
+    double comp1[nc];
+    apf::getComponents(fc->correctflux->correctedFlux, firstTet, n, comp1);
+    int id1 = tet1_pos * 3;
+    comp1[id1] = theta_plus_tk1[0];
+    comp1[id1+1] = theta_plus_tk1[1];
+    comp1[id1+2] = theta_plus_tk1[2];
+    apf::setComponents(fc->correctflux->correctedFlux, firstTet, n, comp1);
+
+    if (up.n == 2) {
+      tk2 = tk * -1.;
+      apf::Vector3 theta_plus_tk2 = theta_vector2 + tk2;
+      double comp2[nc];
+      apf::getComponents(fc->correctflux->correctedFlux, secondTet, n, comp2);
+      int id2 = tet2_pos * 3;
+      comp2[id2] = theta_plus_tk2[0];
+      comp2[id2+1] = theta_plus_tk2[1];
+      comp2[id2+2] = theta_plus_tk2[2];
+      apf::setComponents(fc->correctflux->correctedFlux, secondTet, n, comp2);
+    }
+  }
+}
+
+class FaceCavityOp : public apf::CavityOp
+{
+public:
+  FaceCavityOp(CorrectFlux* cf):
+    apf::CavityOp(cf->mesh)
+  {
+    setupFaceCavity(&face_cavity, cf);
+  }
+  virtual Outcome setEntity(apf::MeshEntity* e)
+  {
+    if (face_cavity.mesh->hasTag(e, face_cavity.correctflux->tag))
+      return SKIP;
+    startFaceCavity(&face_cavity, e);
+    if ( ! buildFaceCavity(&face_cavity, this)) {
+      return REQUEST;
+    }
+    return OK;
+  }
+  virtual void apply()
+  {
+    computeCorrectedFlux(&face_cavity);
+    int n = VISITED;
+    face_cavity.mesh->setIntTag(
+        face_cavity.entity, face_cavity.correctflux->tag, &n);
+  }
+  FaceCavity face_cavity;
+};
+
+apf::Field* computeCorrectedFlux(apf::Field* ef, apf::Field* theta)
+{
+  int dim = apf::getMesh(ef)->getDimension();
+  int order = ef->getShape()->getOrder() + 1; // local BVPs require p+1
+  int int_order = 2*order-1; // TODO 2*order TODO update IPField to handle
+  int nc = 4*3; // 1 flux vector per face
+  apf::Field* correctedFlux = createPackedField(
+      apf::getMesh(ef), "correctedFlux", nc, apf::getIPShape(dim, int_order));
+
+  CorrectFlux correctflux;
+  setupCorrectFlux(&correctflux, ef, theta, correctedFlux);
+  FaceCavityOp op (&correctflux);
+  op.applyToDimension(2);
+  // TODO remove tag
+  return correctedFlux;
+
+  /* debug
+  // testing initial corrected flux field
+  cout << "testing initial field" << endl;
+    int elemNo = 0;
+    apf::MeshEntity* ent;
+    apf::MeshIterator* itr = apf::getMesh(ef)->begin(3);
+    while ((ent = apf::getMesh(ef)->iterate(itr)))
+    {
+      cout << "at tet " << elemNo++ << endl;
+      apf::MeshElement* me = apf::createMeshElement(apf::getMesh(ef), ent);
+      int np = apf::countIntPoints(me, int_order);
+      for (int i = 0; i < np; i++) {
+        cout << " at point " << i << endl;
+        double components[nc];
+        apf::getComponents(correctflux.correctedFlux, ent, i, components);
+        for (int j = 0; j < nc; j++) {
+          cout << components[j] << " ";
+        }
+        cout << endl;
+      }
+      cout << "==================================================" << endl;
+    }*/
+}
+
+
+
+}
diff --git a/em/emEstimateError.cc b/em/emEstimateError.cc
index ed859b56f..1b3e94654 100644
--- a/em/emEstimateError.cc
+++ b/em/emEstimateError.cc
@@ -7,6 +7,7 @@
 #include <iostream>
 #include <cstdlib>
 
+#include <apfCavityOp.h>
 #include "apfElement.h"
 #include "crv.h"
 #include "crvShape.h"
@@ -119,7 +120,7 @@ static void computeLambdaVector(
     apf::MeshEntity* e,
     apf::Field* f,
     apf::Field* fp1,
-    apf::Field* THETA_Field,
+    apf::Field* flux_field,
     mth::Vector<double>& lambda)
 {
   apf::FieldShape* fp1s = fp1->getShape();
@@ -129,6 +130,7 @@ static void computeLambdaVector(
   int nedofs = apf::countElementNodes(fp1s, etype);
   lambda.resize(nedofs);
 
+  int nc = apf::countComponents(flux_field);
 
   // get the downward faces of the element
   apf::Downward faces;
@@ -139,107 +141,33 @@ static void computeLambdaVector(
   for (int ii = 0; ii < nf; ii++) {
     apf::MeshEntity* face = faces[ii];
 
-    // 1. get upward tets of the current face
-    apf::Up up;
-    mesh->getUp(face, up);
-    if (crv::isBoundaryEntity(mesh, face))
-      PCU_ALWAYS_ASSERT( up.n == 1);
-    else
-      PCU_ALWAYS_ASSERT( up.n == 2);
-
-    apf::MeshEntity* firstTet = up.e[0];
-    apf::MeshEntity* secondTet = nullptr;
-    if (up.n == 2)
-      secondTet  = up.e[1];
-
-    // 2. get downward edges of the face
-    apf::Downward edges;
-    int nedges = mesh->getDownward(face, 1, edges);
-
-    // 3. get theta coeffs on the face
-    double components[3];
-    apf::getComponents(THETA_Field, face, 0, components);
-    mth::Vector<double> theta_coeffs(3);
-    theta_coeffs(0) = components[0];
-    theta_coeffs(1) = components[1];
-    theta_coeffs(2) = components[2];
-
     int ftype = mesh->getType(face);
     PCU_ALWAYS_ASSERT(ftype == apf::Mesh::TRIANGLE);
     int nfdofs = apf::countElementNodes(f->getShape(), ftype);
     apf::NewArray<apf::Vector3> vectorshape(nfdofs);
 
     apf::MeshElement* fme = apf::createMeshElement(mesh, face);
-    apf::Element* fel = apf::createElement(f, fme);
-    int np = apf::countIntPoints(fme, 2*order);
+    int np = apf::countIntPoints(fme, 2*order-1); // TODO 2*order
 
     // 4. Compute integral on the face
-    apf::Vector3 p, tet1xi, tet2xi, curl1, curl2, curl,
-      fnormal1, fnormal2, tk, vshape;
+    apf::Vector3 p;
     for (int n = 0; n < np; n++) {
 
-      apf::getIntPoint(fme, 2*order, n, p);
-      double weight = apf::getIntWeight(fme, 2*order, n);
+      apf::getIntPoint(fme, 2*order-1, n, p); // TODO 2*order
+      double weight = apf::getIntWeight(fme, 2*order-1, n); // TODO 2*order
       apf::Matrix3x3 fJ;
       apf::getJacobian(fme, p, fJ);
       double jdet = apf::getJacobianDeterminant(
           fJ, apf::getDimension(mesh, face));
 
-      // evaluate theta vector using theta coeffs
-      apf::Vector3 theta_vector;
-      theta_vector.zero();
-      apf::NewArray<apf::Vector3> triVectorShapes (nfdofs);
-      apf::getVectorShapeValues(fel, p, triVectorShapes);
-      for (int i = 0; i < nedges; i++) {
-        apf::Vector3 v = triVectorShapes[i];
-        v = v * theta_coeffs[i];
-        theta_vector += v;
-      }
-
-      // compute face outward normals wrt tets
-      if (e == firstTet) {
-        fnormal1 = computeFaceOutwardNormal(mesh, firstTet, face, p);
-        fnormal2 = apf::Vector3(0.,0.,0.);
-      }
-      else {
-        fnormal1 = computeFaceOutwardNormal(mesh, secondTet, face, p);
-        fnormal2 = apf::Vector3(0.,0.,0.);
-      }
-      if (up.n == 2) {
-        if (e == firstTet)
-          fnormal2 = computeFaceOutwardNormal(mesh, secondTet, face, p);
-        else {
-          fnormal2 = computeFaceOutwardNormal(mesh, firstTet, face, p);
-          theta_vector = theta_vector * -1.;
-        }
-      }
-
-      curl.zero();
-      // compute curl1
-      tet1xi = apf::boundaryToElementXi(mesh, face, firstTet, p);
-      apf::MeshElement* me1 = apf::createMeshElement(mesh, firstTet);
-      apf::Element* el1 = apf::createElement(f, me1);
-      apf::getCurl(el1, tet1xi, curl1);
-      apf::Vector3 temp1 = apf::cross(fnormal1, curl1);
-      curl += temp1;
-      apf::destroyElement(el1);
-      apf::destroyMeshElement(me1);
-
-      // compute curl2
-      if (up.n == 2) {
-        tet2xi = apf::boundaryToElementXi(mesh, face, secondTet, p);
-        apf::MeshElement* me2 = apf::createMeshElement(mesh, secondTet);
-        apf::Element* el2 = apf::createElement(f, me2);
-        apf::getCurl(el2, tet2xi, curl2);
-        apf::Vector3 temp2 = apf::cross(fnormal2, curl2);
-        curl += (temp2 * -1.);
-        curl = curl * 1./2.;
-        apf::destroyElement(el2);
-        apf::destroyMeshElement(me2);
-      }
-
-      // compute tk (inter-element averaged flux)
-      tk = curl;
+      // obtain corrected flux vector
+      double comp[nc];
+      apf::getComponents(flux_field, e, n, comp);
+      apf::Vector3 theta_plus_tk;
+      int index = ii*3;
+      theta_plus_tk[0] = comp[index];
+      theta_plus_tk[1] = comp[index+1];
+      theta_plus_tk[2] = comp[index+2];
 
       // compute p+1 order 3D vector shapes
       apf::NewArray<apf::Vector3> tetVectorShapes (nedofs);
@@ -252,7 +180,6 @@ static void computeLambdaVector(
       apf::destroyMeshElement(me);
 
       // compute integral
-      apf::Vector3 theta_plus_tk = theta_vector + tk;
       double w = weight * jdet;
       theta_plus_tk = theta_plus_tk * w;
 
@@ -427,7 +354,7 @@ static double computeL2Error(apf::Mesh* mesh, apf::MeshEntity* e,
 	return sqrt(error);
 }
 
-apf::Field* computeErrorField(apf::Field* ef, apf::Field* THETA_Field)
+apf::Field* computeErrorField(apf::Field* ef, apf::Field* correctedFlux)
 {
 
 	// 1. Create per-element SCALAR error field
@@ -461,7 +388,7 @@ apf::Field* computeErrorField(apf::Field* ef, apf::Field* THETA_Field)
     // 2(d). Compute Lambda Vector
     mth::Vector<double> lambda;
     computeLambdaVector(
-        apf::getMesh(ef), el, ef, efp1, THETA_Field, lambda);
+        apf::getMesh(ef), el, ef, efp1, correctedFlux, lambda);
 
     // 2(e). Assemble RHS element vector = blf - lf - lambda
     mth::Vector<double> B(blf.size());
@@ -511,17 +438,26 @@ apf::Field* estimateError(apf::Field* f)
 {
   double t0 = PCU_Time();
   apf::Field* g = em::equilibrateResiduals(f);
-  apf::Field* THETA = em::computeFluxCorrection(f, g);
+  lion_eprint(1,"1/4: residuals equilibrated \n");
+  apf::Field* theta = em::computeFluxCorrection(f, g);
+  lion_eprint(1,"2/4: flux corrections computed \n");
   apf::destroyField(g);
+  PCU_Barrier(); // TODO remove
+
+  apf::Field* correctedFlux = em::computeCorrectedFlux(f, theta);
+  lion_eprint(1,"3/4: corrected flux field computed\n");
+  apf::destroyField(theta);
 
-  apf::Field* error_field = em::computeErrorField(f, THETA);
-  apf::destroyField(THETA);
+  apf::Field* error_field = em::computeErrorField(f, correctedFlux);
+  lion_eprint(1,"4/4: error computed \n");
+  apf::destroyField(correctedFlux);
 
   double t1 = PCU_Time();
   if (!PCU_Comm_Self())
     lion_eprint(1,"EM: Error estimated in %f seconds\n",t1-t0);
 
   return error_field;
+  //return correctedFlux;
 }
 
 }
diff --git a/em/emResidualFunctionals.cc b/em/emResidualFunctionals.cc
index c0cd637e3..31a6c39aa 100644
--- a/em/emResidualFunctionals.cc
+++ b/em/emResidualFunctionals.cc
@@ -19,6 +19,8 @@ using namespace std;
 
 namespace em {
 
+enum {VISITED};
+
 /* overall information useful during equilibration */
 struct Equilibration {
   apf::Mesh* mesh;
@@ -28,6 +30,8 @@ struct Equilibration {
   int order;
   /* input scalar field containing Nedelec dofs for solution electric field */
   apf::Field* ef;
+  /* tags each edge once it has been visited during the equilibration process */
+  apf::MeshTag* tag;
   /* output field containing correction values.
    * currently 3 scalar values are stored on each face
    * in order corresponding to downward edges of the face */
@@ -37,6 +41,7 @@ struct Equilibration {
 static void setupEquilibration(Equilibration* eq, apf::Field* f, apf::Field* g)
 {
   eq->mesh = apf::getMesh(f);
+  eq->tag = eq->mesh->createIntTag("isVisited", 1);
   eq->dim = eq->mesh->getDimension();
   eq->ef = f;
   eq->order = f->getShape()->getOrder();
@@ -306,6 +311,8 @@ static double getLocalEdgeBLF(EdgePatch* ep, apf::MeshEntity* tet)
   apf::Downward e;
   int ne = ep->mesh->getDownward(tet, 1, e);
   int ei = apf::findIn(e, ne, ep->entity);
+  if (PCU_Comm_Self() == 0)
+    cout << "ei " << ei << endl;
   // get Element Dofs
   apf::MeshElement* me = apf::createMeshElement(ep->mesh, tet);
   apf::Element* el = apf::createElement(ep->equilibration->ef, me);
@@ -655,6 +662,13 @@ static void assembleEdgePatchRHS(EdgePatch* p)
     double blfIntegral = getLocalEdgeBLF(p, tet);
     double lfIntegral = getLocalEdgeLF(p, tet);
     double fluxIntegral = getLocalFluxIntegral(p, tet);
+    if (PCU_Comm_Self() == 0) {
+      apf::Vector3 center = apf::getLinearCentroid(p->mesh, tet);
+      cout << center << endl;
+      cout << "bilinear integral " << blfIntegral << endl;
+      cout << "linear integral " << lfIntegral << endl;
+      cout << "flux integral " << fluxIntegral << endl;
+    }
     if(p->isOnBdry)
       p->b(nf+i) = blfIntegral - lfIntegral - fluxIntegral;
     else
@@ -891,6 +905,56 @@ static void runErm(EdgePatch* ep)
     components[ei] = ep->x(i);
     apf::setComponents(ep->equilibration->g, face, 0, components);
   }
+  // debug
+  if (PCU_Comm_Self() == 0) {
+    cout << "Part " << PCU_Comm_Self() << ": ";
+    apf::Vector3 center = apf::getLinearCentroid(ep->mesh, ep->entity);
+    cout << center << endl;
+
+    cout << "LHS: ";
+    for (size_t i = 0; i < ep->T.rows(); i++) {
+      for (size_t j = 0; j < ep->T.cols(); j++) {
+        cout << ep->T(i,j) << " ";
+      }
+      cout << endl;
+    }
+
+    cout << "RHS: ";
+    for (size_t i = 0; i < ep->b.size(); i++)
+      cout << ep->b[i] << " ";
+    cout << endl;
+
+    cout << "gs: ";
+    for (size_t i = 0; i < ep->x.size(); i++)
+      cout << ep->x[i] << " ";
+    cout << endl;
+
+    cout << "Ordered tets" << endl;
+    for (size_t i = 0; i < ep->tets.size(); i++) {
+      apf::Vector3 center = apf::getLinearCentroid(ep->mesh, ep->tets[i]);
+      cout << center << endl;
+    }
+    cout << "Ordered faces" << endl;
+    for (size_t i = 0; i < ep->faces.size(); i++) {
+      apf::Vector3 center = apf::getLinearCentroid(ep->mesh, ep->faces[i]);
+      cout << center << endl;
+    }
+
+
+    cout << "==================" << endl;
+
+
+  }
+  /*if (PCU_Comm_Self() == 1) {
+    cout << "Part " << PCU_Comm_Self() << ": ";
+    apf::Vector3 center = apf::getLinearCentroid(ep->mesh, ep->entity);
+    cout << center << endl;
+
+    cout << "RHS: ";
+    for (size_t i = 0; i < ep->faces.size(); i++)
+      cout << ep->b[i] << " ";
+    cout << endl;
+  }*/
 }
 
 
@@ -904,15 +968,19 @@ class EdgePatchOp : public apf::CavityOp
   }
   virtual Outcome setEntity(apf::MeshEntity* e)
   {
+    if (edgePatch.mesh->hasTag(e, edgePatch.equilibration->tag))
+      return SKIP;
     startEdgePatch(&edgePatch, e);
     if ( ! buildEdgePatch(&edgePatch, this))
       return REQUEST;
     return OK;
-    return SKIP;
   }
   virtual void apply()
   {
     runErm(&edgePatch);
+    int n = VISITED;
+    edgePatch.mesh->setIntTag(
+        edgePatch.entity, edgePatch.equilibration->tag, &n);
   }
   EdgePatch edgePatch;
 };
@@ -925,6 +993,15 @@ apf::Field* equilibrateResiduals(apf::Field* f)
   setupEquilibration(&equilibration, f, g);
   EdgePatchOp op(&equilibration);
   op.applyToDimension(1); // edges
+
+  apf::MeshEntity* ent;
+  apf::MeshIterator* it = apf::getMesh(f)->begin(1);
+  while ((ent = apf::getMesh(f)->iterate(it))) {
+    apf::getMesh(f)->removeTag(ent, equilibration.tag);
+  }
+  apf::getMesh(f)->end(it);
+  apf::getMesh(f)->destroyTag(equilibration.tag);
+
   return g;
 }
 

From de21c9e6a53053af16635fa0c8916159c510b154 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Thu, 13 Aug 2020 13:45:48 -0400
Subject: [PATCH 218/555] Adds a new api to the python wrappers

---
 python_wrappers/apf.i | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/python_wrappers/apf.i b/python_wrappers/apf.i
index 7f7f736ac..2e2d77031 100644
--- a/python_wrappers/apf.i
+++ b/python_wrappers/apf.i
@@ -162,7 +162,25 @@ void lion_set_verbosity(int lvl);
     }
     return sum/count;
   }
+  bool isBoundingModelRegion(int rtag, int dim, int tag)
+  {
+    if (dim != 2) return false;
+    gmi_model* gmodel = self->getModel();
+    gmi_ent* gregion = gmi_find(gmodel, 3, rtag);
+    gmi_set* adj = gmi_adjacent(gmodel, gregion, dim);
+
+    for(int i = 0; i < adj->n; i++) {
+      int adj_g_tag = gmi_tag(gmodel, adj->e[i]);
+      if (adj_g_tag == tag) {
+        gmi_free_set(adj);
+        return true;
+      }
+    }
+    gmi_free_set(adj);
+    return false;
+  }
 }
+
 #define __attribute__(x)
 %ignore apf::fail;
 %include<apf.h>

From 25154654e6e814909473821028339fdc5ed2536f Mon Sep 17 00:00:00 2001
From: Morteza HS <hakimi1364@gmail.com>
Date: Sun, 20 Sep 2020 20:43:17 -0400
Subject: [PATCH 219/555] Initial commit of H1 shapes

---
 apf/apfH1Shapes.cc | 598 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 598 insertions(+)
 create mode 100644 apf/apfH1Shapes.cc

diff --git a/apf/apfH1Shapes.cc b/apf/apfH1Shapes.cc
new file mode 100644
index 000000000..4b0f0520b
--- /dev/null
+++ b/apf/apfH1Shapes.cc
@@ -0,0 +1,598 @@
+/*
+ * Copyright 2011 Scientific Computation Research Center
+ *
+ * This work is open source software, licensed under the terms of the
+ * BSD license as described in the LICENSE file in the top-level directory.
+ */
+
+#include "apfShape.h"
+#include "apfMesh.h"
+#include "apfFieldOf.h"
+#include "apfElement.h"
+#include "apfVectorElement.h"
+#include "apfPolyBasis1D.h"
+#include <mth.h>
+#include <mth_def.h>
+#include <mthQR.h>
+#include <pcu_util.h>
+#include <PCU.h>
+
+#include <iostream>
+using namespace std;
+
+namespace apf {
+
+static unsigned const MAX_ND_ORDER = 10;
+
+static inline int countTriNodes(int P)
+{
+  return (P+1)*(P+2)/2;
+}
+
+static inline int countTetNodes(int P)
+{
+  return (P+1)*(P+2)*(P+3)/6;
+}
+
+static void computeTriangleTi(
+    int P, /*order*/
+    mth::Matrix<double>& Q, /*Q in QR factorization of Ti*/
+    mth::Matrix<double>& R) /*R in QR factorization of Ti*/
+{
+  int non = countTriNodes(P);
+
+  apf::NewArray<double> cp;
+  getClosedPoints(P, cp);
+
+
+  const int p = P;
+  apf::NewArray<double> shape_x(p+1);
+  apf::NewArray<double> shape_y(p+1);
+  apf::NewArray<double> shape_l(p+1);
+
+  apf::DynamicArray<apf::Vector3> nodes (non);
+
+  int o = 0;
+
+  // vertices
+  nodes[o][0] = cp[0]; nodes[o][1] = cp[0]; nodes[o][0] = 0.;
+  o++;
+  nodes[o][0] = cp[p]; nodes[o][1] = cp[0]; nodes[o][0] = 0.;
+  o++;
+  nodes[o][0] = cp[0]; nodes[o][1] = cp[p]; nodes[o][0] = 0.;
+  o++;
+
+  // edges
+  for (int i = 1; i < p; i++) // (0,1)
+  {
+    nodes[o][0] = cp[i];  nodes[o][1] = cp[0];  nodes[o][2] = 0.;
+    o++;
+  }
+
+  for (int i = 1; i < p; i++) // (1,2)
+  {
+    nodes[o][0] = cp[p-i];  nodes[o][1] = cp[i];  nodes[o][2] = 0.;
+    o++;
+  }
+
+  for (int i = 1; i < p; i++) // (2,0)
+  {
+    nodes[o][0] = cp[0];  nodes[o][1] = cp[p-i];  nodes[o][2] = 0.;
+    o++;
+  }
+
+
+
+  for (int j = 0; j <= p; j++) {
+    for (int i = 0; i + j <= p; i++)
+      {
+        double w = op[i] + op[j] + op[p-i-j];
+        nodes[o][0] = op[i]/w;  nodes[o][1] = op[j]/w;  nodes[o][2] = 0.;
+        o++;
+      }
+  }
+
+  // Populate T
+  mth::Matrix<double> T(non,non);
+  for (int m = 0; m < non; m++)
+  {
+    o = 0;
+
+    double x = nodes[m][0]; double y = nodes[m][1];
+
+    getChebyshevT(p, x, &shape_x[0]);
+    getChebyshevT(p, y, &shape_y[0]);
+    getChebyshevT(p, 1. - x - y, &shape_l[0]);
+
+    for (int j = 0; j <= p; j++)
+      for (int i = 0; i + j <= p; i++)
+      	T(o++, m) = shape_x[i]*shape_y[j]*shape_l[p-i-j];
+  }
+  mth::decomposeQR(T, Q, R);
+}
+
+static void computeTetTi(
+    int P, /*order*/
+    mth::Matrix<double>& Q, /*Q in QR factorization of Ti*/
+    mth::Matrix<double>& R) /*R in QR factorization of Ti*/
+{
+  int non = countTetNodes(P);
+
+  apf::NewArray<double> op;
+  getOpenPoints(P, op);
+
+  const int p = P;
+
+  apf::NewArray<double> shape_x(p+1);
+  apf::NewArray<double> shape_y(p+1);
+  apf::NewArray<double> shape_z(p+1);
+  apf::NewArray<double> shape_l(p+1);
+
+  apf::DynamicArray<apf::Vector3> nodes (non);
+  nodes.setSize(non);
+
+  int o = 0;
+  // Region loops to get nodes and dof2tk for regions
+  for (int k = 0; k <= p; k++) {
+    for (int j = 0; j + k <= p; j++) {
+      for (int i = 0; i + j + k <= p; i++) {
+        double w = op[i] + op[j] + op[k] + op[p-i-j-k];
+        nodes[o][0] = op[i]/w;  nodes[o][1] = op[j]/w;  nodes[o][2] = op[k]/w;
+        o++;
+      }
+    }
+  }
+
+  // Populate T
+  mth::Matrix<double> T(non,non);
+  for (int m = 0; m < non; m++)
+  {
+    o = 0;
+    double x = nodes[m][0]; double y = nodes[m][1]; double z = nodes[m][2];
+
+    getChebyshevT(p, x, &shape_x[0]);
+    getChebyshevT(p, y, &shape_y[0]);
+    getChebyshevT(p, z, &shape_z[0]);
+    getChebyshevT(p, 1. - x - y - z, &shape_l[0]);
+
+    for (int k = 0; k <= p; k++)
+      for (int j = 0; j + k <= p; j++)
+        for (int i = 0; i + j + k <= p; i++)
+          T(o++, m) = shape_x[i]*shape_y[j]*shape_z[k]*shape_l[p-i-j-k];
+	}
+  mth::decomposeQR(T, Q, R);
+}
+
+static void getTi(
+    int P,
+    int type,
+    mth::Matrix<double>& Q,
+    mth::Matrix<double>& R)
+{
+
+  bool cond = (type == apf::Mesh::TRIANGLE || type == apf::Mesh::TET);
+  PCU_ALWAYS_ASSERT_VERBOSE(cond,
+      "type should be either apf::Mesh::TRIANGLE or apf::Mesh::TET!");
+
+  static apf::NewArray<double> transformQ[apf::Mesh::TYPES][MAX_ND_ORDER+1];
+  static apf::NewArray<double> transformR[apf::Mesh::TYPES][MAX_ND_ORDER+1];
+  int n = type == apf::Mesh::TRIANGLE ? countTriNodes(P) : countTetNodes(P);
+
+  // get the transform matrices if the are not already computed
+  if (!transformQ[type][P].allocated()) {
+    mth::Matrix<double> LQ(n,n);
+    mth::Matrix<double> LR(n,n);
+    type == apf::Mesh::TRIANGLE ?
+    	  computeTriangleTi(P, LQ, LR) : computeTetTi(P, LQ, LR);
+
+    transformQ[type][P].allocate(n*n);
+    transformR[type][P].allocate(n*n);
+
+    for (int i = 0; i < n; i++) {
+      for (int j = 0; j < n; j++) {
+	      transformQ[type][P][i*n+j] = LQ(i,j);
+      	transformR[type][P][i*n+j] = LR(i,j);
+      }
+    }
+  }
+
+  // set Q and R using transformQ and transformR
+  for (int i = 0; i < n; i++) {
+    for (int j = 0; j < n; j++) {
+      Q(i,j) = transformQ[type][P][i*n+j];
+      R(i,j) = transformR[type][P][i*n+j];
+    }
+  }
+}
+
+template<int P>
+class L2ShapeTri: public FieldShape {
+  public:
+    L2ShapeTri()
+    {
+      std::stringstream ss;
+      ss << "L2ShapeTri" << P;
+      name = ss.str();
+      registerSelf(name.c_str());
+    }
+    const char* getName() const { return name.c_str(); }
+    bool isVectorShape() {return false;}
+    class Triangle : public apf::EntityShape
+    {
+    public:
+      int getOrder() {return P;}
+      void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const& xi, apf::NewArray<double>& shapes) const
+      {
+        const int p = P;
+
+      	apf::NewArray<double> shape_x(p+1);
+      	apf::NewArray<double> shape_y(p+1);
+      	apf::NewArray<double> shape_l(p+1);
+
+        int dof = countNodes();
+        mth::Matrix<double> u(dof, 1);
+
+        double x = xi[0]; double y = xi[1];
+
+        getChebyshevT(p, x, &shape_x[0]);
+        getChebyshevT(p, y, &shape_y[0]);
+        getChebyshevT(p, 1. - x - y, &shape_l[0]);
+
+        int n = 0;
+        for (int j = 0; j <= p; j++)
+          for (int i = 0; i + j <= p; i++)
+            u(n++, 0) = shape_x[i]*shape_y[j]*shape_l[p-i-j];
+
+	      mth::Matrix<double> Q(dof, dof);
+      	mth::Matrix<double> R(dof, dof);
+      	getTi(P, apf::Mesh::TRIANGLE, Q, R);
+
+        mth::Matrix<double> S(dof, 1);
+      	for(int i = 0; i < 1; i++) // S = Ti * u
+        {
+          mth::Vector<double> B (dof);
+          mth::Vector<double> X (dof);
+          for (int j = 0; j < dof; j++) B[j] = u(j,i); // populate b in QR x = b
+          mth::solveFromQR(Q, R, B, X);
+          for (int j = 0; j < dof; j++)  S(j,i) = X[j]; // populate S with x
+        }
+
+        shapes.allocate(dof);
+        for (int i = 0; i < dof; i++) // populate y
+        {
+      	  shapes[i] = S(i,0);
+        }
+      }
+      void getLocalGradients(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
+      {
+      	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getLocalGradients not \
+      	    implemented for L2ShapeTri. Aborting()!");
+      }
+      int countNodes() const {return countTriNodes(P);}
+      void getVectorValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
+      {
+      	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getVectorValues not implemented \
+      	    for L2ShapeTri. Try getValues. Aborting()!");
+      }
+    };
+    EntityShape* getEntityShape(int type)
+    {
+    	PCU_ALWAYS_ASSERT_VERBOSE(type == Mesh::TRIANGLE,
+    			"L2ShapeTri only has entity shapes for TRIANGLEs");
+      static Triangle tri;
+      return &tri;
+    }
+    // For the following to member functions we only need to
+    // consider the interior nodes, i.e.,
+    // Faces: no need to count the nodes associated with bounding edges
+    // Tets: no need to count the nodes associated with bounding edges/faces
+    bool hasNodesIn(int dimension)
+    {
+      if (dimension == Mesh::typeDimension[Mesh::TRIANGLE])
+      	return true;
+      return false;
+    }
+    int countNodesOn(int type)
+    {
+      if (type == apf::Mesh::TRIANGLE) return countTriNodes(P);
+      return 0;
+    }
+    int getOrder() {return P;}
+    void getNodeXi(int type, int node, Vector3& xi)
+    {
+      PCU_ALWAYS_ASSERT_VERBOSE(type == Mesh::TRIANGLE,
+      	  "getNodeXi for L2ShapeTri can be called only for TRIANGLEs");
+      apf::NewArray<double> op;
+      getOpenPoints(P, op);
+      int c = 0;
+      for (int j = 0; j <= P; j++) {
+      	for (int i = 0; i + j <= P; i++) {
+      	  if (node == c) {
+      	    double w = op[i] + op[j] + op[P-i-j];
+      	    xi = Vector3( op[i]/w, op[j]/w, 0. );
+      	    return;
+      	  }
+      	  else
+      	    c++;
+      	}
+      }
+    }
+  private:
+    std::string name;
+};
+
+template<int P>
+class L2ShapeTet: public FieldShape {
+  public:
+    L2ShapeTet()
+    {
+      std::stringstream ss;
+      ss << "L2ShapeTet_" << P;
+      name = ss.str();
+      registerSelf(name.c_str());
+    }
+    const char* getName() const { return name.c_str(); }
+    bool isVectorShape() {return false;}
+    class Tetrahedron : public apf::EntityShape
+    {
+    public:
+      int getOrder() {return P;}
+      void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const& xi, apf::NewArray<double>& shapes) const
+      {
+        const int p = P;
+
+      	apf::NewArray<double> shape_x(p+1);
+      	apf::NewArray<double> shape_y(p+1);
+      	apf::NewArray<double> shape_z(p+1);
+      	apf::NewArray<double> shape_l(p+1);
+
+        int dof = countNodes();
+        mth::Matrix<double> u(dof, 1);
+
+        double x = xi[0]; double y = xi[1]; double z = xi[2];
+
+        getChebyshevT(p, x, &shape_x[0]);
+        getChebyshevT(p, y, &shape_y[0]);
+        getChebyshevT(p, z, &shape_z[0]);
+        getChebyshevT(p, 1. - x - y - z, &shape_l[0]);
+
+        int n = 0;
+        for (int k = 0; k <= p; k++)
+          for (int j = 0; j + k <= p; j++)
+            for (int i = 0; i + j + k <= p; i++)
+            	u(n++, 0) = shape_x[i]*shape_y[j]*shape_z[k]*shape_l[p-i-j-k];
+
+
+      	mth::Matrix<double> Q(dof, dof);
+      	mth::Matrix<double> R(dof, dof);
+      	getTi(P, apf::Mesh::TET, Q, R);
+
+        mth::Matrix<double> S(dof, 1);
+      	for(int i = 0; i < 1; i++) // S = Ti * u
+        {
+          mth::Vector<double> B (dof);
+          mth::Vector<double> X (dof);
+          for (int j = 0; j < dof; j++) B[j] = u(j,i); // populate b
+          mth::solveFromQR(Q, R, B, X);
+          for (int j = 0; j < dof; j++)  S(j,i) = X[j]; // populate S with x
+        }
+
+        shapes.allocate(dof);
+        for (int i = 0; i < dof; i++) // populate y
+        {
+      	  shapes[i] = S(i,0);
+        }
+      }
+      void getLocalGradients(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
+      {
+      	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getLocalGradients not \
+      	    implemented for L2ShapeTet. Aborting()!");
+      }
+      int countNodes() const {return countTetNodes(P);}
+      void getVectorValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const& /*xi*/, apf::NewArray<apf::Vector3>& /*shapes*/) const
+      {
+      	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getVectorValues not implemented for \
+      	    L2ShapeTet. Try getValues. Aborting()!");
+      }
+    };
+    EntityShape* getEntityShape(int type)
+    {
+      PCU_ALWAYS_ASSERT_VERBOSE(type == Mesh::TET,
+      	  "L2ShapeTet only has entity shapes for TETs");
+      static Tetrahedron tet;
+      return &tet;
+    }
+    // For the following to member functions we only need to
+    // consider the interior nodes, i.e.,
+    // Faces: no need to count the nodes associated with bounding edges
+    // Tets: no need to count the nodes associated with bounding edges/faces
+    bool hasNodesIn(int dimension)
+    { 
+    	if (dimension == Mesh::typeDimension[Mesh::TET])
+    		return true;
+      return false;
+    }
+    int countNodesOn(int type)
+    {
+      if (type == apf::Mesh::TET) return countTetNodes(P);
+      return 0;
+    }
+    int getOrder() {return P;}
+    void getNodeXi(int type, int node, Vector3& xi)
+    {
+      PCU_ALWAYS_ASSERT_VERBOSE(type == Mesh::TET,
+      	  "getNodeXi for L2ShapeTet can be called only for TETs");
+      apf::NewArray<double> op;
+      getOpenPoints(P, op);
+      int c = 0;
+      for (int k = 0; k <= P; k++) {
+      	for (int j = 0; j + k <= P; j++) {
+      	  for (int i = 0; i + j + k <= P; i++) {
+      	    if( node == c) {
+      	      double w = op[i] + op[j] + op[k] + op[P-i-j-k];
+      	      xi = Vector3( op[i]/w, op[j]/w,  op[k]/w );
+      	      return;
+      	    }
+      	    else
+      	      c++;
+      	  }
+      	}
+      }
+    }
+  private:
+    std::string name;
+};
+
+
+static apf::FieldShape* getL2ShapeTri(int order)
+{
+  PCU_ALWAYS_ASSERT_VERBOSE(order >= 0,
+      "order is expected to be bigger than or equal to 0!");
+  PCU_ALWAYS_ASSERT_VERBOSE(order <= 10,
+      "order is expected to be less than or equal to 10!");
+  static L2ShapeTri<0>  l2_0;
+  static L2ShapeTri<1>  l2_1;
+  static L2ShapeTri<2>  l2_2;
+  static L2ShapeTri<3>  l2_3;
+  static L2ShapeTri<4>  l2_4;
+  static L2ShapeTri<5>  l2_5;
+  static L2ShapeTri<6>  l2_6;
+  static L2ShapeTri<7>  l2_7;
+  static L2ShapeTri<8>  l2_8;
+  static L2ShapeTri<9>  l2_9;
+  static L2ShapeTri<10> l2_10;
+  static FieldShape* const l2Shapes[11] = {&l2_0,
+                                           &l2_1,
+                                           &l2_2,
+                                           &l2_3,
+                                           &l2_4,
+                                           &l2_5,
+                                           &l2_6,
+                                           &l2_7,
+                                           &l2_8,
+                                           &l2_9,
+                                           &l2_10};
+  return l2Shapes[order];
+}
+
+static apf::FieldShape* getL2ShapeTet(int order)
+{
+  PCU_ALWAYS_ASSERT_VERBOSE(order >= 0,
+      "order is expected to be bigger than or equal to 0!");
+  PCU_ALWAYS_ASSERT_VERBOSE(order <= 10,
+      "order is expected to be less than or equal to 10!");
+  static L2ShapeTet<0>  l2_0;
+  static L2ShapeTet<1>  l2_1;
+  static L2ShapeTet<2>  l2_2;
+  static L2ShapeTet<3>  l2_3;
+  static L2ShapeTet<4>  l2_4;
+  static L2ShapeTet<5>  l2_5;
+  static L2ShapeTet<6>  l2_6;
+  static L2ShapeTet<7>  l2_7;
+  static L2ShapeTet<8>  l2_8;
+  static L2ShapeTet<9>  l2_9;
+  static L2ShapeTet<10> l2_10;
+  static FieldShape* const l2Shapes[11] = {&l2_0,
+                                           &l2_1,
+                                           &l2_2,
+                                           &l2_3,
+                                           &l2_4,
+                                           &l2_5,
+                                           &l2_6,
+                                           &l2_7,
+                                           &l2_8,
+                                           &l2_9,
+                                           &l2_10};
+  return l2Shapes[order];
+}
+
+
+apf::FieldShape* getL2Shape(int order, int type)
+{
+  if (type == Mesh::TRIANGLE)
+    return getL2ShapeTri(order);
+  else if (type == Mesh::TET)
+    return getL2ShapeTet(order);
+  else
+    PCU_ALWAYS_ASSERT_VERBOSE(0,
+    	"L2Shapes are only implemented for tris and tets");
+}
+
+void projectL2Field(Field* to, Field* from)
+{
+  // checks on the from field
+  // checks on the to field
+  apf::FieldShape* tShape = getShape(to);
+  std::string      tName  = tShape->getName();
+  int              tOrder = tShape->getOrder();
+  PCU_ALWAYS_ASSERT_VERBOSE((tName == std::string("Linear")) && (tOrder == 1),
+                "The to field needs to be 1st order Lagrange!");
+
+  Mesh* m = getMesh(from);
+  // auxiliary count fields
+  Field* count = createField(m, "counter", SCALAR, getLagrange(1));
+  double xis[4][3] = {{0., 0., 0.},
+                      {1., 0., 0.},
+                      {0., 1., 0.},
+                      {0., 0., 1.}};
+  // zero out the fields
+  zeroField(to);
+  zeroField(count);
+
+  int nc = countComponents(to);
+  NewArray<double> atXi(nc);
+  NewArray<double> currentVal(nc);
+  NewArray<double> sum(nc);
+
+  MeshEntity* e;
+  MeshIterator* it = m->begin(m->getDimension());
+  while( (e = m->iterate(it)) ) {
+    MeshElement* me = createMeshElement(m, e);
+    Element* el = createElement(from, me);
+    MeshEntity* dvs[4];
+    m->getDownward(e, 0, dvs);
+    for (int i=0; i<4; i++) {
+      getComponents(el, Vector3(xis[i]), &(atXi[0]));
+      getComponents(to, dvs[i], 0, &(currentVal[0]));
+      for (int j = 0; j < nc; j++) {
+        currentVal[j] += atXi[j];
+      }
+      double currentCount = getScalar(count, dvs[i], 0);
+      currentCount += 1.;
+      setComponents(to, dvs[i], 0, &(currentVal[0]));
+      setScalar(count, dvs[i], 0, currentCount);
+    }
+    destroyElement(el);
+    destroyMeshElement(me);
+  }
+  m->end(it);
+
+  // take care of entities on part boundary
+  accumulate(to);
+  accumulate(count);
+
+  it = m->begin(0);
+  while( (e = m->iterate(it)) ) {
+    getComponents(to, e, 0, &(sum[0]));
+    int cnt = getScalar(count, e, 0);
+    for (int i = 0; i < nc; i++) {
+      sum[i] /= cnt;
+    }
+    setComponents(to, e, 0, &(sum[0]));
+  }
+  m->end(it);
+
+  // take care of entities on part boundary
+  synchronize(to);
+
+  m->removeField(count);
+  destroyField(count);
+}
+
+}

From 38fe4e60ef3a173d0a94f5de62f315c2884db3d3 Mon Sep 17 00:00:00 2001
From: Morteza HS <hakimi1364@gmail.com>
Date: Sat, 26 Sep 2020 18:10:45 -0400
Subject: [PATCH 220/555] Adds code for the H1 Shapes

---
 apf/apfH1Shapes.cc | 137 ++++++++++++++++++++++++++++++++++++---------
 1 file changed, 111 insertions(+), 26 deletions(-)

diff --git a/apf/apfH1Shapes.cc b/apf/apfH1Shapes.cc
index 4b0f0520b..6ef7269d3 100644
--- a/apf/apfH1Shapes.cc
+++ b/apf/apfH1Shapes.cc
@@ -82,12 +82,12 @@ static void computeTriangleTi(
   }
 
 
-
-  for (int j = 0; j <= p; j++) {
-    for (int i = 0; i + j <= p; i++)
+  // face
+  for (int j = 1; j < p; j++) {
+    for (int i = 1; i + j < p; i++)
       {
-        double w = op[i] + op[j] + op[p-i-j];
-        nodes[o][0] = op[i]/w;  nodes[o][1] = op[j]/w;  nodes[o][2] = 0.;
+        double w = cp[i] + cp[j] + cp[p-i-j];
+        nodes[o][0] = cp[i]/w;  nodes[o][1] = cp[j]/w;  nodes[o][2] = 0.;
         o++;
       }
   }
@@ -118,8 +118,8 @@ static void computeTetTi(
 {
   int non = countTetNodes(P);
 
-  apf::NewArray<double> op;
-  getOpenPoints(P, op);
+  apf::NewArray<double> cp;
+  getClosedPoints(P, cp);
 
   const int p = P;
 
@@ -131,13 +131,94 @@ static void computeTetTi(
   apf::DynamicArray<apf::Vector3> nodes (non);
   nodes.setSize(non);
 
-  int o = 0;
-  // Region loops to get nodes and dof2tk for regions
-  for (int k = 0; k <= p; k++) {
-    for (int j = 0; j + k <= p; j++) {
-      for (int i = 0; i + j + k <= p; i++) {
-        double w = op[i] + op[j] + op[k] + op[p-i-j-k];
-        nodes[o][0] = op[i]/w;  nodes[o][1] = op[j]/w;  nodes[o][2] = op[k]/w;
+
+  // vertices
+  nodes[o][0] = cp[0]; nodes[o][1] = cp[0]; nodes[o][0] = cp[0];
+  o++;
+  nodes[o][0] = cp[p]; nodes[o][1] = cp[0]; nodes[o][0] = cp[0];
+  o++;
+  nodes[o][0] = cp[0]; nodes[o][1] = cp[p]; nodes[o][0] = cp[0];
+  o++;
+  nodes[o][0] = cp[0]; nodes[o][1] = cp[0]; nodes[o][0] = cp[p];
+  o++;
+
+  // edges
+  for (int i = 1; i < p; i++) // (0,1)
+  {
+    nodes[o][0] = cp[i];  nodes[o][1] = cp[0];  nodes[o][2] = cp[0.];
+    o++;
+  }
+
+  for (int i = 1; i < p; i++) // (1,2)
+  {
+    nodes[o][0] = cp[p-i];  nodes[o][1] = cp[i];  nodes[o][2] = cp[0.];
+    o++;
+  }
+
+  for (int i = 1; i < p; i++) // (2,0)
+  {
+    nodes[o][0] = cp[0];  nodes[o][1] = cp[p-i];  nodes[o][2] = cp[0.];
+    o++;
+  }
+
+  for (int i = 1; i < p; i++) // (0,3)
+  {
+    nodes[o][0] = cp[0];  nodes[o][1] = cp[0];  nodes[o][2] = cp[i];
+    o++;
+  }
+
+  for (int i = 1; i < p; i++) // (1,3)
+  {
+    nodes[o][0] = cp[p-i];  nodes[o][1] = cp[0];  nodes[o][2] = cp[i];
+    o++;
+  }
+
+  for (int i = 1; i < p; i++) // (2,3)
+  {
+    nodes[o][0] = cp[0];  nodes[o][1] = cp[p-i];  nodes[o][2] = cp[i];
+    o++;
+  }
+
+
+  // faces
+  // (0,1,2)
+  for (int j = 1; j < p, j++)
+    for (int i = 1, i + j < p, i++) {
+      double w = cp[i] + cp[j] + cp[p-i-j];
+      nodes[o][0] = cp[i]/w;  nodes[o][1] = cp[j]/w;  nodes[o][2] = cp[0]/w;
+      o++;
+    }
+
+  // (0,1,3)
+  for (int j = 1; j < p, j++)
+    for (int i = 1, i + j < p, i++) {
+      double w = cp[i] + cp[j] + cp[p-i-j];
+      nodes[o][0] = cp[i]/w;  nodes[o][1] = cp[0]/w;  nodes[o][2] = cp[j]/w;
+      o++;
+    }
+
+  // (1,2,3)
+  for (int j = 1; j < p, j++)
+    for (int i = 1, i + j < p, i++) {
+      double w = cp[i] + cp[j] + cp[p-i-j];
+      nodes[o][0] = cp[p-i-j]/w;  nodes[o][1] = cp[i]/w;  nodes[o][2] = cp[j]/w;
+      o++;
+    }
+
+  // (0,2,3)
+  for (int j = 1; j < p, j++)
+    for (int i = 1, i + j < p, i++) {
+      double w = cp[i] + cp[j] + cp[p-i-j];
+      nodes[o][0] = cp[0]/w;  nodes[o][1] = cp[i]/w;  nodes[o][2] = cp[j]/w;
+      o++;
+    }
+
+  // Region
+  for (int k = 1; k < p; k++) {
+    for (int j = 1; j + k < p; j++) {
+      for (int i = 1; i + j + k < p; i++) {
+        double w = cp[i] + cp[j] + cp[k] + cp[p-i-j-k];
+        nodes[o][0] = cp[i]/w;  nodes[o][1] = cp[j]/w;  nodes[o][2] = cp[k]/w;
         o++;
       }
     }
@@ -206,12 +287,12 @@ static void getTi(
 }
 
 template<int P>
-class L2ShapeTri: public FieldShape {
+class H1ShapeTri: public FieldShape {
   public:
-    L2ShapeTri()
+    H1ShapeTri()
     {
       std::stringstream ss;
-      ss << "L2ShapeTri" << P;
+      ss << "H1ShapeTri" << P;
       name = ss.str();
       registerSelf(name.c_str());
     }
@@ -268,27 +349,23 @@ class L2ShapeTri: public FieldShape {
 	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
       {
       	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getLocalGradients not \
-      	    implemented for L2ShapeTri. Aborting()!");
+      	    implemented for H1ShapeTri. Aborting()!");
       }
       int countNodes() const {return countTriNodes(P);}
       void getVectorValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
       {
       	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getVectorValues not implemented \
-      	    for L2ShapeTri. Try getValues. Aborting()!");
+      	    for H1ShapeTri. Try getValues. Aborting()!");
       }
     };
     EntityShape* getEntityShape(int type)
     {
     	PCU_ALWAYS_ASSERT_VERBOSE(type == Mesh::TRIANGLE,
-    			"L2ShapeTri only has entity shapes for TRIANGLEs");
+    			"H1ShapeTri only has entity shapes for TRIANGLEs");
       static Triangle tri;
       return &tri;
     }
-    // For the following to member functions we only need to
-    // consider the interior nodes, i.e.,
-    // Faces: no need to count the nodes associated with bounding edges
-    // Tets: no need to count the nodes associated with bounding edges/faces
     bool hasNodesIn(int dimension)
     {
       if (dimension == Mesh::typeDimension[Mesh::TRIANGLE])
@@ -297,8 +374,16 @@ class L2ShapeTri: public FieldShape {
     }
     int countNodesOn(int type)
     {
-      if (type == apf::Mesh::TRIANGLE) return countTriNodes(P);
-      return 0;
+      switch (type) {
+	case apf::Mesh::VERTEX:
+	  return 1;
+	case apf::Mesh::EDGE:
+	  if (P>1) return P-1; else return 0;
+	case apf::Mesh::TRIANGLE:
+	  if (P>2) return (P-1)*(P-2)/2; else return 0;
+	default:
+	  return 0;
+      }
     }
     int getOrder() {return P;}
     void getNodeXi(int type, int node, Vector3& xi)

From 5158d912457493daa7a58d5fbfb0547ebe48a35c Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Sat, 26 Sep 2020 18:57:59 -0400
Subject: [PATCH 221/555] Adds code to get internal node xis

---
 apf/apfH1Shapes.cc | 88 ++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 78 insertions(+), 10 deletions(-)

diff --git a/apf/apfH1Shapes.cc b/apf/apfH1Shapes.cc
index 6ef7269d3..cd185939c 100644
--- a/apf/apfH1Shapes.cc
+++ b/apf/apfH1Shapes.cc
@@ -286,13 +286,56 @@ static void getTi(
   }
 }
 
+static apf::Vector3 getH1NodeXi(int type, int P, int node)
+{
+  if (type = apf::Mesh::VERTEX)
+    return apf::Vector3(0., 0., 0.);
+
+  apf::NewArray<double> cp;
+  getClosedPoints(P, cp);
+
+  if (type == apf::Mesh::EDGE) {
+    PCU_ALWAYS_ASSERT(node >= 0 && node < P-1);
+    int c = 0;
+    for (int i = 1; i < P; i++)
+      if (node == c)
+      	return apf::Vector3(2*cp[i]-1, 0., 0.);
+      else
+      	c++;
+  }
+
+  if (type == apf::Mesh::TRIANGLE) {
+    PCU_ALWAYS_ASSERT(node >= 0 && node < P-2);
+    int c = 0;
+    for (int j = 1; j < P; j++)
+      for (int i = 1; i + j < P; i++)
+	if (node == c) {
+	  double w = cp[i] + cp[j] + cp[P-i-j];
+	  return apf::Vector3(cp[i]/w, cp[j]/w, 0.);
+	}
+	else
+	  c++;
+  }
+
+  if (type == apf::Mesh::TET) {
+
+    return;
+  }
+
+  PCU_ALWAYS_ASSERT_VERBOSE(0, "Unsupported type!");
+  return;
+
+
+}
+
+
 template<int P>
-class H1ShapeTri: public FieldShape {
+class H1Shape: public FieldShape {
   public:
     H1ShapeTri()
     {
       std::stringstream ss;
-      ss << "H1ShapeTri" << P;
+      ss << "H1Shape" << P;
       name = ss.str();
       registerSelf(name.c_str());
     }
@@ -349,28 +392,50 @@ class H1ShapeTri: public FieldShape {
 	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
       {
       	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getLocalGradients not \
-      	    implemented for H1ShapeTri. Aborting()!");
+      	    implemented for H1Shape. Aborting()!");
       }
       int countNodes() const {return countTriNodes(P);}
       void getVectorValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
       {
       	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getVectorValues not implemented \
-      	    for H1ShapeTri. Try getValues. Aborting()!");
+      	    for H1Shape. Try getValues. Aborting()!");
       }
     };
     EntityShape* getEntityShape(int type)
     {
-    	PCU_ALWAYS_ASSERT_VERBOSE(type == Mesh::TRIANGLE,
-    			"H1ShapeTri only has entity shapes for TRIANGLEs");
+      PCU_ALWAYS_ASSERT_VERBOSE(type == Mesh::TRIANGLE || type == Mesh::TET,
+    			"H1Shape only has entity shapes for TRIANGLEs or TETs");
+      static Vertex vert;
+      static Edge edge;
       static Triangle tri;
-      return &tri;
+      static Tetrahedron tet;
+      static apf::EntityShape* shapes[apf::Mesh::TYPES] =
+      {&vert,
+       &edge,
+       &tri,
+       NULL,
+       &tet,
+       NULL,
+       NULL,
+       NULL};
+      return shapes[type];
     }
     bool hasNodesIn(int dimension)
     {
-      if (dimension == Mesh::typeDimension[Mesh::TRIANGLE])
-      	return true;
-      return false;
+      return P > dimension;
+      /* switch (dimension) { */
+	/* case 0: */
+	  /* return true; */
+	/* case 1: */
+	  /* return P>1; */
+	/* case 2: */
+	  /* return P>2; */
+	/* case 3; */
+	  /* return P>3 */
+	/* default: */
+	  /* return false; */
+      /* } */
     }
     int countNodesOn(int type)
     {
@@ -381,6 +446,8 @@ class H1ShapeTri: public FieldShape {
 	  if (P>1) return P-1; else return 0;
 	case apf::Mesh::TRIANGLE:
 	  if (P>2) return (P-1)*(P-2)/2; else return 0;
+	case apf::Mesh::TET:
+	  if (P>3) return (P-1)*(P-2)*(P-3)/6; else return 0;
 	default:
 	  return 0;
       }
@@ -388,6 +455,7 @@ class H1ShapeTri: public FieldShape {
     int getOrder() {return P;}
     void getNodeXi(int type, int node, Vector3& xi)
     {
+      getH1NodeXi(type, P, node, xi);
       PCU_ALWAYS_ASSERT_VERBOSE(type == Mesh::TRIANGLE,
       	  "getNodeXi for L2ShapeTri can be called only for TRIANGLEs");
       apf::NewArray<double> op;

From c1b4577e01dbe1effb2bebd76868c1f7d3411498 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Sun, 27 Sep 2020 08:37:51 -0400
Subject: [PATCH 222/555] Adds final changes for h1 shapes

---
 apf/apfH1Shapes.cc | 494 ++++++++++++++++++++-------------------------
 1 file changed, 220 insertions(+), 274 deletions(-)

diff --git a/apf/apfH1Shapes.cc b/apf/apfH1Shapes.cc
index cd185939c..bbe1623dc 100644
--- a/apf/apfH1Shapes.cc
+++ b/apf/apfH1Shapes.cc
@@ -22,7 +22,15 @@ using namespace std;
 
 namespace apf {
 
-static unsigned const MAX_ND_ORDER = 10;
+// This is used for static tables only.
+// The following implementation are for general orders but template classes
+// are only instantiated up to order 10 for now.
+static unsigned const MAX_ORDER = 10;
+
+static inline int countEdgeNodes(int P)
+{
+  return (P+1);
+}
 
 static inline int countTriNodes(int P)
 {
@@ -34,6 +42,21 @@ static inline int countTetNodes(int P)
   return (P+1)*(P+2)*(P+3)/6;
 }
 
+static inline int countInternalEdgeNodes(int P)
+{
+  return (P-1);
+}
+
+static inline int countInternalTriNodes(int P)
+{
+  return (P-1)*(P-2)/2;
+}
+
+static inline int countInternalTetNodes(int P)
+{
+  return (P-1)*(P-2)*(P-3)/6;
+}
+
 static void computeTriangleTi(
     int P, /*order*/
     mth::Matrix<double>& Q, /*Q in QR factorization of Ti*/
@@ -255,8 +278,8 @@ static void getTi(
   PCU_ALWAYS_ASSERT_VERBOSE(cond,
       "type should be either apf::Mesh::TRIANGLE or apf::Mesh::TET!");
 
-  static apf::NewArray<double> transformQ[apf::Mesh::TYPES][MAX_ND_ORDER+1];
-  static apf::NewArray<double> transformR[apf::Mesh::TYPES][MAX_ND_ORDER+1];
+  static apf::NewArray<double> transformQ[apf::Mesh::TYPES][MAX_ORDER+1];
+  static apf::NewArray<double> transformR[apf::Mesh::TYPES][MAX_ORDER+1];
   int n = type == apf::Mesh::TRIANGLE ? countTriNodes(P) : countTetNodes(P);
 
   // get the transform matrices if the are not already computed
@@ -286,6 +309,7 @@ static void getTi(
   }
 }
 
+// internal nodes only
 static apf::Vector3 getH1NodeXi(int type, int P, int node)
 {
   if (type = apf::Mesh::VERTEX)
@@ -295,7 +319,7 @@ static apf::Vector3 getH1NodeXi(int type, int P, int node)
   getClosedPoints(P, cp);
 
   if (type == apf::Mesh::EDGE) {
-    PCU_ALWAYS_ASSERT(node >= 0 && node < P-1);
+    PCU_ALWAYS_ASSERT(node >= 0 && node < countInternalEdgeNodes(P));
     int c = 0;
     for (int i = 1; i < P; i++)
       if (node == c)
@@ -305,7 +329,7 @@ static apf::Vector3 getH1NodeXi(int type, int P, int node)
   }
 
   if (type == apf::Mesh::TRIANGLE) {
-    PCU_ALWAYS_ASSERT(node >= 0 && node < P-2);
+    PCU_ALWAYS_ASSERT(node >= 0 && node < countInternalTriNodes(P));
     int c = 0;
     for (int j = 1; j < P; j++)
       for (int i = 1; i + j < P; i++)
@@ -318,29 +342,70 @@ static apf::Vector3 getH1NodeXi(int type, int P, int node)
   }
 
   if (type == apf::Mesh::TET) {
-
-    return;
+    PCU_ALWAYS_ASSERT(node >= 0 && node < countInternalTetNodes(P));
+    int c = 0;
+    for (int k = 1; k < P; k++)
+      for (int j = 1; j + k < P; j++)
+	for (int i = 1; i + j + k < P; i++)
+	  if (c == node) {
+	    double w = cp[i] + cp[j] + cp[k] + cp[p-i-j-k];
+	    return apf::Vector3(cp[i]/w, cp[j]/w, cp[k]/w);
+	  }
+	  else
+	    c++;
   }
 
   PCU_ALWAYS_ASSERT_VERBOSE(0, "Unsupported type!");
-  return;
-
-
+  return apf::Vector3(0., 0., 0.);
 }
 
-
 template<int P>
 class H1Shape: public FieldShape {
   public:
-    H1ShapeTri()
+    H1Shape()
     {
       std::stringstream ss;
-      ss << "H1Shape" << P;
+      ss << "H1Shape_" << P;
       name = ss.str();
       registerSelf(name.c_str());
     }
     const char* getName() const { return name.c_str(); }
     bool isVectorShape() {return false;}
+    class Edge : public apf::EntityShape
+    {
+    public:
+      int getOrder() {return P;}
+      void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const& xi, apf::NewArray<double>& shapes) const
+      {
+      	// TODO
+      	const int p = P;
+      	apf::NewArray<double> shape_x(p+1);
+        int dof = countNodes();
+
+        double x = (xi[0]+1.)/2.; // go from [-1,1] to [0,1]
+
+        getChebyshevT(p, x, &shape_x[0]);
+        shapes.allocate(dof);
+        shapes[0] = shape_x[0];
+        shapes[1] = shape_x[p];
+        for (int i = 1; i < p; i++)
+          shapes[i+1] = shape_x[i];
+      }
+      void getLocalGradients(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
+      {
+      	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getLocalGradients not \
+      	    implemented for H1Shape for Edges. Aborting()!");
+      }
+      int countNodes() const {return countEdgeNodes(P);}
+      void getVectorValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
+      {
+      	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getVectorValues not implemented \
+      	    for H1Shape. Try getValues. Aborting()!");
+      }
+    };
     class Triangle : public apf::EntityShape
     {
     public:
@@ -392,7 +457,7 @@ class H1Shape: public FieldShape {
 	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
       {
       	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getLocalGradients not \
-      	    implemented for H1Shape. Aborting()!");
+      	    implemented for H1Shape for Tris. Aborting()!");
       }
       int countNodes() const {return countTriNodes(P);}
       void getVectorValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
@@ -402,93 +467,6 @@ class H1Shape: public FieldShape {
       	    for H1Shape. Try getValues. Aborting()!");
       }
     };
-    EntityShape* getEntityShape(int type)
-    {
-      PCU_ALWAYS_ASSERT_VERBOSE(type == Mesh::TRIANGLE || type == Mesh::TET,
-    			"H1Shape only has entity shapes for TRIANGLEs or TETs");
-      static Vertex vert;
-      static Edge edge;
-      static Triangle tri;
-      static Tetrahedron tet;
-      static apf::EntityShape* shapes[apf::Mesh::TYPES] =
-      {&vert,
-       &edge,
-       &tri,
-       NULL,
-       &tet,
-       NULL,
-       NULL,
-       NULL};
-      return shapes[type];
-    }
-    bool hasNodesIn(int dimension)
-    {
-      return P > dimension;
-      /* switch (dimension) { */
-	/* case 0: */
-	  /* return true; */
-	/* case 1: */
-	  /* return P>1; */
-	/* case 2: */
-	  /* return P>2; */
-	/* case 3; */
-	  /* return P>3 */
-	/* default: */
-	  /* return false; */
-      /* } */
-    }
-    int countNodesOn(int type)
-    {
-      switch (type) {
-	case apf::Mesh::VERTEX:
-	  return 1;
-	case apf::Mesh::EDGE:
-	  if (P>1) return P-1; else return 0;
-	case apf::Mesh::TRIANGLE:
-	  if (P>2) return (P-1)*(P-2)/2; else return 0;
-	case apf::Mesh::TET:
-	  if (P>3) return (P-1)*(P-2)*(P-3)/6; else return 0;
-	default:
-	  return 0;
-      }
-    }
-    int getOrder() {return P;}
-    void getNodeXi(int type, int node, Vector3& xi)
-    {
-      getH1NodeXi(type, P, node, xi);
-      PCU_ALWAYS_ASSERT_VERBOSE(type == Mesh::TRIANGLE,
-      	  "getNodeXi for L2ShapeTri can be called only for TRIANGLEs");
-      apf::NewArray<double> op;
-      getOpenPoints(P, op);
-      int c = 0;
-      for (int j = 0; j <= P; j++) {
-      	for (int i = 0; i + j <= P; i++) {
-      	  if (node == c) {
-      	    double w = op[i] + op[j] + op[P-i-j];
-      	    xi = Vector3( op[i]/w, op[j]/w, 0. );
-      	    return;
-      	  }
-      	  else
-      	    c++;
-      	}
-      }
-    }
-  private:
-    std::string name;
-};
-
-template<int P>
-class L2ShapeTet: public FieldShape {
-  public:
-    L2ShapeTet()
-    {
-      std::stringstream ss;
-      ss << "L2ShapeTet_" << P;
-      name = ss.str();
-      registerSelf(name.c_str());
-    }
-    const char* getName() const { return name.c_str(); }
-    bool isVectorShape() {return false;}
     class Tetrahedron : public apf::EntityShape
     {
     public:
@@ -544,208 +522,176 @@ class L2ShapeTet: public FieldShape {
 	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
       {
       	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getLocalGradients not \
-      	    implemented for L2ShapeTet. Aborting()!");
+      	    implemented for H1Shape for Tets. Aborting()!");
       }
       int countNodes() const {return countTetNodes(P);}
       void getVectorValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const& /*xi*/, apf::NewArray<apf::Vector3>& /*shapes*/) const
       {
       	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getVectorValues not implemented for \
-      	    L2ShapeTet. Try getValues. Aborting()!");
+      	    H1Shape. Try getValues. Aborting()!");
       }
     };
+
     EntityShape* getEntityShape(int type)
     {
-      PCU_ALWAYS_ASSERT_VERBOSE(type == Mesh::TET,
-      	  "L2ShapeTet only has entity shapes for TETs");
+      static Vertex vert;
+      static Edge edge;
+      static Triangle tri;
       static Tetrahedron tet;
-      return &tet;
+      static apf::EntityShape* shapes[apf::Mesh::TYPES] =
+      {&vert,
+       &edge,
+       &tri,
+       NULL,
+       &tet,
+       NULL,
+       NULL,
+       NULL};
+      return shapes[type];
     }
-    // For the following to member functions we only need to
-    // consider the interior nodes, i.e.,
-    // Faces: no need to count the nodes associated with bounding edges
-    // Tets: no need to count the nodes associated with bounding edges/faces
     bool hasNodesIn(int dimension)
-    { 
-    	if (dimension == Mesh::typeDimension[Mesh::TET])
-    		return true;
-      return false;
+    {
+      return P > dimension;
+      /* switch (dimension) { */
+	/* case 0: */
+	  /* return true; */
+	/* case 1: */
+	  /* return P>1; */
+	/* case 2: */
+	  /* return P>2; */
+	/* case 3; */
+	  /* return P>3 */
+	/* default: */
+	  /* return false; */
+      /* } */
     }
     int countNodesOn(int type)
     {
-      if (type == apf::Mesh::TET) return countTetNodes(P);
-      return 0;
+      switch (type) {
+	case apf::Mesh::VERTEX:
+	  return 1;
+	case apf::Mesh::EDGE:
+	  if (P>1) return countInternalEdgeNodes(P); else return 0;
+	case apf::Mesh::TRIANGLE:
+	  if (P>2) return countInternalTriNodes(P); else return 0;
+	case apf::Mesh::TET:
+	  if (P>3) return countInternalTetNodes(P); else return 0;
+	default:
+	  return 0;
+      }
     }
     int getOrder() {return P;}
     void getNodeXi(int type, int node, Vector3& xi)
     {
-      PCU_ALWAYS_ASSERT_VERBOSE(type == Mesh::TET,
-      	  "getNodeXi for L2ShapeTet can be called only for TETs");
-      apf::NewArray<double> op;
-      getOpenPoints(P, op);
-      int c = 0;
-      for (int k = 0; k <= P; k++) {
-      	for (int j = 0; j + k <= P; j++) {
-      	  for (int i = 0; i + j + k <= P; i++) {
-      	    if( node == c) {
-      	      double w = op[i] + op[j] + op[k] + op[P-i-j-k];
-      	      xi = Vector3( op[i]/w, op[j]/w,  op[k]/w );
-      	      return;
-      	    }
-      	    else
-      	      c++;
-      	  }
-      	}
-      }
+      xi = getH1NodeXi(type, P, node);
     }
   private:
     std::string name;
 };
 
-
-static apf::FieldShape* getL2ShapeTri(int order)
-{
-  PCU_ALWAYS_ASSERT_VERBOSE(order >= 0,
-      "order is expected to be bigger than or equal to 0!");
-  PCU_ALWAYS_ASSERT_VERBOSE(order <= 10,
-      "order is expected to be less than or equal to 10!");
-  static L2ShapeTri<0>  l2_0;
-  static L2ShapeTri<1>  l2_1;
-  static L2ShapeTri<2>  l2_2;
-  static L2ShapeTri<3>  l2_3;
-  static L2ShapeTri<4>  l2_4;
-  static L2ShapeTri<5>  l2_5;
-  static L2ShapeTri<6>  l2_6;
-  static L2ShapeTri<7>  l2_7;
-  static L2ShapeTri<8>  l2_8;
-  static L2ShapeTri<9>  l2_9;
-  static L2ShapeTri<10> l2_10;
-  static FieldShape* const l2Shapes[11] = {&l2_0,
-                                           &l2_1,
-                                           &l2_2,
-                                           &l2_3,
-                                           &l2_4,
-                                           &l2_5,
-                                           &l2_6,
-                                           &l2_7,
-                                           &l2_8,
-                                           &l2_9,
-                                           &l2_10};
-  return l2Shapes[order];
-}
-
-static apf::FieldShape* getL2ShapeTet(int order)
+apf::FieldShape* getH1Shape(int order, int type)
 {
-  PCU_ALWAYS_ASSERT_VERBOSE(order >= 0,
-      "order is expected to be bigger than or equal to 0!");
+  PCU_ALWAYS_ASSERT_VERBOSE(order > 0,
+      "order is expected to be bigger than or equal to 1!");
   PCU_ALWAYS_ASSERT_VERBOSE(order <= 10,
       "order is expected to be less than or equal to 10!");
-  static L2ShapeTet<0>  l2_0;
-  static L2ShapeTet<1>  l2_1;
-  static L2ShapeTet<2>  l2_2;
-  static L2ShapeTet<3>  l2_3;
-  static L2ShapeTet<4>  l2_4;
-  static L2ShapeTet<5>  l2_5;
-  static L2ShapeTet<6>  l2_6;
-  static L2ShapeTet<7>  l2_7;
-  static L2ShapeTet<8>  l2_8;
-  static L2ShapeTet<9>  l2_9;
-  static L2ShapeTet<10> l2_10;
-  static FieldShape* const l2Shapes[11] = {&l2_0,
-                                           &l2_1,
-                                           &l2_2,
-                                           &l2_3,
-                                           &l2_4,
-                                           &l2_5,
-                                           &l2_6,
-                                           &l2_7,
-                                           &l2_8,
-                                           &l2_9,
-                                           &l2_10};
-  return l2Shapes[order];
-}
-
-
-apf::FieldShape* getL2Shape(int order, int type)
-{
-  if (type == Mesh::TRIANGLE)
-    return getL2ShapeTri(order);
-  else if (type == Mesh::TET)
-    return getL2ShapeTet(order);
-  else
-    PCU_ALWAYS_ASSERT_VERBOSE(0,
-    	"L2Shapes are only implemented for tris and tets");
+  // Note: to have higher order H1 fields all you need to do is to
+  // instantiate the class  up to that order in the following table
+  // and change the above assert so that the code does not fail.
+  static H1Shape<1>  h1_1;
+  static H1Shape<2>  h1_2;
+  static H1Shape<3>  h1_3;
+  static H1Shape<4>  h1_4;
+  static H1Shape<5>  h1_5;
+  static H1Shape<6>  h1_6;
+  static H1Shape<7>  h1_7;
+  static H1Shape<8>  h1_8;
+  static H1Shape<9>  h1_9;
+  static H1Shape<10> h1_10;
+  static FieldShape* const H1Shapes[11] = {NULL,
+                                           &h1_1,
+                                           &h1_2,
+                                           &h1_3,
+                                           &h1_4,
+                                           &h1_5,
+                                           &h1_6,
+                                           &h1_7,
+                                           &h1_8,
+                                           &h1_9,
+                                           &h1_10};
+  return h1Shapes[order];
 }
 
-void projectL2Field(Field* to, Field* from)
-{
-  // checks on the from field
-  // checks on the to field
-  apf::FieldShape* tShape = getShape(to);
-  std::string      tName  = tShape->getName();
-  int              tOrder = tShape->getOrder();
-  PCU_ALWAYS_ASSERT_VERBOSE((tName == std::string("Linear")) && (tOrder == 1),
-                "The to field needs to be 1st order Lagrange!");
-
-  Mesh* m = getMesh(from);
-  // auxiliary count fields
-  Field* count = createField(m, "counter", SCALAR, getLagrange(1));
-  double xis[4][3] = {{0., 0., 0.},
-                      {1., 0., 0.},
-                      {0., 1., 0.},
-                      {0., 0., 1.}};
-  // zero out the fields
-  zeroField(to);
-  zeroField(count);
-
-  int nc = countComponents(to);
-  NewArray<double> atXi(nc);
-  NewArray<double> currentVal(nc);
-  NewArray<double> sum(nc);
-
-  MeshEntity* e;
-  MeshIterator* it = m->begin(m->getDimension());
-  while( (e = m->iterate(it)) ) {
-    MeshElement* me = createMeshElement(m, e);
-    Element* el = createElement(from, me);
-    MeshEntity* dvs[4];
-    m->getDownward(e, 0, dvs);
-    for (int i=0; i<4; i++) {
-      getComponents(el, Vector3(xis[i]), &(atXi[0]));
-      getComponents(to, dvs[i], 0, &(currentVal[0]));
-      for (int j = 0; j < nc; j++) {
-        currentVal[j] += atXi[j];
-      }
-      double currentCount = getScalar(count, dvs[i], 0);
-      currentCount += 1.;
-      setComponents(to, dvs[i], 0, &(currentVal[0]));
-      setScalar(count, dvs[i], 0, currentCount);
-    }
-    destroyElement(el);
-    destroyMeshElement(me);
-  }
-  m->end(it);
-
-  // take care of entities on part boundary
-  accumulate(to);
-  accumulate(count);
-
-  it = m->begin(0);
-  while( (e = m->iterate(it)) ) {
-    getComponents(to, e, 0, &(sum[0]));
-    int cnt = getScalar(count, e, 0);
-    for (int i = 0; i < nc; i++) {
-      sum[i] /= cnt;
-    }
-    setComponents(to, e, 0, &(sum[0]));
-  }
-  m->end(it);
-
-  // take care of entities on part boundary
-  synchronize(to);
-
-  m->removeField(count);
-  destroyField(count);
-}
+/* void projectL2Field(Field* to, Field* from) */
+/* { */
+/*   // checks on the from field */
+/*   // checks on the to field */
+/*   apf::FieldShape* tShape = getShape(to); */
+/*   std::string      tName  = tShape->getName(); */
+/*   int              tOrder = tShape->getOrder(); */
+/*   PCU_ALWAYS_ASSERT_VERBOSE((tName == std::string("Linear")) && (tOrder == 1), */
+/*                 "The to field needs to be 1st order Lagrange!"); */
+
+/*   Mesh* m = getMesh(from); */
+/*   // auxiliary count fields */
+/*   Field* count = createField(m, "counter", SCALAR, getLagrange(1)); */
+/*   double xis[4][3] = {{0., 0., 0.}, */
+/*                       {1., 0., 0.}, */
+/*                       {0., 1., 0.}, */
+/*                       {0., 0., 1.}}; */
+/*   // zero out the fields */
+/*   zeroField(to); */
+/*   zeroField(count); */
+
+/*   int nc = countComponents(to); */
+/*   NewArray<double> atXi(nc); */
+/*   NewArray<double> currentVal(nc); */
+/*   NewArray<double> sum(nc); */
+
+/*   MeshEntity* e; */
+/*   MeshIterator* it = m->begin(m->getDimension()); */
+/*   while( (e = m->iterate(it)) ) { */
+/*     MeshElement* me = createMeshElement(m, e); */
+/*     Element* el = createElement(from, me); */
+/*     MeshEntity* dvs[4]; */
+/*     m->getDownward(e, 0, dvs); */
+/*     for (int i=0; i<4; i++) { */
+/*       getComponents(el, Vector3(xis[i]), &(atXi[0])); */
+/*       getComponents(to, dvs[i], 0, &(currentVal[0])); */
+/*       for (int j = 0; j < nc; j++) { */
+/*         currentVal[j] += atXi[j]; */
+/*       } */
+/*       double currentCount = getScalar(count, dvs[i], 0); */
+/*       currentCount += 1.; */
+/*       setComponents(to, dvs[i], 0, &(currentVal[0])); */
+/*       setScalar(count, dvs[i], 0, currentCount); */
+/*     } */
+/*     destroyElement(el); */
+/*     destroyMeshElement(me); */
+/*   } */
+/*   m->end(it); */
+
+/*   // take care of entities on part boundary */
+/*   accumulate(to); */
+/*   accumulate(count); */
+
+/*   it = m->begin(0); */
+/*   while( (e = m->iterate(it)) ) { */
+/*     getComponents(to, e, 0, &(sum[0])); */
+/*     int cnt = getScalar(count, e, 0); */
+/*     for (int i = 0; i < nc; i++) { */
+/*       sum[i] /= cnt; */
+/*     } */
+/*     setComponents(to, e, 0, &(sum[0])); */
+/*   } */
+/*   m->end(it); */
+
+/*   // take care of entities on part boundary */
+/*   synchronize(to); */
+
+/*   m->removeField(count); */
+/*   destroyField(count); */
+/* } */
 
 }

From bdaa86178647b0e646270018629a536375f605d5 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Sun, 27 Sep 2020 08:38:23 -0400
Subject: [PATCH 223/555] Integrates h1 shapes into the rest of the code

---
 apf/CMakeLists.txt | 1 +
 apf/apfShape.cc    | 1 +
 apf/apfShape.h     | 5 +++++
 3 files changed, 7 insertions(+)

diff --git a/apf/CMakeLists.txt b/apf/CMakeLists.txt
index 47b943f46..fc5aaf347 100644
--- a/apf/CMakeLists.txt
+++ b/apf/CMakeLists.txt
@@ -28,6 +28,7 @@ set(SOURCES
   apfPolyBasis1D.cc
   apfNedelec.cc
   apfL2Shapes.cc
+  apfH1Shapes.cc
   apfVector.cc
   apfVectorElement.cc
   apfVectorField.cc
diff --git a/apf/apfShape.cc b/apf/apfShape.cc
index 5a4973e4a..daaf82ff1 100644
--- a/apf/apfShape.cc
+++ b/apf/apfShape.cc
@@ -81,6 +81,7 @@ FieldShape* getShapeByName(const char* name)
   getNedelec(1);
   getL2Shape(0, apf::Mesh::TRIANGLE);
   getL2Shape(0, apf::Mesh::TET);
+  getH1Shape(0);
   std::string s(name);
   if (registry.count(s))
     return registry[s];
diff --git a/apf/apfShape.h b/apf/apfShape.h
index 927e0df18..3439f5d24 100644
--- a/apf/apfShape.h
+++ b/apf/apfShape.h
@@ -172,6 +172,11 @@ FieldShape* getNedelec(int order);
  */
 FieldShape* getL2Shape(int order, int type);
 
+/** \brief Get the H1 shapes of a given order
+  \details These are hierarchic shapes that are compatible with MFEM's impl.
+ */
+FieldShape* getH1Shape(int order);
+
 /** \brief Project a hierarchic field */
 void projectHierarchicField(Field* to, Field* from);
 

From 22af04e3f43d99c47b90d0683ee717ab60dca6c7 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Sun, 27 Sep 2020 08:47:44 -0400
Subject: [PATCH 224/555] Fixes compilation errors

---
 apf/apfH1Shapes.cc | 50 ++++++++++++++++++++++++++++++++++------------
 1 file changed, 37 insertions(+), 13 deletions(-)

diff --git a/apf/apfH1Shapes.cc b/apf/apfH1Shapes.cc
index bbe1623dc..92450e5c4 100644
--- a/apf/apfH1Shapes.cc
+++ b/apf/apfH1Shapes.cc
@@ -155,6 +155,7 @@ static void computeTetTi(
   nodes.setSize(non);
 
 
+  int o = 0;
   // vertices
   nodes[o][0] = cp[0]; nodes[o][1] = cp[0]; nodes[o][0] = cp[0];
   o++;
@@ -205,32 +206,32 @@ static void computeTetTi(
 
   // faces
   // (0,1,2)
-  for (int j = 1; j < p, j++)
-    for (int i = 1, i + j < p, i++) {
+  for (int j = 1; j < p; j++)
+    for (int i = 1; i + j < p; i++) {
       double w = cp[i] + cp[j] + cp[p-i-j];
       nodes[o][0] = cp[i]/w;  nodes[o][1] = cp[j]/w;  nodes[o][2] = cp[0]/w;
       o++;
     }
 
   // (0,1,3)
-  for (int j = 1; j < p, j++)
-    for (int i = 1, i + j < p, i++) {
+  for (int j = 1; j < p; j++)
+    for (int i = 1; i + j < p; i++) {
       double w = cp[i] + cp[j] + cp[p-i-j];
       nodes[o][0] = cp[i]/w;  nodes[o][1] = cp[0]/w;  nodes[o][2] = cp[j]/w;
       o++;
     }
 
   // (1,2,3)
-  for (int j = 1; j < p, j++)
-    for (int i = 1, i + j < p, i++) {
+  for (int j = 1; j < p; j++)
+    for (int i = 1; i + j < p; i++) {
       double w = cp[i] + cp[j] + cp[p-i-j];
       nodes[o][0] = cp[p-i-j]/w;  nodes[o][1] = cp[i]/w;  nodes[o][2] = cp[j]/w;
       o++;
     }
 
   // (0,2,3)
-  for (int j = 1; j < p, j++)
-    for (int i = 1, i + j < p, i++) {
+  for (int j = 1; j < p; j++)
+    for (int i = 1; i + j < p; i++) {
       double w = cp[i] + cp[j] + cp[p-i-j];
       nodes[o][0] = cp[0]/w;  nodes[o][1] = cp[i]/w;  nodes[o][2] = cp[j]/w;
       o++;
@@ -312,7 +313,7 @@ static void getTi(
 // internal nodes only
 static apf::Vector3 getH1NodeXi(int type, int P, int node)
 {
-  if (type = apf::Mesh::VERTEX)
+  if (type == apf::Mesh::VERTEX)
     return apf::Vector3(0., 0., 0.);
 
   apf::NewArray<double> cp;
@@ -348,7 +349,7 @@ static apf::Vector3 getH1NodeXi(int type, int P, int node)
       for (int j = 1; j + k < P; j++)
 	for (int i = 1; i + j + k < P; i++)
 	  if (c == node) {
-	    double w = cp[i] + cp[j] + cp[k] + cp[p-i-j-k];
+	    double w = cp[i] + cp[j] + cp[k] + cp[P-i-j-k];
 	    return apf::Vector3(cp[i]/w, cp[j]/w, cp[k]/w);
 	  }
 	  else
@@ -371,6 +372,30 @@ class H1Shape: public FieldShape {
     }
     const char* getName() const { return name.c_str(); }
     bool isVectorShape() {return false;}
+    class Vertex : public apf::EntityShape
+    {
+    public:
+      int getOrder() {return P;}
+      void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const& /*xi*/, apf::NewArray<double>& shapes) const
+      {
+        shapes.allocate(1);
+        shapes[0] = 1.;
+      }
+      void getLocalGradients(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
+      {
+      	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getLocalGradients not \
+      	    implemented for H1Shape for Verts. Aborting()!");
+      }
+      int countNodes() const {return countEdgeNodes(P);}
+      void getVectorValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
+	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
+      {
+      	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getVectorValues not implemented \
+      	    for H1Shape. Try getValues. Aborting()!");
+      }
+    };
     class Edge : public apf::EntityShape
     {
     public:
@@ -378,7 +403,6 @@ class H1Shape: public FieldShape {
       void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const& xi, apf::NewArray<double>& shapes) const
       {
-      	// TODO
       	const int p = P;
       	apf::NewArray<double> shape_x(p+1);
         int dof = countNodes();
@@ -590,7 +614,7 @@ class H1Shape: public FieldShape {
     std::string name;
 };
 
-apf::FieldShape* getH1Shape(int order, int type)
+apf::FieldShape* getH1Shape(int order)
 {
   PCU_ALWAYS_ASSERT_VERBOSE(order > 0,
       "order is expected to be bigger than or equal to 1!");
@@ -609,7 +633,7 @@ apf::FieldShape* getH1Shape(int order, int type)
   static H1Shape<8>  h1_8;
   static H1Shape<9>  h1_9;
   static H1Shape<10> h1_10;
-  static FieldShape* const H1Shapes[11] = {NULL,
+  static FieldShape* const h1Shapes[11] = {NULL,
                                            &h1_1,
                                            &h1_2,
                                            &h1_3,

From e3a2c16309caaa941e8d760e54cbf1b9df4f7555 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Sun, 27 Sep 2020 13:12:31 -0400
Subject: [PATCH 225/555] Fixes an error in registering h1 shapes

---
 apf/apfShape.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apf/apfShape.cc b/apf/apfShape.cc
index daaf82ff1..5806b07fe 100644
--- a/apf/apfShape.cc
+++ b/apf/apfShape.cc
@@ -81,7 +81,7 @@ FieldShape* getShapeByName(const char* name)
   getNedelec(1);
   getL2Shape(0, apf::Mesh::TRIANGLE);
   getL2Shape(0, apf::Mesh::TET);
-  getH1Shape(0);
+  getH1Shape(1);
   std::string s(name);
   if (registry.count(s))
     return registry[s];

From 344cdc8bfdc5e5d21066838c49218c8c7cb9712e Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Sun, 27 Sep 2020 13:13:30 -0400
Subject: [PATCH 226/555] Adds both serial and parallel test for H1 shapes

---
 test/CMakeLists.txt | 1 +
 test/testing.cmake  | 8 ++++++++
 2 files changed, 9 insertions(+)

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 027f43ed4..169be92b8 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -178,6 +178,7 @@ test_exe_func(test_verify test_verify.cc)
 test_exe_func(hierarchic hierarchic.cc)
 test_exe_func(nedelecShapes nedelecShapes.cc)
 test_exe_func(L2Shapes L2Shapes.cc)
+test_exe_func(H1Shapes H1Shapes.cc)
 test_exe_func(poisson poisson.cc)
 test_exe_func(ph_adapt ph_adapt.cc)
 test_exe_func(assert_timing assert_timing.cc)
diff --git a/test/testing.cmake b/test/testing.cmake
index 5841957fd..4fcb893e4 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -526,6 +526,14 @@ mpi_test(l2_shape_tet_parallel 4
   ./L2Shapes
   ".null"
   "${MDIR}/4/cube.smb")
+mpi_test(h1_shape_serial 1
+  ./H1Shapes
+  ".null"
+  "${MDIR}/cube.smb")
+mpi_test(h1_shape_parallel 4
+  ./H1Shapes
+  ".null"
+  "${MDIR}/4/cube.smb")
 set(MDIR ${MESHES}/cube/pumi24)
 mpi_test(pumiLoadMesh-1p 1
   ./pumiLoadMesh

From 094dc3c3eaaf1ed28b9913d5debfc5a918fd1e67 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Sun, 27 Sep 2020 13:14:12 -0400
Subject: [PATCH 227/555] Updates the tribits.cmake to include apfH1Shapes

---
 apf/pkg_tribits.cmake | 1 +
 1 file changed, 1 insertion(+)

diff --git a/apf/pkg_tribits.cmake b/apf/pkg_tribits.cmake
index 37ad6a0ad..746ba5d94 100644
--- a/apf/pkg_tribits.cmake
+++ b/apf/pkg_tribits.cmake
@@ -31,6 +31,7 @@ set(APF_SOURCES
   apfPolyBasis1D.cc
   apfNedelec.cc
   apfL2Shapes.cc
+  apfH1Shapes.cc
   apfVector.cc
   apfVectorElement.cc
   apfVectorField.cc

From 53339a11dfd579cab935c6b7a2a6da3d7e333aea Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Mon, 28 Sep 2020 09:37:28 -0400
Subject: [PATCH 228/555] replace c++11 features

support building with llvm11
see #306 for c++11 discussion
---
 apf/apfNedelec.cc | 15 +++++++++------
 mds/mdsUgrid.cc   | 44 +++++++++++++++++++++++++-------------------
 2 files changed, 34 insertions(+), 25 deletions(-)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index 9558b7791..02533397e 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -547,9 +547,10 @@ class Nedelec: public FieldShape {
     class Edge : public apf::EntityShape
     {
     private:
-      const int dim = 1; // ref elem dim
-      const double c = 0.; // center of edge
+      const int dim; // ref elem dim
+      const double c; // center of edge
     public:
+      Edge() : dim(1), c(0.0) {};
       int getOrder() {return P;}
       void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<double>&) const
@@ -583,9 +584,10 @@ class Nedelec: public FieldShape {
     class Triangle : public apf::EntityShape
     {
     private:
-      const int dim = 2; // reference element dim
-      const double c = 1./3.; // center of tri
+      const int dim; // reference element dim
+      const double c; // center of tri
     public:
+      Triangle() : dim(2), c(1./3) {};
       int getOrder() {return P;}
       void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<double>&) const
@@ -721,9 +723,10 @@ class Nedelec: public FieldShape {
     class Tetrahedron : public apf::EntityShape
     {
     private:
-      const int dim = 3;
-      const double c = 1./4.;
+      const int dim;
+      const double c;
     public:
+      Tetrahedron() : dim(3), c(1./4) {};
       int getOrder() {return P;}
       void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<double>&) const
diff --git a/mds/mdsUgrid.cc b/mds/mdsUgrid.cc
index 84bf57902..ceb8a001b 100644
--- a/mds/mdsUgrid.cc
+++ b/mds/mdsUgrid.cc
@@ -308,14 +308,20 @@ namespace {
 
       apf::Vector3 vtx_coord;
       m->getPoint(vtx, 0, vtx_coord);
-      bool same_dim = std::all_of(upward_dim.begin(), upward_dim.end(), 
-                                  [upward_dim](const int i) {
-                                    return upward_dim[0] == i;
-                                  });
-      bool same_id = std::all_of(upward_id.begin(), upward_id.end(), 
-                                 [upward_id](const int i) {
-                                   return upward_id[0] == i;
-                                 });
+      bool same_dim = true;
+      for(size_t i=0; i<upward_dim.size(); i++) {
+         if (upward_dim[0] != upward_dim[i]) {
+           same_dim = false;
+           break;
+         }
+      }
+      bool same_id = true;
+      for(size_t i=0; i<upward_id.size(); i++) {
+         if (upward_id[0] != upward_id[i]) {
+           same_id = false;
+           break;
+         }
+      }
       if (same_dim && same_id)
       {
         /// if all edges adjacent to a vertex have the same classification
@@ -335,23 +341,23 @@ namespace {
       {
         /// find all the indices in the vectors where the model entity is 1D
         std::vector<int> edge_indx;
-        std::vector<int>::iterator iter = upward_dim.begin();
-        while ((iter = std::find_if(iter, upward_dim.end(), 
-                                    [](const int i){ return i == 1; }))
-               != upward_dim.end())
-        {
-            edge_indx.push_back(std::distance(upward_dim.begin(), iter));
-            iter++;
+        for(size_t i = 0; i < upward_dim.size(); i++) {
+          if(upward_dim[i] == 1) {
+            edge_indx.push_back(i);
+          }
         }
         std::vector<int> edge_id;
         for (size_t i = 0; i < edge_indx.size(); i++)
         {
           edge_id.push_back(upward_id[edge_indx[i]]);
         }
-        bool same_edge_id = std::all_of(edge_id.begin(), edge_id.end(), 
-                                        [edge_id](const int i) {
-                                          return edge_id[0] == i;
-                                        });
+        bool same_edge_id = true;
+        for(size_t i=0; i<edge_id.size(); i++) {
+          if (edge_id[0] != edge_id[i]) {
+            same_id = false;
+            break;
+          }
+        }
         if (same_edge_id)
         {
           /// if the edges adjacent to a vertex that are classified on a dim 1

From c907ff4770dbd95dbddd3509319759bedfa5772a Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 29 Sep 2020 15:44:19 -0400
Subject: [PATCH 229/555] Implements getGaussLobattoPoints

---
 apf/apfPolyBasis1D.cc | 41 ++++++++++++++++++++++++++++++++++++++---
 apf/apfPolyBasis1D.h  |  2 +-
 2 files changed, 39 insertions(+), 4 deletions(-)

diff --git a/apf/apfPolyBasis1D.cc b/apf/apfPolyBasis1D.cc
index e6703fdae..e6636a49c 100644
--- a/apf/apfPolyBasis1D.cc
+++ b/apf/apfPolyBasis1D.cc
@@ -95,8 +95,43 @@ void getGaussLegendrePoints(int np, double* pts)
   }
 }
 
-void getGaussLobattoPoints(int /*np*/, double* /*pts*/)
-{ /* implement Gauss Lobatto points. Later when needed. */
+void getGaussLobattoPoints(int np, double* pts)
+{
+  PCU_ALWAYS_ASSERT_VERBOSE(np >= 2, "np is expected to be greater or equal to 2!");
+  // end points are 0 and 1
+  pts[0] = 0.;
+  pts[np-1] = 1.;
+
+  // interior points are symmetric
+  for (int i = 1; i <= (np-1)/2; i++) {
+    double x_i = std::sin(M_PI * ((double)(i)/(np-1) - 0.5));
+    double z_i = 0.;
+    double p_l;
+    bool done = false;
+    for (int iter = 0; true; iter++) {
+      double p_lm1 = 1.;
+      p_l = x_i;
+      for (int l = 1; l < (np-1); l++) {
+        double p_lp1 = ((2*l + 1)*x_i*p_l - l*p_lm1)/(l + 1);
+        p_lm1 = p_l;
+        p_l = p_lp1;
+      }
+      if (done) break;
+
+      double dx = (x_i*p_l - p_lm1) / (np*p_l);
+      if (std::abs(dx) < 1e-16)
+      {
+        done = true;
+        z_i = ((1.0 + x_i) - dx)/2;
+      }
+      PCU_ALWAYS_ASSERT_VERBOSE(iter < 8,
+         "something went wrong in getGaussLobattoPoints!");
+      x_i -= dx;
+    }
+
+    pts[i] = z_i;
+    pts[np-i-1] = 1-z_i;
+  }
 }
 
 
@@ -113,7 +148,7 @@ void getClosedPoints(
     apf::NewArray<double>& cp,
     int type)
 {
-    getPoints(order, cp, type);
+  getPoints(order, cp, type);
 }
 
 void getChebyshevT(int order, double xi, double* u)
diff --git a/apf/apfPolyBasis1D.h b/apf/apfPolyBasis1D.h
index ea123b913..7fd083754 100644
--- a/apf/apfPolyBasis1D.h
+++ b/apf/apfPolyBasis1D.h
@@ -24,7 +24,7 @@ enum Gauss{
 void getGaussLegendrePoints(int np, double* pts);
 
 /** \brief Gauss Lobatto points */
-void getGaussLobattoPoints(int, double*);
+void getGaussLobattoPoints(int np, double* pts);
 
 /** \brief get open points for a given order and Gauss point type */
 void getOpenPoints(

From db9f2a1540e9133df5a1ee15dae14d24e3f14a90 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 29 Sep 2020 15:45:55 -0400
Subject: [PATCH 230/555] Fixes a few bugs in node definitions for H1 Shapes

---
 apf/apfH1Shapes.cc | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/apf/apfH1Shapes.cc b/apf/apfH1Shapes.cc
index 92450e5c4..40e791614 100644
--- a/apf/apfH1Shapes.cc
+++ b/apf/apfH1Shapes.cc
@@ -78,11 +78,11 @@ static void computeTriangleTi(
   int o = 0;
 
   // vertices
-  nodes[o][0] = cp[0]; nodes[o][1] = cp[0]; nodes[o][0] = 0.;
+  nodes[o][0] = cp[0]; nodes[o][1] = cp[0]; nodes[o][2] = 0.;
   o++;
-  nodes[o][0] = cp[p]; nodes[o][1] = cp[0]; nodes[o][0] = 0.;
+  nodes[o][0] = cp[p]; nodes[o][1] = cp[0]; nodes[o][2] = 0.;
   o++;
-  nodes[o][0] = cp[0]; nodes[o][1] = cp[p]; nodes[o][0] = 0.;
+  nodes[o][0] = cp[0]; nodes[o][1] = cp[p]; nodes[o][2] = 0.;
   o++;
 
   // edges
@@ -151,37 +151,37 @@ static void computeTetTi(
   apf::NewArray<double> shape_z(p+1);
   apf::NewArray<double> shape_l(p+1);
 
-  apf::DynamicArray<apf::Vector3> nodes (non);
+  apf::DynamicArray<apf::Vector3> nodes(non);
   nodes.setSize(non);
 
 
   int o = 0;
   // vertices
-  nodes[o][0] = cp[0]; nodes[o][1] = cp[0]; nodes[o][0] = cp[0];
+  nodes[o][0] = cp[0]; nodes[o][1] = cp[0]; nodes[o][2] = cp[0];
   o++;
-  nodes[o][0] = cp[p]; nodes[o][1] = cp[0]; nodes[o][0] = cp[0];
+  nodes[o][0] = cp[p]; nodes[o][1] = cp[0]; nodes[o][2] = cp[0];
   o++;
-  nodes[o][0] = cp[0]; nodes[o][1] = cp[p]; nodes[o][0] = cp[0];
+  nodes[o][0] = cp[0]; nodes[o][1] = cp[p]; nodes[o][2] = cp[0];
   o++;
-  nodes[o][0] = cp[0]; nodes[o][1] = cp[0]; nodes[o][0] = cp[p];
+  nodes[o][0] = cp[0]; nodes[o][1] = cp[0]; nodes[o][2] = cp[p];
   o++;
 
   // edges
   for (int i = 1; i < p; i++) // (0,1)
   {
-    nodes[o][0] = cp[i];  nodes[o][1] = cp[0];  nodes[o][2] = cp[0.];
+    nodes[o][0] = cp[i];  nodes[o][1] = cp[0];  nodes[o][2] = cp[0];
     o++;
   }
 
   for (int i = 1; i < p; i++) // (1,2)
   {
-    nodes[o][0] = cp[p-i];  nodes[o][1] = cp[i];  nodes[o][2] = cp[0.];
+    nodes[o][0] = cp[p-i];  nodes[o][1] = cp[i];  nodes[o][2] = cp[0];
     o++;
   }
 
   for (int i = 1; i < p; i++) // (2,0)
   {
-    nodes[o][0] = cp[0];  nodes[o][1] = cp[p-i];  nodes[o][2] = cp[0.];
+    nodes[o][0] = cp[0];  nodes[o][1] = cp[p-i];  nodes[o][2] = cp[0];
     o++;
   }
 
@@ -209,7 +209,7 @@ static void computeTetTi(
   for (int j = 1; j < p; j++)
     for (int i = 1; i + j < p; i++) {
       double w = cp[i] + cp[j] + cp[p-i-j];
-      nodes[o][0] = cp[i]/w;  nodes[o][1] = cp[j]/w;  nodes[o][2] = cp[0]/w;
+      nodes[o][0] = cp[i]/w;  nodes[o][1] = cp[j]/w;  nodes[o][2] = cp[0];
       o++;
     }
 
@@ -233,7 +233,7 @@ static void computeTetTi(
   for (int j = 1; j < p; j++)
     for (int i = 1; i + j < p; i++) {
       double w = cp[i] + cp[j] + cp[p-i-j];
-      nodes[o][0] = cp[0]/w;  nodes[o][1] = cp[i]/w;  nodes[o][2] = cp[j]/w;
+      nodes[o][0] = cp[0];  nodes[o][1] = cp[i]/w;  nodes[o][2] = cp[j]/w;
       o++;
     }
 
@@ -388,7 +388,7 @@ class H1Shape: public FieldShape {
       	PCU_ALWAYS_ASSERT_VERBOSE(0, "error: getLocalGradients not \
       	    implemented for H1Shape for Verts. Aborting()!");
       }
-      int countNodes() const {return countEdgeNodes(P);}
+      int countNodes() const {return 1;}
       void getVectorValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
       {

From 5f750960b6124c7e2fa173446479acb3da31bc36 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 29 Sep 2020 15:46:33 -0400
Subject: [PATCH 231/555] Implements alignSharedNodes for H1 Shapes

---
 apf/apfH1Shapes.cc | 48 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/apf/apfH1Shapes.cc b/apf/apfH1Shapes.cc
index 40e791614..287de143c 100644
--- a/apf/apfH1Shapes.cc
+++ b/apf/apfH1Shapes.cc
@@ -484,6 +484,19 @@ class H1Shape: public FieldShape {
       	    implemented for H1Shape for Tris. Aborting()!");
       }
       int countNodes() const {return countTriNodes(P);}
+      void alignSharedNodes(apf::Mesh* m,
+      	  apf::MeshEntity* elem, apf::MeshEntity* shared, int order[])
+      {
+      	int which, rotate;
+      	bool flip;
+      	getAlignment(m, elem, shared, which, flip, rotate);
+      	if (!flip)
+      	  for (int i = 0; i < P-1; i++)
+      	    order[i] = i;
+	else
+	  for (int i = 0; i < P-1; i++)
+	    order[i] = P-2-i;
+      }
       void getVectorValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const&, apf::NewArray<apf::Vector3>&) const
       {
@@ -549,6 +562,41 @@ class H1Shape: public FieldShape {
       	    implemented for H1Shape for Tets. Aborting()!");
       }
       int countNodes() const {return countTetNodes(P);}
+      void alignSharedNodes(apf::Mesh* m,
+      	  apf::MeshEntity* elem, apf::MeshEntity* shared, int order[])
+      {
+      	int stype = m->getType(shared);
+      	int which, rotate;
+      	bool flip;
+      	getAlignment(m, elem, shared, which, flip, rotate);
+      	if (stype == apf::Mesh::EDGE) {
+	  if (!flip)
+	    for (int i = 0; i < P-1; i++)
+	      order[i] = i;
+	  else
+	    for (int i = 0; i < P-1; i++)
+	      order[i] = P-2-i;
+	  return;
+	}
+	PCU_ALWAYS_ASSERT_VERBOSE(stype == apf::Mesh::TRIANGLE,
+	    "shared type must be triangle!");
+	int idx0, idx1;
+	if (!flip) {
+	  idx0 = (3-rotate) % 3;
+	  idx1 = (4-rotate) % 3;
+	}
+	else {
+	  idx0 = (2+rotate) % 3;
+	  idx1 = (1+rotate) % 3;
+	}
+	int idx = 0;
+	for (int i = 0; i <= P-3; i++)
+	  for (int j = 0; j <= P-3-i; j++) {
+	    int ijk[3] = {i, j, P-3-i-j};
+	    order[idx] = ijk[idx0]*(P-2)-ijk[idx0]*(ijk[idx0]-1)/2+ijk[idx1];
+	    idx++;
+	  }
+      }
       void getVectorValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
 	  apf::Vector3 const& /*xi*/, apf::NewArray<apf::Vector3>& /*shapes*/) const
       {

From 1be35eeb4bbaa161f0c32b532fecf6c767f0fbc6 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 29 Sep 2020 16:03:54 -0400
Subject: [PATCH 232/555] Adds the source for testing of H1 Shapes

---
 test/H1Shapes.cc | 154 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 154 insertions(+)
 create mode 100644 test/H1Shapes.cc

diff --git a/test/H1Shapes.cc b/test/H1Shapes.cc
new file mode 100644
index 000000000..3cddf62d8
--- /dev/null
+++ b/test/H1Shapes.cc
@@ -0,0 +1,154 @@
+#include <PCU.h>
+#include <gmi_mesh.h>
+#include <gmi_null.h>
+#include <apf.h>
+#include <apfMDS.h>
+#include <apfMesh2.h>
+#include <apfShape.h>
+#include <apfField.h>
+#include <lionPrint.h>
+#include <pcu_util.h>
+
+#include <cassert>
+#include <cstdlib>
+#include <iostream>
+#include <string>
+#include <math.h>
+
+using namespace std;
+
+// User defined vector functions E(x,y,z) of order up to 6
+void E_exact(const apf::Vector3& x, apf::Vector3& value, int p);
+
+void testH1(
+    apf::Mesh2* m,
+    const apf::Vector3& testXi,
+    int ndOrder, int exactOrder);
+
+int main(int argc, char** argv)
+{
+  MPI_Init(&argc,&argv);
+  PCU_Comm_Init();
+
+  lion_set_verbosity(1);
+
+  if (argc != 3) {
+    if(0==PCU_Comm_Self())
+      std::cerr << "usage: " << argv[0]
+        << " <model.dmg or .null> <mesh.smb>\n";
+    return EXIT_FAILURE;
+  }
+
+  gmi_register_mesh();
+  gmi_register_null();
+
+  gmi_model* g = gmi_load(argv[1]);
+  apf::Mesh2* m = apf::loadMdsMesh(g,argv[2]);
+  m->verify();
+
+  for (int i = 1; i <= 6; i++) {
+    testH1(
+    	m, /* mesh */
+	apf::Vector3(1./7., 1./11., 1./3.), /* test point */
+	i, /* order of h1 field */
+	i); /* order of test field */
+  }
+
+  apf::destroyMesh(m);
+  PCU_Comm_Free();
+  MPI_Finalize();
+}
+
+void E_exact(const apf::Vector3& x, apf::Vector3& value, int p)
+{
+  // Polynomial coefficients for each component of exact vector field
+  double a[7] = { 1.0, -1.0,  2., -2., -1.0,  1.0, -1.0};
+  double b[7] = {-2.0,  1.0, -2.,  2., -1.0, -1.0,  1.0};
+  double c[7] = { 3.0,  0.0, -1.,  100., -1.0,  1.0,  0.0};
+
+  value[0] = 0.0;
+  value[1] = 0.0;
+  value[2] = 0.0;
+  for (int i = p;  i >= 0; i--) {
+    value[0] += pow(x[0],p)*a[p];
+    value[1] += pow(x[1],p)*b[p];
+    value[2] += pow(x[2],p)*c[p];
+  }
+}
+
+void testH1(
+    apf::Mesh2* m,
+    const apf::Vector3& testXi,
+    int ndOrder, int exactOrder)
+{
+  // Loop over all nodes and set scalar dofs.
+  int dim = m->getDimension();
+  apf::Field* h1Field = apf::createField(
+      m, "h1_test", apf::VECTOR, apf::getH1Shape(ndOrder));
+  apf::MeshEntity* ent;
+  apf::MeshIterator* it;
+
+  for (int d = 0; d <= dim; d++) {
+    if (!h1Field->getShape()->countNodesOn(apf::Mesh::simplexTypes[d])) {
+      if(0==PCU_Comm_Self())
+        lion_oprint(1, "no nodes in dimension %d\n", d);
+      continue;
+    }
+    else
+      if(0==PCU_Comm_Self())
+        lion_oprint(1, "computing dofs for dimension %d\n", d);
+    it = m->begin(d);
+    int count = 0;
+    while( (ent = m->iterate(it)) ) {
+      int type = m->getType(ent);
+      int non = h1Field->getShape()->countNodesOn(type);
+      apf::MeshElement* me = apf::createMeshElement(m, ent);
+      for (int i = 0; i < non; i++)
+      {
+        apf::Vector3 xi, p, value;
+        h1Field->getShape()->getNodeXi(type, i, xi);
+        apf::mapLocalToGlobal(me, xi, p);
+        E_exact(p, value, exactOrder);
+
+      	apf::setVector(h1Field, ent, i, value);
+      }
+      apf::destroyMeshElement(me);
+      count++;
+    }
+    m->end(it);
+  }
+
+
+  // Verify that interpolated solution field agrees with exact field.
+  double L2ErrorE = 0.;
+  it = m->begin(dim);
+  int count = 0;
+  while( (ent = m->iterate(it)) ) {
+    apf::MeshElement* me = apf::createMeshElement(m, ent);
+    apf::Vector3 x;
+    apf::mapLocalToGlobal(me, testXi, x);
+    apf::Vector3 eFieldExact;
+    E_exact(x, eFieldExact, exactOrder);
+
+    // obtain interpolated value
+    apf::Element* el = apf::createElement(h1Field, me);
+    apf::Vector3 eFieldValue;
+    apf::getVector(el, testXi, eFieldValue);
+
+    double err = ((eFieldValue - eFieldExact) * (eFieldValue - eFieldExact));
+    L2ErrorE += err;
+    apf::destroyMeshElement(me);
+    apf::destroyElement(el);
+    count++;
+  }
+  m->end(it);
+
+  // check for field interpolation
+  if(0==PCU_Comm_Self())
+    lion_oprint(1, "L2ErrorE is %e\n", L2ErrorE);
+  PCU_ALWAYS_ASSERT_VERBOSE(L2ErrorE < 1.e-16,
+      "Fields were not interpolated correctly!");
+
+  m->removeField(h1Field);
+  apf::destroyField(h1Field);
+}

From eca2c7d5089a2dd5e46ca76aec7b7b0c7a2d0927 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 29 Sep 2020 16:26:31 -0400
Subject: [PATCH 233/555] Adds normalization to H1 Shape regression test

---
 test/H1Shapes.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/H1Shapes.cc b/test/H1Shapes.cc
index 3cddf62d8..79091be86 100644
--- a/test/H1Shapes.cc
+++ b/test/H1Shapes.cc
@@ -136,6 +136,7 @@ void testH1(
     apf::getVector(el, testXi, eFieldValue);
 
     double err = ((eFieldValue - eFieldExact) * (eFieldValue - eFieldExact));
+    err /= (eFieldExact * eFieldExact); // normalization factor
     L2ErrorE += err;
     apf::destroyMeshElement(me);
     apf::destroyElement(el);

From 5ad15e310d7ee7dda6b328598980d715b1541218 Mon Sep 17 00:00:00 2001
From: "E. Seegyoung Seol" <seols@rpi.edu>
Date: Sat, 3 Oct 2020 22:31:22 -0400
Subject: [PATCH 234/555] adding clearRemotes for part bdry modifications per
 entity

---
 apf/apfMesh2.h   | 2 ++
 apf_sim/apfSIM.h | 1 +
 mds/apfMDS.cc    | 5 +++++
 3 files changed, 8 insertions(+)

diff --git a/apf/apfMesh2.h b/apf/apfMesh2.h
index b0cb56512..7b5795ea2 100644
--- a/apf/apfMesh2.h
+++ b/apf/apfMesh2.h
@@ -36,6 +36,8 @@ class Mesh2 : public Mesh
     virtual void setRemotes(MeshEntity* e, Copies& remotes) = 0;
 /** \brief Add just one remote copy to an entity */
     virtual void addRemote(MeshEntity* e, int p, MeshEntity* r) = 0;
+/** \brief Remove remote copies */
+    virtual void clearRemotes(MeshEntity* e) = 0;
 
 // seol
 /** \brief Add just one ghost copy to an entity */
diff --git a/apf_sim/apfSIM.h b/apf_sim/apfSIM.h
index 9d806ab74..dedfe020e 100644
--- a/apf_sim/apfSIM.h
+++ b/apf_sim/apfSIM.h
@@ -95,6 +95,7 @@ class MeshSIM : public Mesh2
     // Mesh2 interface ==============================
     void setRemotes(MeshEntity*, Copies&) {}
     void addRemote(MeshEntity*, int, MeshEntity*) {}
+    void clearRemotes(MeshEntity*) {}
     void setResidence(MeshEntity*, Parts&) {}
     void setParam(MeshEntity*, Vector3 const &) {}
     void increment(MeshIterator*) {}
diff --git a/mds/apfMDS.cc b/mds/apfMDS.cc
index 3e8dddd94..d1687164b 100644
--- a/mds/apfMDS.cc
+++ b/mds/apfMDS.cc
@@ -632,6 +632,11 @@ class MeshMDS : public Mesh2
     }
 
 //seol
+    void clearRemotes(MeshEntity* e)
+    {
+      mds_set_copies(&mesh->remotes, &mesh->mds, fromEnt(e), 0);
+    }
+
     void addGhost(MeshEntity* e, int p, MeshEntity* r)
     {
       mds_copy c;

From 8072fdbafd53e0c9a63248a269f4cce5000a4a8e Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 14 Oct 2020 14:33:52 -0400
Subject: [PATCH 235/555] version 2.2.4 #324

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index b28dc12ef..567c5fe9c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -6,7 +6,7 @@ endif()
 # This is the top level CMake file for the SCOREC build
 cmake_minimum_required(VERSION 3.0)
 
-project(SCOREC VERSION 2.2.3 LANGUAGES CXX C)
+project(SCOREC VERSION 2.2.4 LANGUAGES CXX C)
 
 include(cmake/bob.cmake)
 include(cmake/xsdk.cmake)

From 09d6b3e1a32fe7cd183b53d4c4775e708e90f16c Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Mon, 26 Oct 2020 21:02:20 -0400
Subject: [PATCH 236/555] Updates CavityTransfer to account for crved meshes

---
 ma/maMap.cc              | 52 ++++++++++++++++++++++++++++++++++++++++
 ma/maMap.h               |  8 +++++++
 ma/maSolutionTransfer.cc | 21 ++++++++++++----
 3 files changed, 77 insertions(+), 4 deletions(-)

diff --git a/ma/maMap.cc b/ma/maMap.cc
index 7aa4bd80f..afd3b7f61 100644
--- a/ma/maMap.cc
+++ b/ma/maMap.cc
@@ -9,6 +9,7 @@
 *******************************************************************************/
 
 #include "maMap.h"
+#include <pcu_util.h>
 #include <algorithm>
 
 namespace ma {
@@ -100,4 +101,55 @@ double getInsideness(apf::Mesh* m, Entity* e, Vector const& xi)
   return 0;
 }
 
+Vector curvedElemInvMap(
+    apf::Mesh* m,
+    Entity* e,
+    const Vector& p,
+    const double tol,
+    const int maxIter)
+{
+  int iter = 0;
+  // put the initial guess at the center of the elements
+  Vector xi0;
+  int type = m->getType(e);
+  if (type == apf::Mesh::VERTEX)
+    xi0 = Vector(0., 0., 0.);
+  else if (type == apf::Mesh::EDGE)
+    xi0 = Vector(0., 0., 0.);
+  else if (type == apf::Mesh::TRIANGLE)
+    xi0 = Vector(1./3., 1./3., 1./3.);
+  else if (type == apf::Mesh::TET)
+    xi0 = Vector(1./4., 1./4., 1./4.);
+  else
+    PCU_ALWAYS_ASSERT_VERBOSE(0, "unsupported type!");
+
+
+  apf::MeshElement* me = apf::createMeshElement(m, e);
+
+  Matrix Jinv;
+  Vector x;
+  Vector xi_new = xi0;
+  Vector xi_old = xi0;
+  double err = 1.e16;
+
+  // initial iteration
+  apf::getJacobianInv(me, xi_old, Jinv);
+  apf::mapLocalToGlobal(me, xi_old, x);
+  xi_new = xi_old - Jinv*(x-p);
+  err = (xi_new - xi_old)*(xi_new - xi_old);
+
+  while (err > tol) {
+    iter++;
+    if (iter > maxIter) break;
+    xi_old = xi_new;
+    apf::getJacobianInv(me, xi_old, Jinv);
+    apf::mapLocalToGlobal(me, xi_old, x);
+    xi_new = xi_old - Jinv*(x-p);
+    err = (xi_new - xi_old)*(xi_new - xi_old);
+  }
+
+  apf::destroyMeshElement(me);
+  return xi_new;
+}
+
 }
diff --git a/ma/maMap.h b/ma/maMap.h
index c26f6551c..d5c351871 100644
--- a/ma/maMap.h
+++ b/ma/maMap.h
@@ -11,6 +11,7 @@
 #ifndef MA_MAP_H
 #define MA_MAP_H
 
+#include <apf.h>
 #include "maAffine.h"
 #include "maMesh.h"
 
@@ -20,6 +21,13 @@ Affine getMap(apf::Mesh* m, Entity* e);
 
 double getInsideness(apf::Mesh* m, Entity* e, Vector const& xi);
 
+Vector curvedElemInvMap(
+    apf::Mesh* m,
+    Entity* e,
+    const Vector& p,
+    const double tol = 1.e-16,
+    const int maxIter = 10);
+
 }
 
 #endif
diff --git a/ma/maSolutionTransfer.cc b/ma/maSolutionTransfer.cc
index 89b564e82..af7dd76d9 100644
--- a/ma/maSolutionTransfer.cc
+++ b/ma/maSolutionTransfer.cc
@@ -131,7 +131,11 @@ class CavityTransfer : public FieldTransfer
       int bestI = 0;
       for (int i = 0; i < n; ++i)
       {
-        Vector xi = elemInvMaps[i] * point;
+        Vector xi;
+        if (mesh->getShape()->getOrder() == 1)
+          xi = elemInvMaps[i] * point;
+	else
+	  xi = curvedElemInvMap(mesh, apf::getMeshEntity(elems[i]), point);
         double value = getInsideness(mesh,apf::getMeshEntity(elems[i]),xi);
         if (value > bestValue)
         {
@@ -148,16 +152,25 @@ class CavityTransfer : public FieldTransfer
         Affine* elemInvMaps,
         apf::Node const& node)
     {
+      // first get the physical coordinate of the node
       Vector xi;
+      Vector point;
       shape->getNodeXi(mesh->getType(node.entity),node.node,xi);
-      Affine childMap = getMap(mesh,node.entity);
-      Vector point = childMap * xi;
+      if (mesh->getShape()->getOrder() == 1) { // if linear mesh use the affine
+      	Affine childMap = getMap(mesh,node.entity);
+      	point = childMap * xi;
+      }
+      else { // else inquire the physical coordinated of local coordinate xi
+	apf::MeshElement* me = apf::createMeshElement(mesh,node.entity);
+	apf::mapLocalToGlobal(me, xi, point);
+	apf::destroyMeshElement(me);
+      }
       Vector elemXi;
       int i = getBestElement(n,elems,elemInvMaps,point,elemXi);
       transferToNodeIn(elems[i],node,elemXi);
     }
     void transfer(
-        int n,
+        int n, // size of the cavity
         Entity** cavity,
         EntityArray& newEntities)
     {

From 505797ceda9f0052a583db7e28c6934d78c4811a Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 27 Oct 2020 16:06:08 -0400
Subject: [PATCH 237/555] Adds Barycentric 1D Basis

This is used for H1 Edges
---
 apf/apfPolyBasis1D.cc | 57 +++++++++++++++++++++++++++++++++++++++++++
 apf/apfPolyBasis1D.h  |  2 ++
 2 files changed, 59 insertions(+)

diff --git a/apf/apfPolyBasis1D.cc b/apf/apfPolyBasis1D.cc
index e6636a49c..7e113ccc4 100644
--- a/apf/apfPolyBasis1D.cc
+++ b/apf/apfPolyBasis1D.cc
@@ -216,4 +216,61 @@ void getChebyshevT(int order, double xi, double* u, double* d, double* dd)
   }
 }
 
+void poly1dBasisBarycentric(int order, double xi, double* u)
+{
+  // order 0 is trivial
+  if (order == 0) {
+    u[0] = 1.;
+    return;
+  }
+
+  apf::NewArray<double> nodes;
+  getClosedPoints(order, nodes);
+  // anything other than 0
+  apf::NewArray<double> x(order+1);
+  apf::NewArray<double> w(order+1);
+
+  for (int i = 0; i < order+1; i++) {
+    x[i] = nodes[i];
+    w[i] = 1.0;
+  }
+
+  for (int i = 0; i < order+1; i++) {
+    for (int j = 0; j < i; j++) {
+      double xij = x[i] - x[j];
+      w[i] *=  xij;
+      w[j] *= -xij;
+    }
+  }
+
+  for (int i = 0; i < order+1; i++) {
+    w[i] = 1./w[i];
+  }
+
+  int i, k, p = order;
+  double l, lk;
+  lk = 1.;
+
+  for (k = 0; k < p; k++) {
+    if (xi >= (x[k] + x[k+1]/2) / 2.)
+      lk *= xi - x[k];
+    else {
+      for (i = k+1; i <= p; i++) {
+        lk *= xi - x[i];
+      }
+      break;
+    }
+  }
+  l = lk * (xi - x[k]);
+
+  for (i = 0; i < k; i++) {
+    u[i] = l * w[i] / (xi - x[i]);
+  }
+
+  u[k] = lk * w[k];
+
+  for(i++; i <= p; i++)
+    u[i] = l * w[i] / (xi - x[i]);
+}
+
 }
diff --git a/apf/apfPolyBasis1D.h b/apf/apfPolyBasis1D.h
index 7fd083754..2ddec82ec 100644
--- a/apf/apfPolyBasis1D.h
+++ b/apf/apfPolyBasis1D.h
@@ -47,6 +47,8 @@ void getChebyshevT(int order, double xi, double* u, double* d);
 /** \brief Chebyshev polynomials of the first kind and 1st and 2nd derivative */
 void getChebyshevT(int order, double xi, double* u, double* d, double* dd);
 
+void poly1dBasisBarycentric(int order, double xi, double* u);
+
 }
 
 #endif

From f5ade54cabe74b5c577ab0673c6ce082de147c2f Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 27 Oct 2020 16:07:29 -0400
Subject: [PATCH 238/555] Fixes a bug in H1 Edges shape function evaluation

---
 apf/apfH1Shapes.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apf/apfH1Shapes.cc b/apf/apfH1Shapes.cc
index 287de143c..4049bff89 100644
--- a/apf/apfH1Shapes.cc
+++ b/apf/apfH1Shapes.cc
@@ -409,7 +409,7 @@ class H1Shape: public FieldShape {
 
         double x = (xi[0]+1.)/2.; // go from [-1,1] to [0,1]
 
-        getChebyshevT(p, x, &shape_x[0]);
+        poly1dBasisBarycentric(p, x, &shape_x[0]);
         shapes.allocate(dof);
         shapes[0] = shape_x[0];
         shapes[1] = shape_x[p];

From 7c5b4a18cec452132f13c94d6881928fee74cda5 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 27 Oct 2020 16:13:56 -0400
Subject: [PATCH 239/555] Updates test for H1 Shapes

Now tests the interpolation error for entities of all dimensions.
---
 test/H1Shapes.cc | 59 +++++++++++++++++++++++++-----------------------
 1 file changed, 31 insertions(+), 28 deletions(-)

diff --git a/test/H1Shapes.cc b/test/H1Shapes.cc
index 79091be86..47ab80447 100644
--- a/test/H1Shapes.cc
+++ b/test/H1Shapes.cc
@@ -120,36 +120,39 @@ void testH1(
 
 
   // Verify that interpolated solution field agrees with exact field.
-  double L2ErrorE = 0.;
-  it = m->begin(dim);
-  int count = 0;
-  while( (ent = m->iterate(it)) ) {
-    apf::MeshElement* me = apf::createMeshElement(m, ent);
-    apf::Vector3 x;
-    apf::mapLocalToGlobal(me, testXi, x);
-    apf::Vector3 eFieldExact;
-    E_exact(x, eFieldExact, exactOrder);
-
-    // obtain interpolated value
-    apf::Element* el = apf::createElement(h1Field, me);
-    apf::Vector3 eFieldValue;
-    apf::getVector(el, testXi, eFieldValue);
-
-    double err = ((eFieldValue - eFieldExact) * (eFieldValue - eFieldExact));
-    err /= (eFieldExact * eFieldExact); // normalization factor
-    L2ErrorE += err;
-    apf::destroyMeshElement(me);
-    apf::destroyElement(el);
-    count++;
-  }
-  m->end(it);
+  for (int d = 0; d <= dim; d++) {
 
-  // check for field interpolation
-  if(0==PCU_Comm_Self())
-    lion_oprint(1, "L2ErrorE is %e\n", L2ErrorE);
-  PCU_ALWAYS_ASSERT_VERBOSE(L2ErrorE < 1.e-16,
-      "Fields were not interpolated correctly!");
+    double L2ErrorE = 0.;
+    it = m->begin(d);
+    int count = 0;
+    while( (ent = m->iterate(it)) ) {
+      apf::MeshElement* me = apf::createMeshElement(m, ent);
+      apf::Vector3 x;
+      apf::mapLocalToGlobal(me, testXi, x);
+      apf::Vector3 eFieldExact;
+      E_exact(x, eFieldExact, exactOrder);
+
+      // obtain interpolated value
+      apf::Element* el = apf::createElement(h1Field, me);
+      apf::Vector3 eFieldValue;
+      apf::getVector(el, testXi, eFieldValue);
+
+      double err = ((eFieldValue - eFieldExact) * (eFieldValue - eFieldExact));
+      err /= (eFieldExact * eFieldExact); // normalization factor
+      L2ErrorE += err;
+      apf::destroyMeshElement(me);
+      apf::destroyElement(el);
+      count++;
+    }
+    m->end(it);
 
+    // check for field interpolation
+    if(0==PCU_Comm_Self())
+      lion_oprint(1, "L2ErrorE for entities of dimension %d is %e\n", d, L2ErrorE);
+    PCU_ALWAYS_ASSERT_VERBOSE(L2ErrorE < 1.e-16,
+	"Fields were not interpolated correctly!");
+
+  }
   m->removeField(h1Field);
   apf::destroyField(h1Field);
 }

From 31eb50aee2b382fdfa45a2cc25405dc565e022f4 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 27 Oct 2020 21:03:02 -0400
Subject: [PATCH 240/555] Fixes a bug in curvedElemInvMap

Because of the way pseudo-inverse is defined for lower rank Jacobian there is a need for an additional transpose.
---
 ma/maMap.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/ma/maMap.cc b/ma/maMap.cc
index afd3b7f61..a596ef9d9 100644
--- a/ma/maMap.cc
+++ b/ma/maMap.cc
@@ -134,6 +134,8 @@ Vector curvedElemInvMap(
 
   // initial iteration
   apf::getJacobianInv(me, xi_old, Jinv);
+  if (type != apf::Mesh::TET)
+    Jinv = apf::transpose(Jinv);
   apf::mapLocalToGlobal(me, xi_old, x);
   xi_new = xi_old - Jinv*(x-p);
   err = (xi_new - xi_old)*(xi_new - xi_old);
@@ -143,6 +145,8 @@ Vector curvedElemInvMap(
     if (iter > maxIter) break;
     xi_old = xi_new;
     apf::getJacobianInv(me, xi_old, Jinv);
+    if (type != apf::Mesh::TET)
+      Jinv = apf::transpose(Jinv);
     apf::mapLocalToGlobal(me, xi_old, x);
     xi_new = xi_old - Jinv*(x-p);
     err = (xi_new - xi_old)*(xi_new - xi_old);

From 17e2efe9e8bdb43c7bb702768a0563db5600668a Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Wed, 28 Oct 2020 09:40:35 -0400
Subject: [PATCH 241/555] Fixes a bug in H1Shapes for Edges

---
 apf/apfPolyBasis1D.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apf/apfPolyBasis1D.cc b/apf/apfPolyBasis1D.cc
index 7e113ccc4..9af87c6e9 100644
--- a/apf/apfPolyBasis1D.cc
+++ b/apf/apfPolyBasis1D.cc
@@ -252,7 +252,7 @@ void poly1dBasisBarycentric(int order, double xi, double* u)
   lk = 1.;
 
   for (k = 0; k < p; k++) {
-    if (xi >= (x[k] + x[k+1]/2) / 2.)
+    if (xi >= (x[k] + x[k+1]) / 2.)
       lk *= xi - x[k];
     else {
       for (i = k+1; i <= p; i++) {

From 3e041cd19898358e924c5377e83d61cb9501e0d4 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Wed, 28 Oct 2020 19:36:13 -0400
Subject: [PATCH 242/555] Adds more changes to curvedElemInvMap

Because Jacobian routines in pumi always return the transpose of the asked quantity, Jinv had to be transposed even for tets.
---
 ma/maMap.cc | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/ma/maMap.cc b/ma/maMap.cc
index a596ef9d9..bdd137c4c 100644
--- a/ma/maMap.cc
+++ b/ma/maMap.cc
@@ -134,8 +134,7 @@ Vector curvedElemInvMap(
 
   // initial iteration
   apf::getJacobianInv(me, xi_old, Jinv);
-  if (type != apf::Mesh::TET)
-    Jinv = apf::transpose(Jinv);
+  Jinv = apf::transpose(Jinv);
   apf::mapLocalToGlobal(me, xi_old, x);
   xi_new = xi_old - Jinv*(x-p);
   err = (xi_new - xi_old)*(xi_new - xi_old);
@@ -145,8 +144,7 @@ Vector curvedElemInvMap(
     if (iter > maxIter) break;
     xi_old = xi_new;
     apf::getJacobianInv(me, xi_old, Jinv);
-    if (type != apf::Mesh::TET)
-      Jinv = apf::transpose(Jinv);
+    Jinv = apf::transpose(Jinv);
     apf::mapLocalToGlobal(me, xi_old, x);
     xi_new = xi_old - Jinv*(x-p);
     err = (xi_new - xi_old)*(xi_new - xi_old);

From 649c6f7935b0504be8f0ffe570207c87c595e742 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Wed, 28 Oct 2020 19:41:42 -0400
Subject: [PATCH 243/555] updates transferElements in maRefine

The coordinate fields has to be taken care of before solutions transfer of other fields can be performed.
---
 ma/maRefine.cc | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/ma/maRefine.cc b/ma/maRefine.cc
index 4325186fb..fbdda77b7 100644
--- a/ma/maRefine.cc
+++ b/ma/maRefine.cc
@@ -344,15 +344,19 @@ void transferElements(Refine* r)
 {
   Adapt* a = r->adapt;
   Mesh* m = a->mesh;
-  SolutionTransfer* st = a->solutionTransfer;
-  int td = st->getTransferDimension();
-  for (int d = td; d <= m->getDimension(); ++d)
-    for (size_t i=0; i < r->toSplit[d].getSize(); ++i)
-      st->onRefine(r->toSplit[d][i],r->newEntities[d][i]);
-  td = a->shape->getTransferDimension();
-  for (int d = td; d <= m->getDimension(); ++d)
-    for (size_t i=0; i < r->toSplit[d].getSize(); ++i)
-      a->shape->onRefine(r->toSplit[d][i],r->newEntities[d][i]);
+  { // first take care of the coordinates
+    int td = a->shape->getTransferDimension();
+    for (int d = td; d <= m->getDimension(); ++d)
+      for (size_t i=0; i < r->toSplit[d].getSize(); ++i)
+	a->shape->onRefine(r->toSplit[d][i],r->newEntities[d][i]);
+  }
+  { // then take care of the fields
+    SolutionTransfer* st = a->solutionTransfer;
+    int td = st->getTransferDimension();
+    for (int d = td; d <= m->getDimension(); ++d)
+      for (size_t i=0; i < r->toSplit[d].getSize(); ++i)
+	st->onRefine(r->toSplit[d][i],r->newEntities[d][i]);
+  }
 }
 
 void forgetNewEntities(Refine* r)

From fe6ccdf9dcd98fc8a6c7c0e7a5618b18946e514f Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Wed, 28 Oct 2020 21:17:45 -0400
Subject: [PATCH 244/555] modifies passing conditions for h1 shapes test

---
 test/H1Shapes.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/H1Shapes.cc b/test/H1Shapes.cc
index 47ab80447..af1859b1f 100644
--- a/test/H1Shapes.cc
+++ b/test/H1Shapes.cc
@@ -149,7 +149,7 @@ void testH1(
     // check for field interpolation
     if(0==PCU_Comm_Self())
       lion_oprint(1, "L2ErrorE for entities of dimension %d is %e\n", d, L2ErrorE);
-    PCU_ALWAYS_ASSERT_VERBOSE(L2ErrorE < 1.e-16,
+    PCU_ALWAYS_ASSERT_VERBOSE(L2ErrorE < 1.e-12,
 	"Fields were not interpolated correctly!");
 
   }

From 2428c4fd41dd007788ba7e61df5484ebb7ad1be0 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Wed, 28 Oct 2020 21:18:44 -0400
Subject: [PATCH 245/555] Adds a test for higher order solution transfers

---
 test/CMakeLists.txt               |   1 +
 test/highOrderSolutionTransfer.cc | 231 ++++++++++++++++++++++++++++++
 test/testing.cmake                |   4 +
 3 files changed, 236 insertions(+)
 create mode 100644 test/highOrderSolutionTransfer.cc

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 169be92b8..f318a6d22 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -197,6 +197,7 @@ if(ENABLE_SIMMETRIX)
   test_exe_func(degenerate_test degenerateSurfs.cc)
   test_exe_func(simZBalance simZBalance.cc)
   test_exe_func(crack_test crack_test.cc)
+  test_exe_func(highOrderSolutionTransfer highOrderSolutionTransfer.cc)
 endif()
 
 # send all the newly added utility executable targets
diff --git a/test/highOrderSolutionTransfer.cc b/test/highOrderSolutionTransfer.cc
new file mode 100644
index 000000000..8fbebcf2e
--- /dev/null
+++ b/test/highOrderSolutionTransfer.cc
@@ -0,0 +1,231 @@
+#include <apf.h>
+#include <apfMDS.h>
+#include <apfShape.h>
+#include <apfField.h>
+#include <gmi_mesh.h>
+#include <ma.h>
+#include <maShape.h>
+#include <PCU.h>
+#include <crv.h>
+#include <lionPrint.h>
+#ifdef HAVE_SIMMETRIX
+#include <SimUtil.h>
+#include <MeshSim.h>
+#include <gmi_sim.h>
+#include <SimModel.h>
+#endif
+#include <cassert>
+#include <stdlib.h>
+#include <iostream>
+
+void E_exact(const apf::Vector3& x, apf::Vector3& value, int p);
+
+apf::Field* addH1Field(
+    apf::Mesh2* m,
+    int ndOrder, int exactOrder);
+
+double testH1Field(
+    apf::Mesh2* m,
+    apf::Field* f,
+    const apf::Vector3& testXi,
+    int exactOrder);
+
+void testCurveAdapt(
+    const char* modelFile,
+    const char* meshFile,
+    const int mesh_order,
+    const int exact_order,
+    const int field_order);
+
+int main(int argc, char** argv)
+{
+  const char* modelFile = argv[1];
+  const char* meshFile = argv[2];
+  MPI_Init(&argc,&argv);
+  PCU_Comm_Init();
+  lion_set_verbosity(1);
+#ifdef HAVE_SIMMETRIX
+  MS_init();
+  SimModel_start();
+  Sim_readLicenseFile(0);
+  gmi_sim_start();
+  gmi_register_sim();
+#endif
+  gmi_register_mesh();
+
+
+  /* Note on choices of the orders:
+   * -- mesh geometry if of order p   [that is x(xi) is a polynomial of order p in xi]
+   * -- test (exact) field is order q [that is F(x) is a polynomial of order q in x]
+   * -- then the finite element basis has to be of order at least p*q. This is because
+   *    F(x) = F(x(xi)) will be a polynomial of order p*q in xi.
+   */
+
+  // linear adapt
+  testCurveAdapt(modelFile, meshFile,
+      1 /*mesh_order*/,
+      2 /*exact_order*/,
+      2 /*field_order*/);
+
+  // quadratic adapts
+  testCurveAdapt(modelFile, meshFile,
+      2 /*mesh_order*/,
+      2 /*exact_order*/,
+      4 /*field_order*/);
+  testCurveAdapt(modelFile, meshFile,
+      2 /*mesh_order*/,
+      3 /*exact_order*/,
+      6 /*field_order*/);
+
+  // cubic adapt
+  testCurveAdapt(modelFile, meshFile,
+      3 /*mesh_order*/,
+      2 /*exact_order*/,
+      6 /*field_order*/);
+
+  PCU_Comm_Free();
+#ifdef HAVE_SIMMETRIX
+  gmi_sim_stop();
+  Sim_unregisterAllKeys();
+  MS_exit();
+  SimModel_stop();
+#endif
+  MPI_Finalize();
+}
+
+
+void E_exact(const apf::Vector3& x, apf::Vector3& value, int p)
+{
+  // Polynomial coefficients for each component of exact vector field
+  double a[7] = { 1.0, -1.0,  2., -2., -1.0,  1.0, -1.0};
+  double b[7] = {-2.0,  1.0, -2.,  2., -1.0, -1.0,  1.0};
+  double c[7] = { 3.0,  0.0, -1.,  100., -1.0,  1.0,  0.0};
+
+  value[0] = 0.0;
+  value[1] = 0.0;
+  value[2] = 0.0;
+  for (int i = p;  i >= 0; i--) {
+    value[0] += pow(x[0],p)*a[p];
+    value[1] += pow(x[1],p)*b[p];
+    value[2] += pow(x[2],p)*c[p];
+  }
+}
+
+apf::Field* addH1Field(
+    apf::Mesh2* m,
+    int ndOrder, int exactOrder)
+{
+  // Loop over all nodes and set scalar dofs.
+  int dim = m->getDimension();
+  apf::Field* h1Field = apf::createField(
+      m, "h1_test", apf::VECTOR, apf::getH1Shape(ndOrder));
+  apf::MeshEntity* ent;
+  apf::MeshIterator* it;
+
+  for (int d = 0; d <= dim; d++) {
+    if (!h1Field->getShape()->countNodesOn(apf::Mesh::simplexTypes[d]))
+      continue;
+
+    it = m->begin(d);
+    int count = 0;
+    while( (ent = m->iterate(it)) ) {
+      int type = m->getType(ent);
+      int non = h1Field->getShape()->countNodesOn(type);
+      apf::MeshElement* me = apf::createMeshElement(m, ent);
+      for (int i = 0; i < non; i++)
+      {
+        apf::Vector3 xi, p, value;
+        h1Field->getShape()->getNodeXi(type, i, xi);
+        apf::mapLocalToGlobal(me, xi, p);
+        E_exact(p, value, exactOrder);
+
+        apf::setVector(h1Field, ent, i, value);
+      }
+      apf::destroyMeshElement(me);
+      count++;
+    }
+    m->end(it);
+  }
+
+  return h1Field;
+}
+
+double testH1Field(
+    apf::Mesh2* m,
+    apf::Field* f,
+    const apf::Vector3& testXi,
+    int exactOrder)
+{
+
+  int dim = m->getDimension();
+  int count;
+  apf::MeshIterator* it = m->begin(0);
+  apf::MeshEntity* ent;
+
+  // Verify that interpolated solution field agrees with exact field.
+  double L2ErrorE = 0.;
+  it = m->begin(dim);
+  count = 0;
+  while( (ent = m->iterate(it)) ) {
+    apf::MeshElement* me = apf::createMeshElement(m, ent);
+    apf::Vector3 x;
+    apf::mapLocalToGlobal(me, testXi, x);
+    apf::Vector3 eFieldExact;
+    E_exact(x, eFieldExact, exactOrder);
+
+    // obtain interpolated value
+    apf::Element* el = apf::createElement(f, me);
+    apf::Vector3 eFieldValue;
+    apf::getVector(el, testXi, eFieldValue);
+
+    double err = ((eFieldValue - eFieldExact) * (eFieldValue - eFieldExact));
+    /* err /= (eFieldExact * eFieldExact); // normalization factor */
+    L2ErrorE += err;
+    apf::destroyMeshElement(me);
+    apf::destroyElement(el);
+    count++;
+  }
+  m->end(it);
+
+  return L2ErrorE;
+}
+
+void testCurveAdapt(
+    const char* modelFile,
+    const char* meshFile,
+    const int mesh_order,
+    const int exact_order,
+    const int field_order)
+{
+
+  apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile);
+  m->verify();
+
+  if (mesh_order > 1) {
+    crv::BezierCurver bc(m, mesh_order, 0);
+    bc.run();
+  }
+
+  apf::Field* f = addH1Field(m, field_order, exact_order);
+  double l2ErrorBefore = testH1Field(m, f, apf::Vector3(1./3., 1./4., 1./5.), exact_order);
+
+  ma::Input* in = ma::configureUniformRefine(m,1);
+  // Snap is off for solutions transfer testing.
+  in->shouldSnap = false;
+  in->goodQuality = 0.3*0.3*0.3;
+  in->shouldFixShape = false;
+  if (mesh_order > 1)
+    crv::adapt(in);
+  else
+    ma::adapt(in);
+
+  double l2ErrorAfter = testH1Field(m, f, apf::Vector3(1./3., 1./4., 1./5.), exact_order);
+
+  lion_oprint(1, "mesh/exact/fields orders  %d/%d/%d,  before/after errors %e/%e\n",
+      mesh_order, exact_order, field_order, l2ErrorBefore, l2ErrorAfter);
+
+  PCU_ALWAYS_ASSERT(l2ErrorAfter < 1.e-16);
+
+  m->destroyNative();
+  apf::destroyMesh(m);
+}
diff --git a/test/testing.cmake b/test/testing.cmake
index 4fcb893e4..5ea5714d3 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -640,6 +640,10 @@ if(ENABLE_SIMMETRIX)
       ./curvetest
       "${MDIR}/sphere1.xmt_txt"
       "${MDIR}/sphere1_4.smb")
+    mpi_test(highOrderSolutionTransfer 1
+      ./highOrderSolutionTransfer
+      "${MDIR}/sphere1.xmt_txt"
+      "${MDIR}/sphere1_4.smb")
     mpi_test(curvedKova 1
       ./curvetest
       "${MDIR}/Kova.xmt_txt"

From 6305c61830f99efd9f76678abc0fccadea906617 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Wed, 28 Oct 2020 21:30:26 -0400
Subject: [PATCH 246/555] Places apf.h header in maMap.cc

---
 ma/maMap.cc | 1 +
 ma/maMap.h  | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/ma/maMap.cc b/ma/maMap.cc
index bdd137c4c..b247e6863 100644
--- a/ma/maMap.cc
+++ b/ma/maMap.cc
@@ -9,6 +9,7 @@
 *******************************************************************************/
 
 #include "maMap.h"
+#include <apf.h>
 #include <pcu_util.h>
 #include <algorithm>
 
diff --git a/ma/maMap.h b/ma/maMap.h
index d5c351871..7e9047c21 100644
--- a/ma/maMap.h
+++ b/ma/maMap.h
@@ -11,7 +11,6 @@
 #ifndef MA_MAP_H
 #define MA_MAP_H
 
-#include <apf.h>
 #include "maAffine.h"
 #include "maMesh.h"
 

From e78c6545ea282e824edf889f8f3c6a221896c83c Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Thu, 29 Oct 2020 18:59:44 -0400
Subject: [PATCH 247/555] Fixes indentations

---
 apf/apfH1Shapes.cc       | 34 +++++++++++++++++-----------------
 ma/maSolutionTransfer.cc |  4 ++--
 2 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/apf/apfH1Shapes.cc b/apf/apfH1Shapes.cc
index 4049bff89..e28471216 100644
--- a/apf/apfH1Shapes.cc
+++ b/apf/apfH1Shapes.cc
@@ -264,7 +264,7 @@ static void computeTetTi(
       for (int j = 0; j + k <= p; j++)
         for (int i = 0; i + j + k <= p; i++)
           T(o++, m) = shape_x[i]*shape_y[j]*shape_z[k]*shape_l[p-i-j-k];
-	}
+  }
   mth::decomposeQR(T, Q, R);
 }
 
@@ -324,9 +324,9 @@ static apf::Vector3 getH1NodeXi(int type, int P, int node)
     int c = 0;
     for (int i = 1; i < P; i++)
       if (node == c)
-      	return apf::Vector3(2*cp[i]-1, 0., 0.);
+        return apf::Vector3(2*cp[i]-1, 0., 0.);
       else
-      	c++;
+        c++;
   }
 
   if (type == apf::Mesh::TRIANGLE) {
@@ -334,12 +334,12 @@ static apf::Vector3 getH1NodeXi(int type, int P, int node)
     int c = 0;
     for (int j = 1; j < P; j++)
       for (int i = 1; i + j < P; i++)
-	if (node == c) {
-	  double w = cp[i] + cp[j] + cp[P-i-j];
-	  return apf::Vector3(cp[i]/w, cp[j]/w, 0.);
-	}
-	else
-	  c++;
+        if (node == c) {
+          double w = cp[i] + cp[j] + cp[P-i-j];
+          return apf::Vector3(cp[i]/w, cp[j]/w, 0.);
+        }
+        else
+          c++;
   }
 
   if (type == apf::Mesh::TET) {
@@ -401,10 +401,10 @@ class H1Shape: public FieldShape {
     public:
       int getOrder() {return P;}
       void getValues(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
-	  apf::Vector3 const& xi, apf::NewArray<double>& shapes) const
+          apf::Vector3 const& xi, apf::NewArray<double>& shapes) const
       {
-      	const int p = P;
-      	apf::NewArray<double> shape_x(p+1);
+        const int p = P;
+        apf::NewArray<double> shape_x(p+1);
         int dof = countNodes();
 
         double x = (xi[0]+1.)/2.; // go from [-1,1] to [0,1]
@@ -457,12 +457,12 @@ class H1Shape: public FieldShape {
           for (int i = 0; i + j <= p; i++)
             u(n++, 0) = shape_x[i]*shape_y[j]*shape_l[p-i-j];
 
-	      mth::Matrix<double> Q(dof, dof);
-      	mth::Matrix<double> R(dof, dof);
-      	getTi(P, apf::Mesh::TRIANGLE, Q, R);
+        mth::Matrix<double> Q(dof, dof);
+        mth::Matrix<double> R(dof, dof);
+        getTi(P, apf::Mesh::TRIANGLE, Q, R);
 
         mth::Matrix<double> S(dof, 1);
-      	for(int i = 0; i < 1; i++) // S = Ti * u
+        for(int i = 0; i < 1; i++) // S = Ti * u
         {
           mth::Vector<double> B (dof);
           mth::Vector<double> X (dof);
@@ -474,7 +474,7 @@ class H1Shape: public FieldShape {
         shapes.allocate(dof);
         for (int i = 0; i < dof; i++) // populate y
         {
-      	  shapes[i] = S(i,0);
+          shapes[i] = S(i,0);
         }
       }
       void getLocalGradients(apf::Mesh* /*m*/, apf::MeshEntity* /*e*/,
diff --git a/ma/maSolutionTransfer.cc b/ma/maSolutionTransfer.cc
index af7dd76d9..71a85017e 100644
--- a/ma/maSolutionTransfer.cc
+++ b/ma/maSolutionTransfer.cc
@@ -134,8 +134,8 @@ class CavityTransfer : public FieldTransfer
         Vector xi;
         if (mesh->getShape()->getOrder() == 1)
           xi = elemInvMaps[i] * point;
-	else
-	  xi = curvedElemInvMap(mesh, apf::getMeshEntity(elems[i]), point);
+        else
+          xi = curvedElemInvMap(mesh, apf::getMeshEntity(elems[i]), point);
         double value = getInsideness(mesh,apf::getMeshEntity(elems[i]),xi);
         if (value > bestValue)
         {

From ca5bf7c0da839d3aad31035f2fda548269851ce2 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Thu, 29 Oct 2020 19:08:58 -0400
Subject: [PATCH 248/555] Removes the unnecesary iostream and namespace stds

---
 apf/apfH1Shapes.cc | 2 --
 apf/apfL2Shapes.cc | 2 --
 apf/apfNedelec.cc  | 2 --
 3 files changed, 6 deletions(-)

diff --git a/apf/apfH1Shapes.cc b/apf/apfH1Shapes.cc
index e28471216..fa80c2008 100644
--- a/apf/apfH1Shapes.cc
+++ b/apf/apfH1Shapes.cc
@@ -17,8 +17,6 @@
 #include <pcu_util.h>
 #include <PCU.h>
 
-#include <iostream>
-using namespace std;
 
 namespace apf {
 
diff --git a/apf/apfL2Shapes.cc b/apf/apfL2Shapes.cc
index 783f993a2..9f240b7d6 100644
--- a/apf/apfL2Shapes.cc
+++ b/apf/apfL2Shapes.cc
@@ -17,8 +17,6 @@
 #include <pcu_util.h>
 #include <PCU.h>
 
-#include <iostream>
-using namespace std;
 
 namespace apf {
 
diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index 02533397e..a8c9d2faa 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -17,8 +17,6 @@
 #include <pcu_util.h>
 #include <PCU.h>
 
-#include <iostream>
-using namespace std;
 
 namespace apf {
 

From 3193b6b497078e32439614e4a4cd72901daff9b8 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Thu, 29 Oct 2020 19:24:58 -0400
Subject: [PATCH 249/555] Removes the unnecessary namespace std

---
 test/H1Shapes.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/H1Shapes.cc b/test/H1Shapes.cc
index af1859b1f..7c4878692 100644
--- a/test/H1Shapes.cc
+++ b/test/H1Shapes.cc
@@ -15,7 +15,6 @@
 #include <string>
 #include <math.h>
 
-using namespace std;
 
 // User defined vector functions E(x,y,z) of order up to 6
 void E_exact(const apf::Vector3& x, apf::Vector3& value, int p);

From 202fcca13469bb36bf373e92211021943cb695a9 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Thu, 29 Oct 2020 19:37:37 -0400
Subject: [PATCH 250/555] Updates testing.cmake to include the models

---
 test/testing.cmake | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/test/testing.cmake b/test/testing.cmake
index 5ea5714d3..fd989d5e3 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -520,19 +520,19 @@ mpi_test(nedelec 1
 set(MDIR ${MESHES}/cube/pumi670)
 mpi_test(l2_shape_tet_serial 1
   ./L2Shapes
-  ".null"
+  "${MDIR}/../cube.dmg"
   "${MDIR}/cube.smb")
 mpi_test(l2_shape_tet_parallel 4
   ./L2Shapes
-  ".null"
+  "${MDIR}/../cube.dmg"
   "${MDIR}/4/cube.smb")
 mpi_test(h1_shape_serial 1
   ./H1Shapes
-  ".null"
+  "${MDIR}/../cube.dmg"
   "${MDIR}/cube.smb")
 mpi_test(h1_shape_parallel 4
   ./H1Shapes
-  ".null"
+  "${MDIR}/../cube.dmg"
   "${MDIR}/4/cube.smb")
 set(MDIR ${MESHES}/cube/pumi24)
 mpi_test(pumiLoadMesh-1p 1

From 23fa9142cba996717c5b252472186b1b2b008df4 Mon Sep 17 00:00:00 2001
From: Avinash <moharana.avin@gmail.com>
Date: Sun, 1 Nov 2020 11:51:57 -0500
Subject: [PATCH 251/555] generalizes logM field.

---
 ma/maSize.cc | 40 +++++++++++++++++++++++++++-------------
 1 file changed, 27 insertions(+), 13 deletions(-)

diff --git a/ma/maSize.cc b/ma/maSize.cc
index c0fab50c7..6636fd222 100644
--- a/ma/maSize.cc
+++ b/ma/maSize.cc
@@ -439,19 +439,33 @@ struct LogAnisoSizeField : public MetricSizeField
   void init(Mesh* m, apf::Field* sizes, apf::Field* frames)
   {
     mesh = m;
-    logMField = apf::createField(m, "ma_logM", apf::MATRIX, apf::getLagrange(1));
-    Entity* v;
-    Iterator* it = m->begin(0);
-    while ( (v = m->iterate(it)) ) {
-      Vector h;
-      Matrix f;
-      apf::getVector(sizes, v, 0, h);
-      apf::getMatrix(frames, v, 0, f);
-      Vector s(log(1/h[0]/h[0]), log(1/h[1]/h[1]), log(1/h[2]/h[2]));
-      Matrix S(s[0], 0   , 0,
-              0    , s[1], 0,
-              0    , 0   , s[2]);
-      apf::setMatrix(logMField, v, 0, f * S * transpose(f));
+    logMField = apf::createField(m, "ma_logM", apf::MATRIX,
+        apf::getShape(sizes));
+    int dim = m->getDimension();
+    Entity* ent;
+    Iterator* it;
+    for (int d = 0; d <= dim; d++) {
+      if (!apf::getShape(logMField)->countNodesOn(apf::Mesh::simplexTypes[d]))
+        continue;
+      else {
+        it = m->begin(d);
+        while( (ent = m->iterate(it)) ){
+	  int type = m->getType(ent);
+	  int non = apf::getShape(logMField)->countNodesOn(type);
+	  for (int i = 0; i < non; i++) {
+	    Vector h;
+	    Matrix f;
+	    apf::getVector(sizes, ent, i, h);
+	    apf::getMatrix(frames, ent, i, f);
+	    Vector s(log(1/h[0]/h[0]), log(1/h[1]/h[1]), log(1/h[2]/h[2]));
+	    Matrix S(s[0], 0   , 0,
+		0    , s[1], 0,
+		0    , 0   , s[2]);
+	    apf::setMatrix(logMField, ent, i, f * S * transpose(f));
+	  }
+	}
+	m->end(it);
+      }
     }
   }
   void getTransform(

From b86ec3968325866b825f27604acc492f641fa44d Mon Sep 17 00:00:00 2001
From: Avinash <moharana.avin@gmail.com>
Date: Sun, 1 Nov 2020 12:15:09 -0500
Subject: [PATCH 252/555] abstracts out transfer routine from
 maSolutionTransfer.

---
 ma/CMakeLists.txt              |   1 +
 ma/maSolutionTransfer.cc       | 105 ++--------------------------
 ma/maSolutionTransferHelper.cc | 123 +++++++++++++++++++++++++++++++++
 ma/maSolutionTransferHelper.h  |  31 +++++++++
 ma/pkg_tribits.cmake           |   1 +
 5 files changed, 160 insertions(+), 101 deletions(-)
 create mode 100644 ma/maSolutionTransferHelper.cc
 create mode 100644 ma/maSolutionTransferHelper.h

diff --git a/ma/CMakeLists.txt b/ma/CMakeLists.txt
index 68e72d2ea..59c0abb21 100644
--- a/ma/CMakeLists.txt
+++ b/ma/CMakeLists.txt
@@ -25,6 +25,7 @@ set(SOURCES
   maLayerCollapse.cc
   maMatch.cc
   maSolutionTransfer.cc
+  maSolutionTransferHelper.cc
   maSnap.cc
   maEdgeSwap.cc
   maShape.cc
diff --git a/ma/maSolutionTransfer.cc b/ma/maSolutionTransfer.cc
index 71a85017e..bf8c0d6c7 100644
--- a/ma/maSolutionTransfer.cc
+++ b/ma/maSolutionTransfer.cc
@@ -8,8 +8,7 @@
  
 *******************************************************************************/
 #include "maSolutionTransfer.h"
-#include "maAffine.h"
-#include "maMap.h"
+#include "maSolutionTransferHelper.h"
 #include <apfShape.h>
 #include <apfNumbering.h>
 #include <float.h>
@@ -39,18 +38,6 @@ void SolutionTransfer::onCavity(
 {
 }
 
-static int getMinimumDimension(apf::FieldShape* s)
-{
-  int transferDimension = 4;
-  for (int d=1; d <= 3; ++d)
-    if (s->hasNodesIn(d))
-    {
-      transferDimension = d;
-      break;
-    }
-  return transferDimension;
-}
-
 int SolutionTransfer::getTransferDimension()
 {
   int transferDimension = 4;
@@ -112,102 +99,18 @@ class CavityTransfer : public FieldTransfer
     {
       minDim = getMinimumDimension(getShape(f));
     }
-    void transferToNodeIn(
-        apf::Element* elem,
-        apf::Node const& node,
-        Vector const& elemXi)
-    {
-      apf::getComponents(elem,elemXi,&(value[0]));
-      apf::setComponents(field,node.entity,node.node,&(value[0]));
-    }
-    int getBestElement(
-        int n,
-        apf::Element** elems,
-        Affine* elemInvMaps,
-        Vector const& point,
-        Vector& bestXi)
-    {
-      double bestValue = -DBL_MAX;
-      int bestI = 0;
-      for (int i = 0; i < n; ++i)
-      {
-        Vector xi;
-        if (mesh->getShape()->getOrder() == 1)
-          xi = elemInvMaps[i] * point;
-        else
-          xi = curvedElemInvMap(mesh, apf::getMeshEntity(elems[i]), point);
-        double value = getInsideness(mesh,apf::getMeshEntity(elems[i]),xi);
-        if (value > bestValue)
-        {
-          bestValue = value;
-          bestI = i;
-          bestXi = xi;
-        }
-      }
-      return bestI;
-    }
-    void transferToNode(
-        int n,
-        apf::Element** elems,
-        Affine* elemInvMaps,
-        apf::Node const& node)
-    {
-      // first get the physical coordinate of the node
-      Vector xi;
-      Vector point;
-      shape->getNodeXi(mesh->getType(node.entity),node.node,xi);
-      if (mesh->getShape()->getOrder() == 1) { // if linear mesh use the affine
-      	Affine childMap = getMap(mesh,node.entity);
-      	point = childMap * xi;
-      }
-      else { // else inquire the physical coordinated of local coordinate xi
-	apf::MeshElement* me = apf::createMeshElement(mesh,node.entity);
-	apf::mapLocalToGlobal(me, xi, point);
-	apf::destroyMeshElement(me);
-      }
-      Vector elemXi;
-      int i = getBestElement(n,elems,elemInvMaps,point,elemXi);
-      transferToNodeIn(elems[i],node,elemXi);
-    }
-    void transfer(
-        int n, // size of the cavity
-        Entity** cavity,
-        EntityArray& newEntities)
-    {
-      if (getDimension(mesh, cavity[0]) < minDim)
-        return;
-      apf::NewArray<apf::Element*> elems(n);
-      for (int i = 0; i < n; ++i)
-        elems[i] = apf::createElement(field,cavity[i]);
-      apf::NewArray<Affine> elemInvMaps(n);
-      for (int i = 0; i < n; ++i)
-        elemInvMaps[i] = invert(getMap(mesh,cavity[i]));
-      for (size_t i = 0; i < newEntities.getSize(); ++i)
-      {
-        int type = mesh->getType(newEntities[i]);
-        if (type == apf::Mesh::VERTEX)
-          continue; //vertices will have been handled specially beforehand
-        int nnodes = shape->countNodesOn(type);
-        for (int j = 0; j < nnodes; ++j)
-        {
-          apf::Node node(newEntities[i],j);
-          transferToNode(n,&(elems[0]),&(elemInvMaps[0]),node);
-        }
-      }
-      for (int i = 0; i < n; ++i)
-        apf::destroyElement(elems[i]);
-    }
     virtual void onRefine(
         Entity* parent,
         EntityArray& newEntities)
     {
-      transfer(1,&parent,newEntities);
+      transfer(field, &(value[0]), 1,&parent,newEntities);
     }
     virtual void onCavity(
         EntityArray& oldElements,
         EntityArray& newEntities)
     {
-      transfer(oldElements.getSize(),&(oldElements[0]),newEntities);
+      transfer(field, &(value[0]),
+           oldElements.getSize(),&(oldElements[0]),newEntities);
     }
   private:
     int minDim;
diff --git a/ma/maSolutionTransferHelper.cc b/ma/maSolutionTransferHelper.cc
new file mode 100644
index 000000000..d799d9b1b
--- /dev/null
+++ b/ma/maSolutionTransferHelper.cc
@@ -0,0 +1,123 @@
+
+/*******************************************************************************
+
+opyright 2013 Scientific Computation Research Center, 
+      Rensselaer Polytechnic Institute. All rights reserved.
+  
+  The LICENSE file included with this distribution describes the terms
+  of the SCOREC Non-Commercial License this program is distributed under.
+ 
+*******************************************************************************/
+#include "maSolutionTransferHelper.h"
+#include "maAffine.h"
+#include "maMap.h"
+#include <apfShape.h>
+#include <apfNumbering.h>
+#include <float.h>
+
+namespace ma {
+
+int getMinimumDimension(apf::FieldShape* s)
+{
+  int transferDimension = 4;
+  for (int d=1; d <= 3; ++d)
+    if (s->hasNodesIn(d))
+    {
+      transferDimension = d;
+      break;
+    }
+  return transferDimension;
+}
+
+int getBestElement(
+    apf::Mesh* mesh,
+    int n,
+    apf::Element** elems,
+    Affine* elemInvMaps,
+    Vector const& point,
+    Vector& bestXi)
+{
+  double bestValue = -DBL_MAX;
+  int bestI = 0;
+  for (int i = 0; i < n; ++i)
+  {
+    Vector xi;
+    if (mesh->getShape()->getOrder() == 1)
+      xi = elemInvMaps[i] * point;
+    else
+      xi = curvedElemInvMap(mesh, apf::getMeshEntity(elems[i]), point);
+    double value = getInsideness(mesh,apf::getMeshEntity(elems[i]),xi);
+    if (value > bestValue)
+    {
+      bestValue = value;
+      bestI = i;
+      bestXi = xi;
+    }
+  }
+  return bestI;
+}
+
+void transferToNode(
+    apf::Field* field,
+    double *value,
+    int n,
+    apf::Element** elems,
+    Affine* elemInvMaps,
+    apf::Node const& node)
+{
+  apf::Mesh* mesh = apf::getMesh(field);
+  // first get the physical coordinate of the node
+  Vector xi;
+  Vector point;
+  apf::FieldShape* shape = apf::getShape(field);
+  shape->getNodeXi(mesh->getType(node.entity),node.node,xi);
+  if (mesh->getShape()->getOrder() == 1) { // if linear mesh use the affine
+    Affine childMap = getMap(mesh,node.entity);
+    point = childMap * xi;
+  }
+  else { // else inquire the physical coordinated of local coordinate xi
+    apf::MeshElement* me = apf::createMeshElement(mesh,node.entity);
+    apf::mapLocalToGlobal(me, xi, point);
+    apf::destroyMeshElement(me);
+  }
+  Vector elemXi;
+  int i = getBestElement(mesh,n,elems,elemInvMaps,point,elemXi);
+  apf::getComponents(elems[i],elemXi,value);
+  apf::setComponents(field,node.entity,node.node,value);
+}
+
+void transfer(
+    apf::Field* field,
+    double *value,
+    int n, // size of the cavity
+    Entity** cavity,
+    EntityArray& newEntities)
+{
+  apf::Mesh* mesh = apf::getMesh(field);
+  apf::FieldShape* shape = apf::getShape(field);
+  int minDim = getMinimumDimension(shape);
+  if (getDimension(mesh, cavity[0]) < minDim)
+    return;
+  apf::NewArray<apf::Element*> elems(n);
+  for (int i = 0; i < n; ++i)
+    elems[i] = apf::createElement(field,cavity[i]);
+  apf::NewArray<Affine> elemInvMaps(n);
+  for (int i = 0; i < n; ++i)
+    elemInvMaps[i] = invert(getMap(mesh,cavity[i]));
+  for (size_t i = 0; i < newEntities.getSize(); ++i)
+  {
+    int type = mesh->getType(newEntities[i]);
+    if (type == apf::Mesh::VERTEX)
+      continue; //vertices will have been handled specially beforehand
+    int nnodes = shape->countNodesOn(type);
+    for (int j = 0; j < nnodes; ++j)
+    {
+      apf::Node node(newEntities[i],j);
+      transferToNode(field, value,
+      	  n,&(elems[0]),&(elemInvMaps[0]),node);
+    }
+  }
+  for (int i = 0; i < n; ++i)
+    apf::destroyElement(elems[i]);
+}
+}
diff --git a/ma/maSolutionTransferHelper.h b/ma/maSolutionTransferHelper.h
new file mode 100644
index 000000000..b1834a799
--- /dev/null
+++ b/ma/maSolutionTransferHelper.h
@@ -0,0 +1,31 @@
+
+/*******************************************************************************
+
+Copyright 2013 Scientific Computation Research Center, 
+      Rensselaer Polytechnic Institute. All rights reserved.
+  
+  The LICENSE file included with this distribution describes the terms
+  of the SCOREC Non-Commercial License this program is distributed under.
+ 
+*******************************************************************************/
+
+#ifndef MA_SOLUTIONTRANSFERHELPER_H
+#define MA_SOLUTIONTRANSFERHELPER_H
+
+#include <apf.h>
+#include "maMesh.h"
+
+#include "maAffine.h"
+#include "maMap.h"
+
+namespace ma {
+int getMinimumDimension(apf::FieldShape* s);
+void transfer(
+    apf::Field* field,
+    double *value,
+    int n, // size of the cavity
+    Entity** cavity,
+    EntityArray& newEntities);
+}
+
+#endif
diff --git a/ma/pkg_tribits.cmake b/ma/pkg_tribits.cmake
index f7fe9f20e..d1db5c276 100644
--- a/ma/pkg_tribits.cmake
+++ b/ma/pkg_tribits.cmake
@@ -22,6 +22,7 @@ set(SOURCES
   maLayerCollapse.cc
   maMatch.cc
   maSolutionTransfer.cc
+  maSolutionTransferHelper.cc
   maSnap.cc
   maEdgeSwap.cc
   maShape.cc

From ba2f09ddf4e9399d7dd411705f496ee6cf812cf8 Mon Sep 17 00:00:00 2001
From: Avinash <moharana.avin@gmail.com>
Date: Sun, 1 Nov 2020 18:58:36 -0500
Subject: [PATCH 253/555] onRefine and onCavity for SizeField transfer.

---
 ma/maSize.cc | 30 ++++++++++++++++++++++++++++++
 ma/maSize.h  | 13 +++++++++++++
 2 files changed, 43 insertions(+)

diff --git a/ma/maSize.cc b/ma/maSize.cc
index 6636fd222..26520daa0 100644
--- a/ma/maSize.cc
+++ b/ma/maSize.cc
@@ -9,6 +9,7 @@
 *******************************************************************************/
 #include <PCU.h>
 #include "maSize.h"
+#include "maSolutionTransferHelper.h"
 #include "apfMatrix.h"
 #include <apfShape.h>
 #include <cstdlib>
@@ -65,6 +66,14 @@ double IdentitySizeField::getWeight(Entity*)
   return 1.0;
 }
 
+void IdentitySizeField::onRefine(Entity*, EntityArray&)
+{
+}
+
+void IdentitySizeField::onCavity(EntityArray&, EntityArray&)
+{
+}
+
 static void orthogonalizeR(Matrix& R)
 {
   /* by the way, the principal direction vectors
@@ -413,6 +422,12 @@ struct AnisoSizeField : public MetricSizeField
                           0,0,1),
                    Vector(value,value,value));
   }
+  void onRefine(Entity*, EntityArray&)
+  {
+  }
+  void onCavity(EntityArray&, EntityArray&)
+  {
+  }
   apf::Field* hField;
   apf::Field* rField;
   BothEval bothEval;
@@ -467,6 +482,7 @@ struct LogAnisoSizeField : public MetricSizeField
 	m->end(it);
       }
     }
+    fieldVal.allocate(apf::countComponents(logMField));
   }
   void getTransform(
       apf::MeshElement* me,
@@ -511,6 +527,20 @@ struct LogAnisoSizeField : public MetricSizeField
                           0,value,0,
                           0,0,value));
   }
+  void onRefine(
+      Entity* parent,
+      EntityArray& newEntities)
+  {
+    transfer(logMField, &(fieldVal[0]), 1, &parent, newEntities);
+  }
+  void onCavity(
+      EntityArray& oldElements,
+      EntityArray& newEntities)
+  {
+    transfer(logMField, &(fieldVal[0]),
+        oldElements.getSize(), &(oldElements[0]), newEntities);
+  }
+  apf::NewArray<double> fieldVal;
   apf::Field* logMField;
   LogMEval logMEval;
 };
diff --git a/ma/maSize.h b/ma/maSize.h
index f3daec934..821fc9114 100644
--- a/ma/maSize.h
+++ b/ma/maSize.h
@@ -36,6 +36,12 @@ class SizeField
         Vector const& xi,
         Matrix& t) = 0;
     virtual double getWeight(Entity* e) = 0;
+    virtual void onRefine(
+        Entity* parent,
+        EntityArray& newEntities) = 0;
+    virtual void onCavity(
+        EntityArray& oldElements,
+        EntityArray& newEntities) = 0;
 };
 
 struct IdentitySizeField : public SizeField
@@ -53,7 +59,14 @@ struct IdentitySizeField : public SizeField
           Vector const&,
           Matrix& t);
   double getWeight(Entity*);
+  virtual void onRefine(
+      Entity* parent,
+      EntityArray& newEntities);
+  virtual void onCavity(
+      EntityArray& oldElements,
+      EntityArray& newEntities);
   Mesh* mesh;
+
 };
 
 struct UniformRefiner : public IdentitySizeField

From 1fa42363efd55774ca47b59357ba117f2b2f0531 Mon Sep 17 00:00:00 2001
From: Avinash <moharana.avin@gmail.com>
Date: Sun, 1 Nov 2020 22:37:19 -0500
Subject: [PATCH 254/555] fixes onRefine and onCavity declaration for Isotropic
 sizeField class.

---
 ma/maSize.h | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/ma/maSize.h b/ma/maSize.h
index 821fc9114..67deba872 100644
--- a/ma/maSize.h
+++ b/ma/maSize.h
@@ -42,6 +42,7 @@ class SizeField
     virtual void onCavity(
         EntityArray& oldElements,
         EntityArray& newEntities) = 0;
+    virtual void getTransferDimension() = 0;
 };
 
 struct IdentitySizeField : public SizeField
@@ -59,12 +60,13 @@ struct IdentitySizeField : public SizeField
           Vector const&,
           Matrix& t);
   double getWeight(Entity*);
-  virtual void onRefine(
+  void onRefine(
       Entity* parent,
       EntityArray& newEntities);
-  virtual void onCavity(
+  void onCavity(
       EntityArray& oldElements,
       EntityArray& newEntities);
+
   Mesh* mesh;
 
 };

From 6aec619e887f9ff9e0d67433b23589459a9f2be1 Mon Sep 17 00:00:00 2001
From: Avinash <moharana.avin@gmail.com>
Date: Sun, 1 Nov 2020 22:40:14 -0500
Subject: [PATCH 255/555] removes getTransferDimension() declaration in
 sizeField header.

---
 ma/maSize.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ma/maSize.h b/ma/maSize.h
index 67deba872..099763b90 100644
--- a/ma/maSize.h
+++ b/ma/maSize.h
@@ -42,7 +42,6 @@ class SizeField
     virtual void onCavity(
         EntityArray& oldElements,
         EntityArray& newEntities) = 0;
-    virtual void getTransferDimension() = 0;
 };
 
 struct IdentitySizeField : public SizeField

From 13bf7d62da9cc9f400a1b06704bc2c2fb7123624 Mon Sep 17 00:00:00 2001
From: Avinash <moharana.avin@gmail.com>
Date: Mon, 2 Nov 2020 01:23:30 -0500
Subject: [PATCH 256/555] implments higher order sizeField transfer on mesh
 adapt.

---
 ma/maAdapt.cc  | 10 ++++++++++
 ma/maAdapt.h   |  2 ++
 ma/maRefine.cc |  7 +++++++
 ma/maSize.cc   | 19 +++++++++++++++++++
 ma/maSize.h    |  2 ++
 5 files changed, 40 insertions(+)

diff --git a/ma/maAdapt.cc b/ma/maAdapt.cc
index 65c011a66..8a62234ce 100644
--- a/ma/maAdapt.cc
+++ b/ma/maAdapt.cc
@@ -353,12 +353,14 @@ Cavity::Cavity()
   adapter = 0;
   solutionTransfer = 0;
   shape = 0;
+  shouldTransferSizeField = false;
 }
 
 void Cavity::init(Adapt* a)
 {
   adapter = a;
   solutionTransfer = a->solutionTransfer;
+  sizeField = a->sizeField;
   shape = a->shape;
   Mesh* m = a->mesh;
   shouldTransfer = false;
@@ -368,6 +370,8 @@ void Cavity::init(Adapt* a)
       shouldTransfer = true;
     if (shape->hasNodesOn(d))
       shouldFit = true;
+    if (sizeField->getTransferDimension())
+      shouldTransferSizeField = true;
   }
 }
 
@@ -409,6 +413,12 @@ void Cavity::transfer(EntityArray& oldElements)
     newEntities.retrieve(a);
     solutionTransfer->onCavity(oldElements,a);
   }
+  if (shouldTransferSizeField)
+  {
+    EntityArray a;
+    newEntities.retrieve(a);
+    sizeField->onCavity(oldElements,a);
+  }
 }
 
 void Cavity::fit(EntityArray& oldElements)
diff --git a/ma/maAdapt.h b/ma/maAdapt.h
index c41675422..c1ddab7c0 100644
--- a/ma/maAdapt.h
+++ b/ma/maAdapt.h
@@ -137,11 +137,13 @@ class Cavity
     void fit(EntityArray& oldElements);
     bool shouldTransfer;
     bool shouldFit;
+    bool shouldTransferSizeField;
   private:
     Adapt* adapter;
     SolutionTransfer* solutionTransfer;
     ShapeHandler* shape;
     NewEntities newEntities;
+    SizeField* sizeField;
 };
 
 Entity* buildVertex(
diff --git a/ma/maRefine.cc b/ma/maRefine.cc
index fbdda77b7..df7f5c0ed 100644
--- a/ma/maRefine.cc
+++ b/ma/maRefine.cc
@@ -357,6 +357,13 @@ void transferElements(Refine* r)
       for (size_t i=0; i < r->toSplit[d].getSize(); ++i)
 	st->onRefine(r->toSplit[d][i],r->newEntities[d][i]);
   }
+  {
+    SizeField* sf = a->sizeField;
+    int td = sf->getTransferDimension();
+    for (int d = td; d <= m->getDimension(); ++d)
+      for (size_t i=0; i < r->toSplit[d].getSize(); ++i)
+	sf->onRefine(r->toSplit[d][i],r->newEntities[d][i]);
+  }
 }
 
 void forgetNewEntities(Refine* r)
diff --git a/ma/maSize.cc b/ma/maSize.cc
index 26520daa0..c3a90cb20 100644
--- a/ma/maSize.cc
+++ b/ma/maSize.cc
@@ -74,6 +74,11 @@ void IdentitySizeField::onCavity(EntityArray&, EntityArray&)
 {
 }
 
+int IdentitySizeField::getTransferDimension()
+{
+  return 0;
+}
+
 static void orthogonalizeR(Matrix& R)
 {
   /* by the way, the principal direction vectors
@@ -428,6 +433,10 @@ struct AnisoSizeField : public MetricSizeField
   void onCavity(EntityArray&, EntityArray&)
   {
   }
+  int getTransferDimension()
+  {
+    return 0;
+  }
   apf::Field* hField;
   apf::Field* rField;
   BothEval bothEval;
@@ -540,6 +549,16 @@ struct LogAnisoSizeField : public MetricSizeField
     transfer(logMField, &(fieldVal[0]),
         oldElements.getSize(), &(oldElements[0]), newEntities);
   }
+  int getTransferDimension()
+  {
+    int transferDimension = 4;
+    for (int d = 1; d <=3; d++)
+      if (apf::getShape(logMField)->hasNodesIn(d)) {
+        transferDimension = d;
+        break;
+      }
+    return transferDimension;
+  }
   apf::NewArray<double> fieldVal;
   apf::Field* logMField;
   LogMEval logMEval;
diff --git a/ma/maSize.h b/ma/maSize.h
index 099763b90..cefb55236 100644
--- a/ma/maSize.h
+++ b/ma/maSize.h
@@ -42,6 +42,7 @@ class SizeField
     virtual void onCavity(
         EntityArray& oldElements,
         EntityArray& newEntities) = 0;
+    virtual int getTransferDimension() = 0;
 };
 
 struct IdentitySizeField : public SizeField
@@ -65,6 +66,7 @@ struct IdentitySizeField : public SizeField
   void onCavity(
       EntityArray& oldElements,
       EntityArray& newEntities);
+  int getTransferDimension();
 
   Mesh* mesh;
 

From 73c16eae073b179e45ec625a5abe4915bc589af2 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Mon, 2 Nov 2020 13:17:50 -0500
Subject: [PATCH 257/555] pumi version 2.2.5

see issue #326
---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 567c5fe9c..7d10f3780 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -6,7 +6,7 @@ endif()
 # This is the top level CMake file for the SCOREC build
 cmake_minimum_required(VERSION 3.0)
 
-project(SCOREC VERSION 2.2.4 LANGUAGES CXX C)
+project(SCOREC VERSION 2.2.5 LANGUAGES CXX C)
 
 include(cmake/bob.cmake)
 include(cmake/xsdk.cmake)

From 8f2ee41df3ba7aacdbbc67a3e0433f2833307e67 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Mon, 2 Nov 2020 19:46:35 -0500
Subject: [PATCH 258/555] Adds minor changes to H1 shapes.

---
 apf/apfH1Shapes.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/apf/apfH1Shapes.cc b/apf/apfH1Shapes.cc
index fa80c2008..50d9d4deb 100644
--- a/apf/apfH1Shapes.cc
+++ b/apf/apfH1Shapes.cc
@@ -23,7 +23,7 @@ namespace apf {
 // This is used for static tables only.
 // The following implementation are for general orders but template classes
 // are only instantiated up to order 10 for now.
-static unsigned const MAX_ORDER = 10;
+static const int MAX_ORDER = 10;
 
 static inline int countEdgeNodes(int P)
 {
@@ -664,7 +664,7 @@ apf::FieldShape* getH1Shape(int order)
 {
   PCU_ALWAYS_ASSERT_VERBOSE(order > 0,
       "order is expected to be bigger than or equal to 1!");
-  PCU_ALWAYS_ASSERT_VERBOSE(order <= 10,
+  PCU_ALWAYS_ASSERT_VERBOSE(order <= MAX_ORDER,
       "order is expected to be less than or equal to 10!");
   // Note: to have higher order H1 fields all you need to do is to
   // instantiate the class  up to that order in the following table

From 166b1d0f807ac1478b54eed964e2901c2519a85d Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Mon, 2 Nov 2020 19:48:15 -0500
Subject: [PATCH 259/555] Updates projectNedelecField

Nedelec fields can now be projected to higher order (e.g., H1) fields.
---
 apf/apfNedelec.cc | 85 +++++++++++++++++++++++++++++------------------
 1 file changed, 52 insertions(+), 33 deletions(-)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index a8c9d2faa..4939f4971 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -20,6 +20,12 @@
 
 namespace apf {
 
+// MAX_ND_ORDER is used for static tables.
+// The following implementations are for general orders but template classes
+// are only instantiated for up to order 10
+
+static const int MAX_ND_ORDER = 10;
+
 static void alignFaceNodes(
     int init_order[],
     int final_order[],
@@ -177,8 +183,6 @@ static void alignFaceNodes(
   }
 }
 
-static unsigned const MAX_ND_ORDER = 10;
-
 // This is all nodes, including the nodes associated with bounding edges
 static inline int countTriNodes(int P)
 {
@@ -1124,7 +1128,7 @@ apf::FieldShape* getNedelec(int order)
 {
   PCU_ALWAYS_ASSERT_VERBOSE(order >= 1,
       "order is expected to be bigger than or equal to 1!");
-  PCU_ALWAYS_ASSERT_VERBOSE(order <= 10,
+  PCU_ALWAYS_ASSERT_VERBOSE(order <= MAX_ND_ORDER,
       "order is expected to be less than or equal to 10!");
   static Nedelec<1>  ND1;
   static Nedelec<2>  ND2;
@@ -1136,7 +1140,7 @@ apf::FieldShape* getNedelec(int order)
   static Nedelec<8>  ND8;
   static Nedelec<9>  ND9;
   static Nedelec<10> ND10;
-  static FieldShape* const nedelecShapes[10] =
+  static FieldShape* const nedelecShapes[MAX_ND_ORDER] =
   {&ND1, &ND2, &ND3, &ND4, &ND5, &ND6, &ND7, &ND8, &ND9, &ND10};
   return nedelecShapes[order-1];
 }
@@ -1145,19 +1149,17 @@ void projectNedelecField(Field* to, Field* from)
 {
   // checks on the from field
   // checks on the to field
-  apf::FieldShape* tShape = getShape(to);
-  std::string      tName  = tShape->getName();
-  int              tOrder = tShape->getOrder();
-  PCU_ALWAYS_ASSERT_VERBOSE((tName == std::string("Linear")) && (tOrder == 1),
-  		"The to field needs to be 1st order Lagrange!");
+  /* apf::FieldShape* tShape = getShape(to); */
+  /* std::string      tName  = tShape->getName(); */
+  /* int              tOrder = tShape->getOrder(); */
+  /* PCU_ALWAYS_ASSERT_VERBOSE((tName == std::string("Linear")) && (tOrder == 1), */
+  /* 		"The to field needs to be 1st order Lagrange!"); */
 
   Mesh* m = getMesh(from);
+  apf::FieldShape* fs = getShape(to);
   // auxiliary count fields
-  Field* count = createField(m, "counter", SCALAR, getLagrange(1));
-  double xis[4][3] = {{0., 0., 0.},
-  	              {1., 0., 0.},
-  	              {0., 1., 0.},
-  	              {0., 0., 1.}};
+  Field* count = createField(m, "counter", SCALAR, fs);
+
   // zero out the fields
   zeroField(to);
   zeroField(count);
@@ -1167,18 +1169,29 @@ void projectNedelecField(Field* to, Field* from)
   while( (e = m->iterate(it)) ) {
     MeshElement* me = createMeshElement(m, e);
     Element* el = createElement(from, me);
-    MeshEntity* dvs[4];
-    m->getDownward(e, 0, dvs);
-    for (int i=0; i<4; i++) {
-      Vector3 atXi;
-      getVector(el, Vector3(xis[i]), atXi);
-      Vector3 currentVal;
-      getVector(to, dvs[i], 0, currentVal);
-      double currentCount = getScalar(count, dvs[i], 0);
-      currentVal += atXi;
-      currentCount += 1.;
-      setVector(to, dvs[i], 0, currentVal);
-      setScalar(count, dvs[i], 0, currentCount);
+    for (int d = 0; d <= m->getDimension(); d++) {
+      if (!fs->hasNodesIn(d)) continue;
+      MeshEntity* down[12];
+      int nd = m->getDownward(e, d, down);
+      for (int i=0; i<nd; i++) {
+        int type = m->getType(down[i]);
+        int non = fs->countNodesOn(type);
+        for (int j = 0; j < non; j++) {
+          Vector3 nodeXiChild;
+          fs->getNodeXi(type, j, nodeXiChild);
+          Vector3 nodeXiParent = boundaryToElementXi(
+              m, down[i], e, nodeXiChild);
+          Vector3 atXi;
+          getVector(el, nodeXiParent, atXi);
+          Vector3 currentVal;
+          getVector(to, down[i], j, currentVal);
+          double currentCount = getScalar(count, down[i], j);
+          currentVal += atXi;
+          currentCount += 1.;
+          setVector(to, down[i], j, currentVal);
+          setScalar(count, down[i], j, currentCount);
+        }
+      }
     }
     destroyElement(el);
     destroyMeshElement(me);
@@ -1189,14 +1202,20 @@ void projectNedelecField(Field* to, Field* from)
   accumulate(to);
   accumulate(count);
 
-  it = m->begin(0);
-  while( (e = m->iterate(it)) ) {
-    Vector3 sum;
-    getVector(to, e, 0, sum);
-    setVector(to, e, 0, sum/getScalar(count, e, 0));
+  for (int d = 0; d <= m->getDimension(); d++) {
+    if (!fs->hasNodesIn(d)) continue;
+    it = m->begin(d);
+    while( (e = m->iterate(it)) ) {
+      int type = m->getType(e);
+      int non = fs->countNodesOn(type);
+      for (int j = 0; j < non; j++) {
+	Vector3 sum;
+	getVector(to, e, j, sum);
+	setVector(to, e, j, sum/getScalar(count, e, j));
+      }
+    }
+    m->end(it);
   }
-  m->end(it);
-
   // take care of entities on part boundary
   synchronize(to);
 

From ae0952ce38c1062946c28208b32eae2ba6857dff Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Thu, 5 Nov 2020 14:28:41 -0500
Subject: [PATCH 260/555] Nedelec fields can now be projected to IPShapes

---
 apf/apfNedelec.cc | 145 ++++++++++++++++++++++++++--------------------
 1 file changed, 83 insertions(+), 62 deletions(-)

diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc
index 4939f4971..deb9b7d3b 100644
--- a/apf/apfNedelec.cc
+++ b/apf/apfNedelec.cc
@@ -1147,80 +1147,101 @@ apf::FieldShape* getNedelec(int order)
 
 void projectNedelecField(Field* to, Field* from)
 {
-  // checks on the from field
-  // checks on the to field
-  /* apf::FieldShape* tShape = getShape(to); */
-  /* std::string      tName  = tShape->getName(); */
-  /* int              tOrder = tShape->getOrder(); */
-  /* PCU_ALWAYS_ASSERT_VERBOSE((tName == std::string("Linear")) && (tOrder == 1), */
-  /* 		"The to field needs to be 1st order Lagrange!"); */
-
   Mesh* m = getMesh(from);
   apf::FieldShape* fs = getShape(to);
-  // auxiliary count fields
-  Field* count = createField(m, "counter", SCALAR, fs);
-
-  // zero out the fields
-  zeroField(to);
-  zeroField(count);
-
-  MeshEntity* e;
-  MeshIterator* it = m->begin(m->getDimension());
-  while( (e = m->iterate(it)) ) {
-    MeshElement* me = createMeshElement(m, e);
-    Element* el = createElement(from, me);
-    for (int d = 0; d <= m->getDimension(); d++) {
-      if (!fs->hasNodesIn(d)) continue;
-      MeshEntity* down[12];
-      int nd = m->getDownward(e, d, down);
-      for (int i=0; i<nd; i++) {
-        int type = m->getType(down[i]);
-        int non = fs->countNodesOn(type);
-        for (int j = 0; j < non; j++) {
-          Vector3 nodeXiChild;
-          fs->getNodeXi(type, j, nodeXiChild);
-          Vector3 nodeXiParent = boundaryToElementXi(
-              m, down[i], e, nodeXiChild);
-          Vector3 atXi;
-          getVector(el, nodeXiParent, atXi);
-          Vector3 currentVal;
-          getVector(to, down[i], j, currentVal);
-          double currentCount = getScalar(count, down[i], j);
-          currentVal += atXi;
-          currentCount += 1.;
-          setVector(to, down[i], j, currentVal);
-          setScalar(count, down[i], j, currentCount);
-        }
+  std::string name(fs->getName());
+
+  apf::MeshEntity* e;
+  apf::MeshIterator* it;
+
+  // Integration Point Shapes (IPShapes) has to be treated differently
+  if (name.find(std::string("IP")) != std::string::npos)
+  {
+    int ip_order = fs->getOrder();
+    it = m->begin(m->getDimension());
+    while ( (e = m->iterate(it)) ) {
+      apf::MeshElement* me = apf::createMeshElement(m, e);
+      apf::Element*     el = apf::createElement(from, me);
+      int np = apf::countIntPoints(me, ip_order);
+      for (int i = 0; i < np; i++) {
+	apf::Vector3 xi;
+	apf::getIntPoint(me, ip_order, i, xi);
+	apf::Vector3 val;
+	apf::getVector(el, xi, val);
+	apf::setVector(to, e, i, val);
       }
+      apf::destroyElement(el);
+      apf::destroyMeshElement(me);
     }
-    destroyElement(el);
-    destroyMeshElement(me);
+    m->end(it);
   }
-  m->end(it);
+  // Other fields such as Lagrange and H1 can be treated here
+  else
+  {
+    // auxiliary count fields
+    Field* count = createField(m, "counter", SCALAR, fs);
 
-  // take care of entities on part boundary
-  accumulate(to);
-  accumulate(count);
+    // zero out the fields
+    zeroField(to);
+    zeroField(count);
 
-  for (int d = 0; d <= m->getDimension(); d++) {
-    if (!fs->hasNodesIn(d)) continue;
-    it = m->begin(d);
+    it = m->begin(m->getDimension());
     while( (e = m->iterate(it)) ) {
-      int type = m->getType(e);
-      int non = fs->countNodesOn(type);
-      for (int j = 0; j < non; j++) {
-	Vector3 sum;
-	getVector(to, e, j, sum);
-	setVector(to, e, j, sum/getScalar(count, e, j));
+      MeshElement* me = createMeshElement(m, e);
+      Element* el = createElement(from, me);
+      for (int d = 0; d <= m->getDimension(); d++) {
+	if (!fs->hasNodesIn(d)) continue;
+	MeshEntity* down[12];
+	int nd = m->getDownward(e, d, down);
+	for (int i=0; i<nd; i++) {
+	  int type = m->getType(down[i]);
+	  int non = fs->countNodesOn(type);
+	  for (int j = 0; j < non; j++) {
+	    Vector3 nodeXiChild;
+	    fs->getNodeXi(type, j, nodeXiChild);
+	    Vector3 nodeXiParent = boundaryToElementXi(
+		m, down[i], e, nodeXiChild);
+	    Vector3 atXi;
+	    getVector(el, nodeXiParent, atXi);
+	    Vector3 currentVal;
+	    getVector(to, down[i], j, currentVal);
+	    double currentCount = getScalar(count, down[i], j);
+	    currentVal += atXi;
+	    currentCount += 1.;
+	    setVector(to, down[i], j, currentVal);
+	    setScalar(count, down[i], j, currentCount);
+	  }
+	}
       }
+      destroyElement(el);
+      destroyMeshElement(me);
     }
     m->end(it);
-  }
-  // take care of entities on part boundary
-  synchronize(to);
 
-  m->removeField(count);
-  destroyField(count);
+    // take care of entities on part boundary
+    accumulate(to);
+    accumulate(count);
+
+    for (int d = 0; d <= m->getDimension(); d++) {
+      if (!fs->hasNodesIn(d)) continue;
+      it = m->begin(d);
+      while( (e = m->iterate(it)) ) {
+	int type = m->getType(e);
+	int non = fs->countNodesOn(type);
+	for (int j = 0; j < non; j++) {
+	  Vector3 sum;
+	  getVector(to, e, j, sum);
+	  setVector(to, e, j, sum/getScalar(count, e, j));
+	}
+      }
+      m->end(it);
+    }
+    // take care of entities on part boundary
+    synchronize(to);
+
+    m->removeField(count);
+    destroyField(count);
+  }
 }
 
 }

From 95449123fa78ea304cd4482d288dad402ca75553 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Thu, 5 Nov 2020 20:56:11 -0500
Subject: [PATCH 261/555] Adds a few new apis to python_wrappers

---
 python_wrappers/apf.i | 78 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 78 insertions(+)

diff --git a/python_wrappers/apf.i b/python_wrappers/apf.i
index 2e2d77031..caad92fc1 100644
--- a/python_wrappers/apf.i
+++ b/python_wrappers/apf.i
@@ -162,6 +162,84 @@ void lion_set_verbosity(int lvl);
     }
     return sum/count;
   }
+  apf::Field* getCurrentIsoSize(const char* name)
+  {
+    apf::Field* currentSize = apf::createField(
+        self, name, apf::SCALAR, apf::getLagrange(1));
+
+    apf::Field* cnt = apf::createField(
+        self, "current_size_cnt", apf::SCALAR, apf::getLagrange(1));
+
+    apf::zeroField(currentSize);
+    apf::zeroField(cnt);
+
+
+    apf::MeshEntity* e;
+    apf::MeshIterator* it = self->begin(0);
+    while ( (e = self->iterate(it)) ) {
+      double local_sum = 0.;
+      double local_cnt = 0.;
+      for (int i = 0; i < self->countUpward(e); i++) {
+        local_sum += apf::measure(self, self->getUpward(e, i));
+        local_cnt += 1.0;
+      }
+      apf::setScalar(currentSize, e, 0, local_sum);
+      apf::setScalar(cnt, e, 0, local_cnt);
+
+    }
+    self->end(it);
+
+    apf::accumulate(currentSize);
+    apf::accumulate(cnt);
+
+    it = self->begin(0);
+    while ( (e = self->iterate(it)) ) {
+      if (!self->isOwned(e)) continue;
+      double sum = apf::getScalar(currentSize, e, 0);
+      double count = apf::getScalar(cnt, e, 0);
+      apf::setScalar(currentSize, e, 0, sum/count);
+    }
+    self->end(it);
+
+    apf::synchronize(currentSize);
+    self->removeField(cnt);
+    apf::destroyField(cnt);
+    return currentSize;
+  }
+  double getMinOfScalarField(apf::Field* field)
+  {
+    PCU_ALWAYS_ASSERT(apf::getValueType(field) == apf::SCALAR);
+    double local_min = 1.0e32;
+    apf::MeshEntity* e;
+    apf::MeshIterator* it = self->begin(0);
+    while ( (e = self->iterate(it)) ) {
+      if (!self->isOwned(e))
+        continue;
+      double val = apf::getScalar(field, e, 0);
+      if (val < local_min)
+        local_min = val;
+    }
+    self->end(it);
+    PCU_Min_Doubles(&local_min, 1);
+    return local_min;
+  }
+  double getMaxOfScalarField(apf::Field* field)
+  {
+    PCU_ALWAYS_ASSERT(apf::getValueType(field) == apf::SCALAR);
+    double local_max = -1.0e32;
+    apf::MeshEntity* e;
+    apf::MeshIterator* it = self->begin(0);
+    while ( (e = self->iterate(it)) ) {
+      if (!self->isOwned(e))
+        continue;
+      double val = apf::getScalar(field, e, 0);
+      if (val > local_max)
+        local_max = val;
+    }
+    self->end(it);
+    PCU_Max_Doubles(&local_max, 1);
+    return local_max;
+  }
   bool isBoundingModelRegion(int rtag, int dim, int tag)
   {
     if (dim != 2) return false;

From e460dba423f75bfcc8ea18aca140395e0db6646c Mon Sep 17 00:00:00 2001
From: Avinash <moharana.avin@gmail.com>
Date: Fri, 6 Nov 2020 19:28:17 -0500
Subject: [PATCH 262/555] removed declaration of sizeField in Cavity class +
 implements hasNodesOn for sizeField.

---
 ma/maAdapt.cc |  5 ++---
 ma/maAdapt.h  |  1 -
 ma/maSize.cc  | 49 +++++++++++++++++++++++++++++++------------------
 ma/maSize.h   |  2 ++
 4 files changed, 35 insertions(+), 22 deletions(-)

diff --git a/ma/maAdapt.cc b/ma/maAdapt.cc
index 8a62234ce..606e095d5 100644
--- a/ma/maAdapt.cc
+++ b/ma/maAdapt.cc
@@ -360,7 +360,6 @@ void Cavity::init(Adapt* a)
 {
   adapter = a;
   solutionTransfer = a->solutionTransfer;
-  sizeField = a->sizeField;
   shape = a->shape;
   Mesh* m = a->mesh;
   shouldTransfer = false;
@@ -370,7 +369,7 @@ void Cavity::init(Adapt* a)
       shouldTransfer = true;
     if (shape->hasNodesOn(d))
       shouldFit = true;
-    if (sizeField->getTransferDimension())
+    if (adapter->sizeField->hasNodesOn(d))
       shouldTransferSizeField = true;
   }
 }
@@ -417,7 +416,7 @@ void Cavity::transfer(EntityArray& oldElements)
   {
     EntityArray a;
     newEntities.retrieve(a);
-    sizeField->onCavity(oldElements,a);
+    adapter->sizeField->onCavity(oldElements,a);
   }
 }
 
diff --git a/ma/maAdapt.h b/ma/maAdapt.h
index c1ddab7c0..81a943e0a 100644
--- a/ma/maAdapt.h
+++ b/ma/maAdapt.h
@@ -143,7 +143,6 @@ class Cavity
     SolutionTransfer* solutionTransfer;
     ShapeHandler* shape;
     NewEntities newEntities;
-    SizeField* sizeField;
 };
 
 Entity* buildVertex(
diff --git a/ma/maSize.cc b/ma/maSize.cc
index c3a90cb20..2825c3f69 100644
--- a/ma/maSize.cc
+++ b/ma/maSize.cc
@@ -79,6 +79,12 @@ int IdentitySizeField::getTransferDimension()
   return 0;
 }
 
+bool IdentitySizeField::hasNodesOn(int dimension)
+{
+  std::ignore = (dimension);
+  return false;
+}
+
 static void orthogonalizeR(Matrix& R)
 {
   /* by the way, the principal direction vectors
@@ -437,6 +443,11 @@ struct AnisoSizeField : public MetricSizeField
   {
     return 0;
   }
+  bool hasNodesOn(int dimension)
+  {
+    std::ignore = (dimension);
+    return false;
+  }
   apf::Field* hField;
   apf::Field* rField;
   BothEval bothEval;
@@ -471,25 +482,23 @@ struct LogAnisoSizeField : public MetricSizeField
     for (int d = 0; d <= dim; d++) {
       if (!apf::getShape(logMField)->countNodesOn(apf::Mesh::simplexTypes[d]))
         continue;
-      else {
-        it = m->begin(d);
-        while( (ent = m->iterate(it)) ){
-	  int type = m->getType(ent);
-	  int non = apf::getShape(logMField)->countNodesOn(type);
-	  for (int i = 0; i < non; i++) {
-	    Vector h;
-	    Matrix f;
-	    apf::getVector(sizes, ent, i, h);
-	    apf::getMatrix(frames, ent, i, f);
-	    Vector s(log(1/h[0]/h[0]), log(1/h[1]/h[1]), log(1/h[2]/h[2]));
-	    Matrix S(s[0], 0   , 0,
-		0    , s[1], 0,
-		0    , 0   , s[2]);
-	    apf::setMatrix(logMField, ent, i, f * S * transpose(f));
-	  }
+      it = m->begin(d);
+      while( (ent = m->iterate(it)) ){
+	int type = m->getType(ent);
+	int non = apf::getShape(logMField)->countNodesOn(type);
+	for (int i = 0; i < non; i++) {
+	  Vector h;
+	  Matrix f;
+	  apf::getVector(sizes, ent, i, h);
+	  apf::getMatrix(frames, ent, i, f);
+	  Vector s(log(1/h[0]/h[0]), log(1/h[1]/h[1]), log(1/h[2]/h[2]));
+	  Matrix S(s[0], 0   , 0,
+	      0    , s[1], 0,
+	      0    , 0   , s[2]);
+	  apf::setMatrix(logMField, ent, i, f * S * transpose(f));
 	}
-	m->end(it);
       }
+      m->end(it);
     }
     fieldVal.allocate(apf::countComponents(logMField));
   }
@@ -553,12 +562,16 @@ struct LogAnisoSizeField : public MetricSizeField
   {
     int transferDimension = 4;
     for (int d = 1; d <=3; d++)
-      if (apf::getShape(logMField)->hasNodesIn(d)) {
+      if (hasNodesOn(d)) {
         transferDimension = d;
         break;
       }
     return transferDimension;
   }
+  bool hasNodesOn(int dimension)
+  {
+    return apf::getShape(logMField)->hasNodesIn(dimension);
+  }
   apf::NewArray<double> fieldVal;
   apf::Field* logMField;
   LogMEval logMEval;
diff --git a/ma/maSize.h b/ma/maSize.h
index cefb55236..f229412ee 100644
--- a/ma/maSize.h
+++ b/ma/maSize.h
@@ -43,6 +43,7 @@ class SizeField
         EntityArray& oldElements,
         EntityArray& newEntities) = 0;
     virtual int getTransferDimension() = 0;
+    virtual bool hasNodesOn(int dimension) = 0;
 };
 
 struct IdentitySizeField : public SizeField
@@ -59,6 +60,7 @@ struct IdentitySizeField : public SizeField
           apf::MeshElement*,
           Vector const&,
           Matrix& t);
+  bool hasNodesOn(int dimension);
   double getWeight(Entity*);
   void onRefine(
       Entity* parent,

From a540535163ebfda8d47636c6243b98a8fe71c5ca Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Sat, 7 Nov 2020 16:38:07 -0500
Subject: [PATCH 263/555] Exposes curve adapt options in python wrappers

---
 python_wrappers/CMakeLists.txt |  1 +
 python_wrappers/apf.i          | 16 ++++++++++++++++
 2 files changed, 17 insertions(+)

diff --git a/python_wrappers/CMakeLists.txt b/python_wrappers/CMakeLists.txt
index 074b0f388..1a16cd3e0 100644
--- a/python_wrappers/CMakeLists.txt
+++ b/python_wrappers/CMakeLists.txt
@@ -44,6 +44,7 @@ set_property(TARGET pyCore
   ${CMAKE_SOURCE_DIR}/ma
   ${CMAKE_SOURCE_DIR}/can
   ${CMAKE_SOURCE_DIR}/spr
+  ${CMAKE_SOURCE_DIR}/crv
 )
 
 set(pyCoreDepLibs
diff --git a/python_wrappers/apf.i b/python_wrappers/apf.i
index caad92fc1..63f49565e 100644
--- a/python_wrappers/apf.i
+++ b/python_wrappers/apf.i
@@ -27,6 +27,7 @@
 #include <spr.h>
 #include <maInput.h>
 #include <ma.h>
+#include <crv.h>
 
 #ifdef HAVE_SIMMETRIX
   #include <sim_helper.h>
@@ -310,3 +311,18 @@ namespace ma {
   void adapt(Input* in);
   void adaptVerbose(Input* in, bool verbosef = false);
 }
+
+/* CRV RELATED WRAPPERS */
+%rename(crvadapt) crv::adapt;
+namespace crv {
+  void adapt(ma::Input* in);
+  class BezierCurver : public MeshCurver
+  {
+    public:
+      BezierCurver(apf::Mesh2* m, int P, int B) : MeshCurver(m,P)
+      {
+        setBlendingOrder(apf::Mesh::TYPES,B);
+      };
+      virtual bool run();
+  };
+}

From 6cea22ba9797315123ac3503ddd02c708ed0300b Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Sun, 8 Nov 2020 08:41:57 -0500
Subject: [PATCH 264/555] Exposes cruved_vtk_writers for python

---
 python_wrappers/apf.i | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/python_wrappers/apf.i b/python_wrappers/apf.i
index 63f49565e..68cc8372e 100644
--- a/python_wrappers/apf.i
+++ b/python_wrappers/apf.i
@@ -325,4 +325,6 @@ namespace crv {
       };
       virtual bool run();
   };
+  void writeCurvedVtuFiles(apf::Mesh* m, int type, int n, const char* prefix);
+  void writeCurvedWireFrame(apf::Mesh* m, int n, const char* prefix);
 }

From 1b30311b74af9c88c19a126d234ef9dbf648989e Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 10 Nov 2020 20:33:14 -0500
Subject: [PATCH 265/555] Updates H1Shape test to test the matrix fields too

---
 test/H1Shapes.cc | 121 +++++++++++++++++++++++++++++++++++++----------
 1 file changed, 95 insertions(+), 26 deletions(-)

diff --git a/test/H1Shapes.cc b/test/H1Shapes.cc
index 7c4878692..0ac0e8c9e 100644
--- a/test/H1Shapes.cc
+++ b/test/H1Shapes.cc
@@ -18,11 +18,13 @@
 
 // User defined vector functions E(x,y,z) of order up to 6
 void E_exact(const apf::Vector3& x, apf::Vector3& value, int p);
+void M_exact(const apf::Vector3& x, apf::Vector3& value, int p);
 
 void testH1(
     apf::Mesh2* m,
     const apf::Vector3& testXi,
-    int ndOrder, int exactOrder);
+    int ndOrder, int exactOrder,
+    int dataType); /* apf::VECTOR or apf::MATRIX */
 
 int main(int argc, char** argv)
 {
@@ -45,12 +47,28 @@ int main(int argc, char** argv)
   apf::Mesh2* m = apf::loadMdsMesh(g,argv[2]);
   m->verify();
 
+  // test fields interpolating a user-defined vector field
   for (int i = 1; i <= 6; i++) {
+    if(0==PCU_Comm_Self())
+      lion_oprint(1, "----TESTING VECTOR FIELD OF ORDER %d----\n", i);
     testH1(
-    	m, /* mesh */
-	apf::Vector3(1./7., 1./11., 1./3.), /* test point */
-	i, /* order of h1 field */
-	i); /* order of test field */
+        m, /* mesh */
+        apf::Vector3(1./7., 1./11., 1./3.), /* test point */
+        i, /* order of h1 field */
+        i, /* order of test field */
+        apf::VECTOR);
+  }
+
+  // test fields interpolating a user-defined matrix field
+  for (int i = 1; i <= 6; i++) {
+    if(0==PCU_Comm_Self())
+      lion_oprint(1, "----TESTING MATRIX FIELD OF ORDER %d----\n", i);
+    testH1(
+        m, /* mesh */
+        apf::Vector3(1./7., 1./11., 1./3.), /* test point */
+        i, /* order of h1 field */
+        i, /* order of test field */
+        apf::MATRIX);
   }
 
   apf::destroyMesh(m);
@@ -75,15 +93,48 @@ void E_exact(const apf::Vector3& x, apf::Vector3& value, int p)
   }
 }
 
+void M_exact(const apf::Vector3& x, apf::Matrix3x3& mat, int p)
+{
+  // Polynomial coefficients for each component of exact vector field
+  double a[7] = { 1.0, -1.0,  2., -2., -1.0,  1.0, -1.0};
+  double b[7] = {-2.0,  1.0, -2.,  2., -1.0, -1.0,  1.0};
+  double c[7] = { 3.0,  0.0, -1.,  100., -1.0,  1.0,  0.0};
+
+  apf::Vector3 value;
+  value[0] = 0.0;
+  value[1] = 0.0;
+  value[2] = 0.0;
+  for (int i = p;  i >= 0; i--) {
+    value[0] += pow(x[0],p)*a[p];
+    value[1] += pow(x[1],p)*b[p];
+    value[2] += pow(x[2],p)*c[p];
+  }
+
+  mat[0] = value;
+  mat[1] = apf::Vector3(value[2], 1., 0.);
+  mat[2] = apf::Vector3(0., value[0], 1.);
+}
+
+
+
 void testH1(
     apf::Mesh2* m,
     const apf::Vector3& testXi,
-    int ndOrder, int exactOrder)
+    int ndOrder, int exactOrder,
+    int dataType)
 {
+  PCU_ALWAYS_ASSERT(dataType == apf::VECTOR || dataType == apf::MATRIX);
   // Loop over all nodes and set scalar dofs.
   int dim = m->getDimension();
-  apf::Field* h1Field = apf::createField(
+  apf::Field* h1Field;
+
+  if (dataType == apf::VECTOR)
+    h1Field = apf::createField(
       m, "h1_test", apf::VECTOR, apf::getH1Shape(ndOrder));
+  else
+    h1Field = apf::createField(
+      m, "h1_test", apf::MATRIX, apf::getH1Shape(ndOrder));
+
   apf::MeshEntity* ent;
   apf::MeshIterator* it;
 
@@ -104,12 +155,19 @@ void testH1(
       apf::MeshElement* me = apf::createMeshElement(m, ent);
       for (int i = 0; i < non; i++)
       {
-        apf::Vector3 xi, p, value;
+        apf::Vector3 xi, p;
         h1Field->getShape()->getNodeXi(type, i, xi);
         apf::mapLocalToGlobal(me, xi, p);
-        E_exact(p, value, exactOrder);
-
-      	apf::setVector(h1Field, ent, i, value);
+        if (dataType == apf::VECTOR) {
+          apf::Vector3 value;
+          E_exact(p, value, exactOrder);
+          apf::setVector(h1Field, ent, i, value);
+        }
+        else {
+          apf::Matrix3x3 mat;
+          M_exact(p, mat, exactOrder);
+          apf::setMatrix(h1Field, ent, i, mat);
+        }
       }
       apf::destroyMeshElement(me);
       count++;
@@ -121,34 +179,45 @@ void testH1(
   // Verify that interpolated solution field agrees with exact field.
   for (int d = 0; d <= dim; d++) {
 
-    double L2ErrorE = 0.;
+    double L2Error = 0.;
     it = m->begin(d);
     int count = 0;
     while( (ent = m->iterate(it)) ) {
       apf::MeshElement* me = apf::createMeshElement(m, ent);
       apf::Vector3 x;
       apf::mapLocalToGlobal(me, testXi, x);
-      apf::Vector3 eFieldExact;
-      E_exact(x, eFieldExact, exactOrder);
-
-      // obtain interpolated value
       apf::Element* el = apf::createElement(h1Field, me);
-      apf::Vector3 eFieldValue;
-      apf::getVector(el, testXi, eFieldValue);
-
-      double err = ((eFieldValue - eFieldExact) * (eFieldValue - eFieldExact));
-      err /= (eFieldExact * eFieldExact); // normalization factor
-      L2ErrorE += err;
-      apf::destroyMeshElement(me);
+      double err;
+      if (dataType == apf::VECTOR) {
+        apf::Vector3 eFieldExact;
+        E_exact(x, eFieldExact, exactOrder);
+        apf::Vector3 eFieldValue;
+        apf::getVector(el, testXi, eFieldValue);
+        err = ((eFieldValue - eFieldExact) * (eFieldValue - eFieldExact));
+        err /= (eFieldExact * eFieldExact); // normalization factor
+      }
+      else {
+        apf::Matrix3x3 mFieldExact;
+        M_exact(x, mFieldExact, exactOrder);
+        apf::Matrix3x3 mFieldValue;
+        apf::getMatrix(el, testXi, mFieldValue);
+        err = apf::getInnerProduct(
+            mFieldValue - mFieldExact,
+            mFieldValue - mFieldExact);
+        err /= apf::getInnerProduct(mFieldExact, mFieldExact);
+      }
+      L2Error += err;
       apf::destroyElement(el);
+      apf::destroyMeshElement(me);
       count++;
     }
     m->end(it);
 
     // check for field interpolation
-    if(0==PCU_Comm_Self())
-      lion_oprint(1, "L2ErrorE for entities of dimension %d is %e\n", d, L2ErrorE);
-    PCU_ALWAYS_ASSERT_VERBOSE(L2ErrorE < 1.e-12,
+    if(0==PCU_Comm_Self()) {
+      lion_oprint(1, "L2Error for entities of dimension %d is %e\n", d, L2Error);
+    }
+    PCU_ALWAYS_ASSERT_VERBOSE(L2Error < 1.e-12,
 	"Fields were not interpolated correctly!");
 
   }

From 5898aa3e4e4b00afc09e497423b0bbaaf02ac996 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 10 Nov 2020 20:34:49 -0500
Subject: [PATCH 266/555] Removes the obsolete comment

---
 apf/apfFieldData.cc | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/apf/apfFieldData.cc b/apf/apfFieldData.cc
index 5d46a5f59..940cd3d45 100644
--- a/apf/apfFieldData.cc
+++ b/apf/apfFieldData.cc
@@ -199,10 +199,6 @@ void FieldDataOf<T>::getNodeComponents(MeshEntity* e, int node, T* components)
     components[i] = allComponents[node*nc+i];
 }
 
-// Note that the values in the array "order" are allowed to be negative
-// for cases with vector shape functions such as Nedelec Shapes.
-// For such cases the absolute value of (order+1) is used to locate the data,
-// and the sign is multiplied to the data value at that location.
 template <class T>
 void reorderData(T const dataIn[], T dataOut[], int const order[], int nc, int nn)
 {

From 08f4a3193ce969712b18e7184d6b7c0d8352e4da Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 10 Nov 2020 20:46:02 -0500
Subject: [PATCH 267/555] Fixes a bug in getElementData

Previously, for efficiency reasons, the size of the intermediate array
"adata" was set ot "nen" (the total number of nodes associated with the
entity). This is not enough for cases where the number of components for
the fields is large. For example for a cubic triangle nen is 10 (3 vertex nodes,
6 edge nodes, and 1 face node). Now for the same triangle "adata" for each edge
will need the min size of 18 if the underlying field type is matrix (9 components
per node x 2 nodes).
---
 apf/apfFieldData.cc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/apf/apfFieldData.cc b/apf/apfFieldData.cc
index 940cd3d45..2c3f1156a 100644
--- a/apf/apfFieldData.cc
+++ b/apf/apfFieldData.cc
@@ -297,6 +297,12 @@ int FieldDataOf<T>::getElementData(MeshEntity* entity, NewArray<T>& data)
 	if (nan > 1 && ed != d) { /* multiple shared nodes, check alignment */
 	  order.setSize(nen); /* nen >= nan */
 	  adata.setSize(nen); /* setSize is no-op for the same size */
+	  // Note: The above efficiency consideration does not account for the
+	  // fact that nc might be very large (e.g. nc = 9 for matrix fields)
+	  // and for such cases setting the size of adata to "nen" is not enough.
+	  // Hence the need for the following line.
+	  if (nan*nc > nen)
+	    adata.setSize(nan*nc);
 	  es->alignSharedNodes(mesh, entity, a[i], &order[0]);
 	  get(a[i], &adata[0]);
 	  reorderData<T>(&adata[0], &data[n], &order[0], nc, nan);

From f77fee2e544139b077d56db15abf28f60642842f Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 10 Nov 2020 20:54:55 -0500
Subject: [PATCH 268/555] Moves the c-tor/d-tor for MatrixElement to
 apfMatrixElement.cc

---
 apf/apfMatrixElement.cc | 11 +++++++++++
 apf/apfMatrixField.cc   |  9 ---------
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/apf/apfMatrixElement.cc b/apf/apfMatrixElement.cc
index 6c6065d3c..f609f912e 100644
--- a/apf/apfMatrixElement.cc
+++ b/apf/apfMatrixElement.cc
@@ -5,10 +5,21 @@
  * BSD license as described in the LICENSE file in the top-level directory.
  */
 
+#include "apfMatrixField.h"
 #include "apfMatrixElement.h"
 #include "apfVectorElement.h"
 
 namespace apf {
+
+MatrixElement::MatrixElement(MatrixField* f, MeshElement* e):
+    ElementOf<Matrix3x3>(f,e)
+{
+}
+
+MatrixElement::~MatrixElement()
+{
+}
+
 // laid out in array as F_i*3+j+9*d
 void MatrixElement::grad(Vector3 const& xi, Vector<27>& g)
 {
diff --git a/apf/apfMatrixField.cc b/apf/apfMatrixField.cc
index aa4bd4e8f..600acfa07 100644
--- a/apf/apfMatrixField.cc
+++ b/apf/apfMatrixField.cc
@@ -10,15 +10,6 @@
 
 namespace apf {
 
-MatrixElement::MatrixElement(MatrixField* f, MeshElement* e):
-    ElementOf<Matrix3x3>(f,e)
-{
-}
-
-MatrixElement::~MatrixElement()
-{
-}
-
 Element* MatrixField::getElement(VectorElement* e)
 {
   return new MatrixElement(this,e);

From 80c3e1cfd4f37b5b5a3d9302cd920d9f2a5e7ba9 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Wed, 11 Nov 2020 15:43:34 -0500
Subject: [PATCH 269/555] Removes commented code

---
 apf/apfH1Shapes.cc | 71 ----------------------------------------------
 1 file changed, 71 deletions(-)

diff --git a/apf/apfH1Shapes.cc b/apf/apfH1Shapes.cc
index 50d9d4deb..673b2b781 100644
--- a/apf/apfH1Shapes.cc
+++ b/apf/apfH1Shapes.cc
@@ -693,75 +693,4 @@ apf::FieldShape* getH1Shape(int order)
   return h1Shapes[order];
 }
 
-/* void projectL2Field(Field* to, Field* from) */
-/* { */
-/*   // checks on the from field */
-/*   // checks on the to field */
-/*   apf::FieldShape* tShape = getShape(to); */
-/*   std::string      tName  = tShape->getName(); */
-/*   int              tOrder = tShape->getOrder(); */
-/*   PCU_ALWAYS_ASSERT_VERBOSE((tName == std::string("Linear")) && (tOrder == 1), */
-/*                 "The to field needs to be 1st order Lagrange!"); */
-
-/*   Mesh* m = getMesh(from); */
-/*   // auxiliary count fields */
-/*   Field* count = createField(m, "counter", SCALAR, getLagrange(1)); */
-/*   double xis[4][3] = {{0., 0., 0.}, */
-/*                       {1., 0., 0.}, */
-/*                       {0., 1., 0.}, */
-/*                       {0., 0., 1.}}; */
-/*   // zero out the fields */
-/*   zeroField(to); */
-/*   zeroField(count); */
-
-/*   int nc = countComponents(to); */
-/*   NewArray<double> atXi(nc); */
-/*   NewArray<double> currentVal(nc); */
-/*   NewArray<double> sum(nc); */
-
-/*   MeshEntity* e; */
-/*   MeshIterator* it = m->begin(m->getDimension()); */
-/*   while( (e = m->iterate(it)) ) { */
-/*     MeshElement* me = createMeshElement(m, e); */
-/*     Element* el = createElement(from, me); */
-/*     MeshEntity* dvs[4]; */
-/*     m->getDownward(e, 0, dvs); */
-/*     for (int i=0; i<4; i++) { */
-/*       getComponents(el, Vector3(xis[i]), &(atXi[0])); */
-/*       getComponents(to, dvs[i], 0, &(currentVal[0])); */
-/*       for (int j = 0; j < nc; j++) { */
-/*         currentVal[j] += atXi[j]; */
-/*       } */
-/*       double currentCount = getScalar(count, dvs[i], 0); */
-/*       currentCount += 1.; */
-/*       setComponents(to, dvs[i], 0, &(currentVal[0])); */
-/*       setScalar(count, dvs[i], 0, currentCount); */
-/*     } */
-/*     destroyElement(el); */
-/*     destroyMeshElement(me); */
-/*   } */
-/*   m->end(it); */
-
-/*   // take care of entities on part boundary */
-/*   accumulate(to); */
-/*   accumulate(count); */
-
-/*   it = m->begin(0); */
-/*   while( (e = m->iterate(it)) ) { */
-/*     getComponents(to, e, 0, &(sum[0])); */
-/*     int cnt = getScalar(count, e, 0); */
-/*     for (int i = 0; i < nc; i++) { */
-/*       sum[i] /= cnt; */
-/*     } */
-/*     setComponents(to, e, 0, &(sum[0])); */
-/*   } */
-/*   m->end(it); */
-
-/*   // take care of entities on part boundary */
-/*   synchronize(to); */
-
-/*   m->removeField(count); */
-/*   destroyField(count); */
-/* } */
-
 }

From 7e5aa0bba36c6dab792c4e39356a47b3ccac4372 Mon Sep 17 00:00:00 2001
From: Avinash <moharana.avin@gmail.com>
Date: Mon, 16 Nov 2020 17:24:01 -0500
Subject: [PATCH 270/555] updates the transfer in FaceSplit to account for
 SizeFiled.

---
 ma/maFaceSplit.cc | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/ma/maFaceSplit.cc b/ma/maFaceSplit.cc
index 0b3ac04fe..73f6da425 100644
--- a/ma/maFaceSplit.cc
+++ b/ma/maFaceSplit.cc
@@ -123,16 +123,22 @@ void FaceSplit::cancel()
 void FaceSplit::transfer()
 {
   Mesh* m = adapter->mesh;
+  int td;
+  td = adapter->shape->getTransferDimension();
+  for (int d = td; d <= m->getDimension(); ++d)
+    for (size_t i = 0; i < toSplit[d].getSize(); ++i)
+      adapter->shape->onRefine(toSplit[d][i], newEntities[d][i]);
+
   SolutionTransfer* st = adapter->solutionTransfer;
-  int td = st->getTransferDimension();
+  td = st->getTransferDimension();
   for (int d = td; d <= m->getDimension(); ++d)
     for (size_t i = 0; i < toSplit[d].getSize(); ++i)
       st->onRefine(toSplit[d][i], newEntities[d][i]);
 
-  td = adapter->shape->getTransferDimension();
+  td = adapter->sizeField->getTransferDimension();
   for (int d = td; d <= m->getDimension(); ++d)
     for (size_t i = 0; i < toSplit[d].getSize(); ++i)
-      adapter->shape->onRefine(toSplit[d][i], newEntities[d][i]);
+      adapter->sizeField->onRefine(toSplit[d][i], newEntities[d][i]);
 }
 
 void FaceSplit::destroyOldElements()

From 7a4f450673ade186eba82821ba32ae6de882f160 Mon Sep 17 00:00:00 2001
From: Avinash <moharana.avin@gmail.com>
Date: Mon, 23 Nov 2020 10:26:55 -0500
Subject: [PATCH 271/555] using order 3 integrator for sizeField->measure

---
 ma/maSize.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ma/maSize.cc b/ma/maSize.cc
index 2825c3f69..f34df9fcb 100644
--- a/ma/maSize.cc
+++ b/ma/maSize.cc
@@ -153,7 +153,7 @@ class SizeFieldIntegrator : public apf::Integrator
 {
   public:
     SizeFieldIntegrator(SizeField* sF):
-      Integrator(2),
+      Integrator(3),
       measurement(0),
       sizeField(sF),
       meshElement(0),

From c4f914eed45339cb44853e08167d18eaa9a019ce Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Mon, 23 Nov 2020 17:13:37 -0500
Subject: [PATCH 272/555] Cleans up ma::SizeField class

---
 ma/maSize.cc | 66 +++++++++++++++++++++++-----------------------------
 ma/maSize.h  | 17 ++++----------
 2 files changed, 33 insertions(+), 50 deletions(-)

diff --git a/ma/maSize.cc b/ma/maSize.cc
index f34df9fcb..4b661fe56 100644
--- a/ma/maSize.cc
+++ b/ma/maSize.cc
@@ -21,6 +21,32 @@ SizeField::~SizeField()
 {
 }
 
+void SizeField::onRefine(Entity*, EntityArray&)
+{
+  PCU_ALWAYS_ASSERT_VERBOSE(0,
+      "unimplemented onRefine was called for a size-field!");
+}
+
+void SizeField::onCavity(EntityArray&, EntityArray&)
+{
+  PCU_ALWAYS_ASSERT_VERBOSE(0,
+      "unimplemented onCavity was called for a size-field!");
+}
+
+int SizeField::getTransferDimension()
+{
+  // By default there should be now size_field transfer.
+  // This is used in a loop to get the lowest dimension
+  // entities that require transfer. So something bigger
+  // than mesh dimension will ignore that loop
+  return 4;
+}
+
+bool SizeField::hasNodesOn(int)
+{
+  return false;
+}
+
 IdentitySizeField::IdentitySizeField(Mesh* m):
   mesh(m)
 {
@@ -66,25 +92,6 @@ double IdentitySizeField::getWeight(Entity*)
   return 1.0;
 }
 
-void IdentitySizeField::onRefine(Entity*, EntityArray&)
-{
-}
-
-void IdentitySizeField::onCavity(EntityArray&, EntityArray&)
-{
-}
-
-int IdentitySizeField::getTransferDimension()
-{
-  return 0;
-}
-
-bool IdentitySizeField::hasNodesOn(int dimension)
-{
-  std::ignore = (dimension);
-  return false;
-}
-
 static void orthogonalizeR(Matrix& R)
 {
   /* by the way, the principal direction vectors
@@ -116,14 +123,14 @@ static void orthogonalizeR(Matrix& R)
 
 static void orthogonalEigenDecompForSymmetricMatrix(Matrix const& A, Vector& v, Matrix& R)
 {
-  /* here we assume A to be real symmetric 3x3 matrix, 
+  /* here we assume A to be real symmetric 3x3 matrix,
    * we should be able to get 3 orthogonal eigen vectors
    * we also normalize the eigen vectors */
   double eigenValues[3];
   Vector eigenVectors[3];
-  
+
   apf::eigen(A, eigenVectors, eigenValues);
-  
+
   Matrix RT(eigenVectors); // eigen vectors are stored in the rows of RT
 
   RT[0] = RT[0].normalize();
@@ -433,21 +440,6 @@ struct AnisoSizeField : public MetricSizeField
                           0,0,1),
                    Vector(value,value,value));
   }
-  void onRefine(Entity*, EntityArray&)
-  {
-  }
-  void onCavity(EntityArray&, EntityArray&)
-  {
-  }
-  int getTransferDimension()
-  {
-    return 0;
-  }
-  bool hasNodesOn(int dimension)
-  {
-    std::ignore = (dimension);
-    return false;
-  }
   apf::Field* hField;
   apf::Field* rField;
   BothEval bothEval;
diff --git a/ma/maSize.h b/ma/maSize.h
index f229412ee..b48a03cfa 100644
--- a/ma/maSize.h
+++ b/ma/maSize.h
@@ -38,12 +38,12 @@ class SizeField
     virtual double getWeight(Entity* e) = 0;
     virtual void onRefine(
         Entity* parent,
-        EntityArray& newEntities) = 0;
+        EntityArray& newEntities);
     virtual void onCavity(
         EntityArray& oldElements,
-        EntityArray& newEntities) = 0;
-    virtual int getTransferDimension() = 0;
-    virtual bool hasNodesOn(int dimension) = 0;
+        EntityArray& newEntities);
+    virtual int getTransferDimension();
+    virtual bool hasNodesOn(int dimension);
 };
 
 struct IdentitySizeField : public SizeField
@@ -60,16 +60,7 @@ struct IdentitySizeField : public SizeField
           apf::MeshElement*,
           Vector const&,
           Matrix& t);
-  bool hasNodesOn(int dimension);
   double getWeight(Entity*);
-  void onRefine(
-      Entity* parent,
-      EntityArray& newEntities);
-  void onCavity(
-      EntityArray& oldElements,
-      EntityArray& newEntities);
-  int getTransferDimension();
-
   Mesh* mesh;
 
 };

From af885e3a4c3c33f031ba987040904ee616d3c2b0 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Mon, 23 Nov 2020 18:01:19 -0500
Subject: [PATCH 273/555] Updates the SizeFieldIntegrator

Uses max(mesh_order, size_field_order) + 1 as the integration order.
---
 ma/maSize.cc | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/ma/maSize.cc b/ma/maSize.cc
index 4b661fe56..9b9508a64 100644
--- a/ma/maSize.cc
+++ b/ma/maSize.cc
@@ -159,8 +159,8 @@ static double parentMeasure[apf::Mesh::TYPES] =
 class SizeFieldIntegrator : public apf::Integrator
 {
   public:
-    SizeFieldIntegrator(SizeField* sF):
-      Integrator(3),
+    SizeFieldIntegrator(SizeField* sF, int order):
+      Integrator(order),
       measurement(0),
       sizeField(sF),
       meshElement(0),
@@ -208,7 +208,8 @@ struct MetricSizeField : public SizeField
 {
   double measure(Entity* e)
   {
-    SizeFieldIntegrator sFI(this); 
+    SizeFieldIntegrator sFI(this,
+    	std::max(mesh->getShape()->getOrder(), order)+1);
     apf::MeshElement* me = apf::createMeshElement(mesh, e);
     sFI.process(me);
     apf::destroyMeshElement(me);
@@ -228,6 +229,7 @@ struct MetricSizeField : public SizeField
     return measure(e) / parentMeasure[mesh->getType(e)];
   }
   Mesh* mesh;
+  int order; // this is the underlying sizefield order
 };
 
 AnisotropicFunction::~AnisotropicFunction()
@@ -371,6 +373,7 @@ struct AnisoSizeField : public MetricSizeField
     frameEval(&bothEval)
   {
     mesh = m;
+    order = 1;
     hField = apf::createUserField(m, "ma_sizes", apf::VECTOR,
         apf::getLagrange(1), &sizesEval);
     rField = apf::createUserField(m, "ma_frame", apf::MATRIX,
@@ -384,6 +387,7 @@ struct AnisoSizeField : public MetricSizeField
   void init(Mesh* m, apf::Field* sizes, apf::Field* frames)
   {
     mesh = m;
+    order = apf::getShape(sizes)->getOrder();
     hField = sizes;
     rField = frames;
   }
@@ -456,6 +460,7 @@ struct LogAnisoSizeField : public MetricSizeField
     logMEval(f)
   {
     mesh = m;
+    order = 1;
     logMField = apf::createUserField(m, "ma_logM", apf::MATRIX,
         apf::getLagrange(1), &logMEval);
   }
@@ -466,6 +471,7 @@ struct LogAnisoSizeField : public MetricSizeField
   void init(Mesh* m, apf::Field* sizes, apf::Field* frames)
   {
     mesh = m;
+    order = apf::getShape(sizes)->getOrder();
     logMField = apf::createField(m, "ma_logM", apf::MATRIX,
         apf::getShape(sizes));
     int dim = m->getDimension();

From c19dba2f47de34e69560c7ee0b0e0e1b0a7b7504 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Mon, 23 Nov 2020 18:13:02 -0500
Subject: [PATCH 274/555] Adds member initializations for the default c-tors

---
 ma/maSize.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/ma/maSize.cc b/ma/maSize.cc
index 9b9508a64..f9d920d18 100644
--- a/ma/maSize.cc
+++ b/ma/maSize.cc
@@ -229,7 +229,7 @@ struct MetricSizeField : public SizeField
     return measure(e) / parentMeasure[mesh->getType(e)];
   }
   Mesh* mesh;
-  int order; // this is the underlying sizefield order
+  int order; // this is the underlying sizefield order (default 1)
 };
 
 AnisotropicFunction::~AnisotropicFunction()
@@ -366,6 +366,8 @@ struct AnisoSizeField : public MetricSizeField
 {
   AnisoSizeField()
   {
+    mesh = 0;
+    order = 1;
   }
   AnisoSizeField(Mesh* m, AnisotropicFunction* f):
     bothEval(f),
@@ -455,6 +457,8 @@ struct LogAnisoSizeField : public MetricSizeField
 {
   LogAnisoSizeField()
   {
+    mesh = 0;
+    order = 1;
   }
   LogAnisoSizeField(Mesh* m, AnisotropicFunction* f):
     logMEval(f)

From 8e013bcb70e135d5d54a135bc2da5665124f0afe Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 24 Nov 2020 16:30:44 -0500
Subject: [PATCH 275/555] Adds the test for higher order size fields

For now the test only runs linear mesh adapt (i.e., ma::adapt).
The corresponding tests for curved mesh adapt (i.e., crv::adapt), will
be added later
---
 test/CMakeLists.txt         |   1 +
 test/highOrderSizeFields.cc | 213 ++++++++++++++++++++++++++++++++++++
 test/testing.cmake          |   4 +
 3 files changed, 218 insertions(+)
 create mode 100644 test/highOrderSizeFields.cc

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index f318a6d22..b6d79a90e 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -198,6 +198,7 @@ if(ENABLE_SIMMETRIX)
   test_exe_func(simZBalance simZBalance.cc)
   test_exe_func(crack_test crack_test.cc)
   test_exe_func(highOrderSolutionTransfer highOrderSolutionTransfer.cc)
+  test_exe_func(highOrderSizeFields highOrderSizeFields.cc)
 endif()
 
 # send all the newly added utility executable targets
diff --git a/test/highOrderSizeFields.cc b/test/highOrderSizeFields.cc
new file mode 100644
index 000000000..a853e88e9
--- /dev/null
+++ b/test/highOrderSizeFields.cc
@@ -0,0 +1,213 @@
+#include <apf.h>
+#include <apfMDS.h>
+#include <apfShape.h>
+#include <apfField.h>
+#include <gmi_mesh.h>
+#include <ma.h>
+#include <maShape.h>
+#include <PCU.h>
+#include <crv.h>
+#include <lionPrint.h>
+#ifdef HAVE_SIMMETRIX
+#include <SimUtil.h>
+#include <MeshSim.h>
+#include <gmi_sim.h>
+#include <SimModel.h>
+#endif
+#include <cassert>
+#include <cstdlib>
+#include <iostream>
+#include <sstream>
+#include <string>
+#include <math.h>
+
+
+void computeSizesFrames(
+    apf::Mesh2* m,
+    const apf::Vector3 &p,
+    apf::Vector3 &sz,
+    apf::Matrix3x3 &frm);
+
+void computeLogM(
+    const apf::Vector3& sz,
+    const apf::Matrix3x3& frm,
+    apf::Matrix3x3& logm);
+
+void testAdapt(
+    const char* model,
+    const char* mesh,
+    int order);
+
+int main(int argc, char** argv)
+{
+  MPI_Init(&argc, &argv);
+  PCU_Comm_Init();
+  lion_set_verbosity(1);
+
+#ifdef HAVE_SIMMETRIX
+  MS_init();
+  SimModel_start();
+  Sim_readLicenseFile(0);
+  gmi_sim_start();
+  gmi_register_sim();
+#endif
+  gmi_register_mesh();
+
+  if (argc != 3) {
+    if(0==PCU_Comm_Self())
+      std::cerr <<"usage: " << argv[0]
+      	<< " <model.smd> <mesh.smb> <sizefield>\n";
+    return EXIT_FAILURE;
+  }
+
+  for (int order = 3; order < 5; order++) {
+    if(0==PCU_Comm_Self())
+      lion_oprint(1, "Testing aniso adapt w/ sizefield order %d\n", order);
+    testAdapt(argv[1], argv[2], order);
+  }
+
+  PCU_Comm_Free();
+#ifdef HAVE_SIMMETRIX
+  gmi_sim_stop();
+  Sim_unregisterAllKeys();
+  SimModel_stop();
+  MS_exit();
+#endif
+  MPI_Finalize();
+}
+
+void computeSizesFrames(
+    apf::Mesh2* m,
+    const apf::Vector3 &p,
+    apf::Vector3 &sz,
+    apf::Matrix3x3 &frm)
+{
+  apf::Vector3 llb;
+  apf::Vector3 umb;
+  ma::getBoundingBox(m, llb, umb);
+  const double pi = 3.14152;
+  apf::Vector3 diff = umb - llb;
+  diff[1] = 0.0;
+  // total length in the direction of the wave
+  double length = diff.getLength();
+  // normal in the direction of the wave
+  apf::Vector3 normal = diff / length;
+  apf::Vector3 tangent(-normal[2], 0., normal[0]);
+
+  double maxSz = length / 12.;
+  double minSz = length / 200.;
+  double wavelength = length / 4.;
+
+
+  double alpha = (maxSz - minSz)/2.;
+  double beta =  (maxSz + minSz)/2.;
+
+  double s = normal * p;
+
+  sz = apf::Vector3(
+      length/4.,
+      length/4.,
+      beta + alpha * sin(2*pi*s/wavelength)
+      );
+
+  frm[0] = normal;
+  frm[1] = apf::Vector3(0., 1., 0.);
+  frm[2] = tangent;
+}
+
+void computeLogM(
+    const apf::Vector3& sz,
+    const apf::Matrix3x3& frm,
+    apf::Matrix3x3& logm)
+{
+  apf::Vector3 s(log(1./sz[0]/sz[0]), log(1./sz[1]/sz[1]), log(1./sz[2]/sz[2]));
+  apf::Matrix3x3 S(s[0],   0.,   0.,
+                     0., s[1],   0.,
+                     0.,   0., s[2]);
+  logm = frm * S * transpose(frm);
+}
+
+void testAdapt(
+    const char* model,
+    const char* mesh,
+    int order)
+{
+  gmi_model* g = gmi_load(model);
+  apf::Mesh2* m = apf::loadMdsMesh(g,mesh);
+  m->verify();
+
+  ma::Input* in = ma::configureUniformRefine(m, 1, 0);
+  ma::adapt(in);
+
+  apf::FieldShape* fs = apf::getH1Shape(order);
+  apf::Field* sizes = apf::createField(m, "sizes", apf::VECTOR, fs);
+  apf::Field* frames = apf::createField(m, "frames", apf::MATRIX, fs);
+
+  apf::Field* sizes_transfer = apf::createField(m, "sizest", apf::VECTOR,
+      apf::getH1Shape(order));
+  apf::Field* frames_transfer = apf::createField(m, "framest", apf::MATRIX,
+      apf::getH1Shape(order));
+  apf::Field* logm_transfer = apf::createField(m, "logmt", apf::MATRIX,
+      apf::getH1Shape(order));
+
+  int dim = m->getDimension();
+  apf::MeshEntity* ent;
+  apf::MeshIterator* it;
+
+  for (int d = 0; d <= dim; d++) {
+    if (!fs->countNodesOn(apf::Mesh::simplexTypes[d]))
+      continue;
+    it = m->begin(d);
+    while( (ent = m->iterate(it)) ) {
+      int type = m->getType(ent);
+      int non = fs->countNodesOn(type);
+      apf::MeshElement* me = apf::createMeshElement(m, ent);
+      for (int i = 0; i < non; i++) {
+	apf::Vector3 xi, p, value;
+	fs->getNodeXi(type, i, xi);
+	apf::mapLocalToGlobal(me, xi, p);
+	apf::Vector3 sz;
+	apf::Matrix3x3 frm;
+	computeSizesFrames(m, p, sz, frm);
+	apf::setVector(sizes, ent, i, sz);
+	apf::setMatrix(frames, ent, i, frm);
+
+        apf::setVector(sizes_transfer, ent, i, sz);
+        apf::setMatrix(frames_transfer, ent, i, frm);
+        apf::Matrix3x3 logm;
+        computeLogM(sz, frm, logm);
+        apf::setMatrix(logm_transfer, ent, i, logm);
+      }
+      apf::destroyMeshElement(me);
+    }
+    m->end(it);
+  }
+
+  in = ma::configure(m, sizes, frames, 0, true);
+  in->shouldFixShape = true;
+  in->maximumIterations = 10;
+  in->shouldForceAdaptation = true;
+
+  std::stringstream ss;
+  ss << "before_adapt_with_ho_sizefield_order_" << order;
+  apf::writeVtkFiles(ss.str().c_str(), m);
+  ss.str("");
+  ma::adaptVerbose(in);
+  ss << "after_adapt_with_ho_sizefield_order_" << order;
+  apf::writeVtkFiles(ss.str().c_str(), m);
+
+  m->removeField(sizes);
+  m->removeField(frames);
+  m->removeField(sizes_transfer);
+  m->removeField(frames_transfer);
+  m->removeField(logm_transfer);
+
+  apf::destroyField(sizes);
+  apf::destroyField(frames);
+  apf::destroyField(sizes_transfer);
+  apf::destroyField(frames_transfer);
+  apf::destroyField(logm_transfer);
+
+  m->destroyNative();
+  apf::destroyMesh(m);
+}
diff --git a/test/testing.cmake b/test/testing.cmake
index fd989d5e3..c82aa4565 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -66,6 +66,10 @@ if(ENABLE_SIMMETRIX)
   mpi_test(modelInfo_smd 1
     ./modelInfo
     "${MESHES}/cube/cube.smd")
+  mpi_test(highorder_sizefield 1
+    ./highOrderSizeFields
+    "${MESHES}/cube/cube.smd"
+    "${MESHES}/cube/pumi11/cube.smb")
 endif(ENABLE_SIMMETRIX)
 
 if(ENABLE_SIMMETRIX)

From 473e074eba45c9c9a15f42d9bd83acda76ee2a0c Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 24 Nov 2020 16:34:13 -0500
Subject: [PATCH 276/555] Removes extra fields info from HO sizefield test

---
 test/highOrderSizeFields.cc | 36 ------------------------------------
 1 file changed, 36 deletions(-)

diff --git a/test/highOrderSizeFields.cc b/test/highOrderSizeFields.cc
index a853e88e9..8e6fa9b4c 100644
--- a/test/highOrderSizeFields.cc
+++ b/test/highOrderSizeFields.cc
@@ -28,11 +28,6 @@ void computeSizesFrames(
     apf::Vector3 &sz,
     apf::Matrix3x3 &frm);
 
-void computeLogM(
-    const apf::Vector3& sz,
-    const apf::Matrix3x3& frm,
-    apf::Matrix3x3& logm);
-
 void testAdapt(
     const char* model,
     const char* mesh,
@@ -115,18 +110,6 @@ void computeSizesFrames(
   frm[2] = tangent;
 }
 
-void computeLogM(
-    const apf::Vector3& sz,
-    const apf::Matrix3x3& frm,
-    apf::Matrix3x3& logm)
-{
-  apf::Vector3 s(log(1./sz[0]/sz[0]), log(1./sz[1]/sz[1]), log(1./sz[2]/sz[2]));
-  apf::Matrix3x3 S(s[0],   0.,   0.,
-                     0., s[1],   0.,
-                     0.,   0., s[2]);
-  logm = frm * S * transpose(frm);
-}
-
 void testAdapt(
     const char* model,
     const char* mesh,
@@ -143,13 +126,6 @@ void testAdapt(
   apf::Field* sizes = apf::createField(m, "sizes", apf::VECTOR, fs);
   apf::Field* frames = apf::createField(m, "frames", apf::MATRIX, fs);
 
-  apf::Field* sizes_transfer = apf::createField(m, "sizest", apf::VECTOR,
-      apf::getH1Shape(order));
-  apf::Field* frames_transfer = apf::createField(m, "framest", apf::MATRIX,
-      apf::getH1Shape(order));
-  apf::Field* logm_transfer = apf::createField(m, "logmt", apf::MATRIX,
-      apf::getH1Shape(order));
-
   int dim = m->getDimension();
   apf::MeshEntity* ent;
   apf::MeshIterator* it;
@@ -171,12 +147,6 @@ void testAdapt(
 	computeSizesFrames(m, p, sz, frm);
 	apf::setVector(sizes, ent, i, sz);
 	apf::setMatrix(frames, ent, i, frm);
-
-        apf::setVector(sizes_transfer, ent, i, sz);
-        apf::setMatrix(frames_transfer, ent, i, frm);
-        apf::Matrix3x3 logm;
-        computeLogM(sz, frm, logm);
-        apf::setMatrix(logm_transfer, ent, i, logm);
       }
       apf::destroyMeshElement(me);
     }
@@ -198,15 +168,9 @@ void testAdapt(
 
   m->removeField(sizes);
   m->removeField(frames);
-  m->removeField(sizes_transfer);
-  m->removeField(frames_transfer);
-  m->removeField(logm_transfer);
 
   apf::destroyField(sizes);
   apf::destroyField(frames);
-  apf::destroyField(sizes_transfer);
-  apf::destroyField(frames_transfer);
-  apf::destroyField(logm_transfer);
 
   m->destroyNative();
   apf::destroyMesh(m);

From 4d1072353cf96e154a190852786752b60a1a2f31 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 24 Nov 2020 16:38:40 -0500
Subject: [PATCH 277/555] Fixes typo

---
 ma/maSize.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ma/maSize.cc b/ma/maSize.cc
index f9d920d18..1dea70eab 100644
--- a/ma/maSize.cc
+++ b/ma/maSize.cc
@@ -35,7 +35,7 @@ void SizeField::onCavity(EntityArray&, EntityArray&)
 
 int SizeField::getTransferDimension()
 {
-  // By default there should be now size_field transfer.
+  // By default there should be no size_field transfer.
   // This is used in a loop to get the lowest dimension
   // entities that require transfer. So something bigger
   // than mesh dimension will ignore that loop

From b63a826ef927d659eebe0095bed90f2aeeddaf32 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Tue, 24 Nov 2020 13:52:27 -0800
Subject: [PATCH 278/555] NAS changes

---
 phasta/phGeomBC.cc    | 2 +-
 phasta/phPartition.cc | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/phasta/phGeomBC.cc b/phasta/phGeomBC.cc
index eec37316f..fcb3ea55b 100644
--- a/phasta/phGeomBC.cc
+++ b/phasta/phGeomBC.cc
@@ -367,7 +367,7 @@ void writeGeomBC(Output& o, std::string path, int timestep)
 // start effort to write coords to a flat ascii file for each part
   int npts=params[0];
   char coordfilename[64];
-  bzero((void*)coordfilename,64);
+  //bzero((void*)coordfilename,64);
   int rank = PCU_Comm_Self() + 1;
   sprintf(coordfilename, "coords.%d",rank);
   FILE* fc = fopen(coordfilename, "w");
diff --git a/phasta/phPartition.cc b/phasta/phPartition.cc
index bb0c0a207..6711598ef 100644
--- a/phasta/phPartition.cc
+++ b/phasta/phPartition.cc
@@ -22,7 +22,7 @@ apf::Migration* getSplitPlan(Input& in, apf::Mesh2* m)
 {
   PCU_ALWAYS_ASSERT(in.splitFactor >= 1);
   apf::Migration* plan;
-  if (in.splitFactor != 1) {
+  if (in.splitFactor != 0 ) { // 1) {
     apf::Splitter* splitter;
     if (in.partitionMethod == "rib") { //prefer SCOREC RIB over Zoltan RIB
       splitter = Parma_MakeRibSplitter(m);

From ace1b97185115c501a98b0c571521a44569fce7f Mon Sep 17 00:00:00 2001
From: Cameron Smith <cwsmith@users.noreply.github.com>
Date: Wed, 25 Nov 2020 08:23:30 -0500
Subject: [PATCH 279/555] citing pumi

---
 README.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/README.md b/README.md
index 992c62876..7b21e5acb 100644
--- a/README.md
+++ b/README.md
@@ -53,3 +53,11 @@ For more information, start at our
 
 * If you have a usage question or have found a bug please post an [issue](https://github.com/SCOREC/core/issues).
 * Otherwise, email [Cameron Smith](https://www.scorec.rpi.edu/~cwsmith) and pumi@scorec.rpi.edu.
+
+### Citing PUMI
+
+If you use these tools, please cite the following paper:
+
+Daniel A. Ibanez, E. Seegyoung Seol, Cameron W. Smith, and Mark S. Shephard. 2016. PUMI: Parallel Unstructured Mesh Infrastructure. ACM Trans. Math. Softw. 42, 3, Article 17 (May 2016), 28 pages. DOI: https://doi.org/10.1145/2814935
+
+We would be happy to provide feedback on journal submissions using PUMI prior to publication.

From 896298e0d4c4a9a6a501693f30930a5a84658a7d Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Thu, 26 Nov 2020 17:38:00 -0800
Subject: [PATCH 280/555] attempts to set weights

---
 apf/apfConstruct.cc   | 23 +++++++++++++++---
 apf/apfConvertTags.h  |  2 +-
 phasta/phPartition.cc | 55 ++++++++++++++++++++++++++-----------------
 3 files changed, 55 insertions(+), 25 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index 1d24e3b55..400d36f4c 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -27,12 +27,29 @@ static void constructElements(
 {
   ModelEntity* interior = m->findModelEntity(m->getDimension(), 0);
   int nev = apf::Mesh::adjacentCount[etype][0];
+  unsigned etypeL=1;
   for (int i = 0; i < nelem; ++i) {
+    int irep=0;
     Downward verts;
     int offset = i * nev;
-    for (int j = 0; j < nev; ++j)
-      verts[j] = globalToVert[conn[j + offset]];
-    buildElement(m, interior, etype, verts);
+    int vCur=conn[offset];
+    int vNext=-1;
+    int uniqueVerts=1;
+    for (int j = 0; j < nev; ++j) {
+      if(irep ==0){
+        verts[j] = globalToVert[vCur]; // conn[j + offset]];
+        vNext=conn[j+1+offset];
+        if(vNext == vCur) irep=1; // this was last one
+        else vCur=vNext; // Keep going but set this to what it needs next
+        uniqueVerts=j;
+      } 
+    }
+    uniqueVerts++;
+    if(uniqueVerts==4) etypeL=apf::Mesh::TET;
+    if(uniqueVerts==5) etypeL=apf::Mesh::PYRAMID;
+    if(uniqueVerts==6) etypeL=apf::Mesh::PRISM;
+    if(uniqueVerts==8) etypeL=apf::Mesh::HEX;
+    buildElement(m, interior, etypeL, verts);
   }
 }
 
diff --git a/apf/apfConvertTags.h b/apf/apfConvertTags.h
index 3d5a0e1ea..bf99c54cb 100644
--- a/apf/apfConvertTags.h
+++ b/apf/apfConvertTags.h
@@ -18,7 +18,7 @@ namespace {
   template <class T> inline
   apf::MeshTag* createTag(apf::Mesh*,
       const char*, const int) {
-    exit(EXIT_FAILURE);
+    exit(EXIT_FAILURE); 
     return 0;
   }
   
diff --git a/phasta/phPartition.cc b/phasta/phPartition.cc
index 6711598ef..7411a752d 100644
--- a/phasta/phPartition.cc
+++ b/phasta/phPartition.cc
@@ -9,6 +9,8 @@
 #include <pcu_util.h>
 #include <lionPrint.h>
 
+
+
 #ifdef HAVE_SIMMETRIX
 #include <apfSIM.h>
 #include "SimPartitionedMesh.h"
@@ -18,6 +20,31 @@
 
 namespace ph {
 
+void setWeight(apf::Mesh* m, apf::MeshTag* tag, int dim) {
+  double w = 1.0;
+  apf::MeshEntity* e;
+  apf::MeshIterator* it = m->begin(dim);
+  int nverts =1;
+  apf::Downward verts;
+  while ((e = m->iterate(it)))
+// copied from function defn: int getDimension(Mesh* m, MeshEntity* e);
+//  if(getDimension(m,e)==3) {
+// No-OP    if(0==1){
+//      nverts = m->getDownward(e, 0, verts);
+//      w=1.0*nverts;
+// No-OP   }
+    m->setDoubleTag(e, tag, &w);
+  m->end(it);
+}
+
+apf::MeshTag* setWeights(apf::Mesh* m) {
+  apf::MeshTag* tag = m->createDoubleTag("parma_weight", 1);
+  setWeight(m, tag, 0);
+  setWeight(m, tag, m->getDimension());
+  return tag;
+}
+
+
 apf::Migration* getSplitPlan(Input& in, apf::Mesh2* m)
 {
   PCU_ALWAYS_ASSERT(in.splitFactor >= 1);
@@ -37,7 +64,8 @@ apf::Migration* getSplitPlan(Input& in, apf::Mesh2* m)
       else
         splitter = apf::makeZoltanGlobalSplitter(m, method, apf::REPARTITION);
     }
-    apf::MeshTag* weights = Parma_WeighByMemory(m);
+//    apf::MeshTag* weights = Parma_WeighByMemory(m);
+    apf::MeshTag* weights = setWeights(m);
     plan = splitter->split(weights, 1.01, in.splitFactor);
     apf::removeTagFromDimension(m, weights, m->getDimension());
     m->destroyTag(weights);
@@ -66,22 +94,6 @@ bool isMixed(apf::Mesh2* m) {
   return PCU_Max_Int(mixed);
 }
 
-void setWeight(apf::Mesh* m, apf::MeshTag* tag, int dim) {
-  double w = 1.0;
-  apf::MeshEntity* e;
-  apf::MeshIterator* it = m->begin(dim);
-  while ((e = m->iterate(it)))
-    m->setDoubleTag(e, tag, &w);
-  m->end(it);
-}
-
-apf::MeshTag* setWeights(apf::Mesh* m) {
-  apf::MeshTag* tag = m->createDoubleTag("parma_weight", 1);
-  setWeight(m, tag, 0);
-  setWeight(m, tag, m->getDimension());
-  return tag;
-}
-
 void clearTags(apf::Mesh* m, apf::MeshTag* t) {
   apf::removeTagFromDimension(m, t, 0);
   apf::removeTagFromDimension(m, t, m->getDimension());
@@ -101,7 +113,8 @@ void neighborReduction(apf::Mesh2* m, apf::MeshTag* weights, int verbose, bool f
 void parmaMixed(Input& in, apf::Mesh2* m) {
   bool fineStats=false; // set to true for per part stats
   Parma_PrintPtnStats(m, "preRefine", fineStats); //FIXME
-  apf::MeshTag* weights = Parma_WeighByMemory(m);
+//  apf::MeshTag* weights = Parma_WeighByMemory(m);
+  apf::MeshTag* weights = setWeights(m);
   const double step = 0.2;
   const int verbose = 0;
   apf::Balancer* balancer = Parma_MakeElmBalancer(m, step, verbose);
@@ -139,9 +152,9 @@ void parmaTet(Input& in, apf::Mesh2* m, bool runGap) {
 }
 
 void parmaBalance(Input& in, apf::Mesh2* m, bool runGap) {
-  if( isMixed(m) )
-    parmaMixed(in,m);
-  else
+//  if( isMixed(m) )
+//    parmaMixed(in,m);
+//  else
     parmaTet(in,m,runGap);
 }
 

From 6f22ab0406164a5454d300264fb5774acc1b7c41 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Thu, 26 Nov 2020 18:39:17 -0700
Subject: [PATCH 281/555] mergedWrongVersion:

---
 apf/apfConstruct.cc | 23 ++++++++++++++++++++---
 gmi_sim/gmi_sim.cc  |  1 +
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index 1d24e3b55..400d36f4c 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -27,12 +27,29 @@ static void constructElements(
 {
   ModelEntity* interior = m->findModelEntity(m->getDimension(), 0);
   int nev = apf::Mesh::adjacentCount[etype][0];
+  unsigned etypeL=1;
   for (int i = 0; i < nelem; ++i) {
+    int irep=0;
     Downward verts;
     int offset = i * nev;
-    for (int j = 0; j < nev; ++j)
-      verts[j] = globalToVert[conn[j + offset]];
-    buildElement(m, interior, etype, verts);
+    int vCur=conn[offset];
+    int vNext=-1;
+    int uniqueVerts=1;
+    for (int j = 0; j < nev; ++j) {
+      if(irep ==0){
+        verts[j] = globalToVert[vCur]; // conn[j + offset]];
+        vNext=conn[j+1+offset];
+        if(vNext == vCur) irep=1; // this was last one
+        else vCur=vNext; // Keep going but set this to what it needs next
+        uniqueVerts=j;
+      } 
+    }
+    uniqueVerts++;
+    if(uniqueVerts==4) etypeL=apf::Mesh::TET;
+    if(uniqueVerts==5) etypeL=apf::Mesh::PYRAMID;
+    if(uniqueVerts==6) etypeL=apf::Mesh::PRISM;
+    if(uniqueVerts==8) etypeL=apf::Mesh::HEX;
+    buildElement(m, interior, etypeL, verts);
   }
 }
 
diff --git a/gmi_sim/gmi_sim.cc b/gmi_sim/gmi_sim.cc
index adb62ba4b..5bae99156 100644
--- a/gmi_sim/gmi_sim.cc
+++ b/gmi_sim/gmi_sim.cc
@@ -484,6 +484,7 @@ struct gmi_model* gmi_sim_load(const char* nativefile, const char* smdfile)
   pGModel sm;
   if (!smdfile) {
     if (NM_isAssemblyModel(nm)) {
+//      pGModel am = GAM_createFromNativeModel(nm, NULL);
       pGAModel am = GAM_createFromNativeModel(nm, NULL);
       NM_release(nm);
       sm = GM_createFromAssemblyModel(am, NULL, NULL);

From b96d2e3c98d0c26b6a9002d9096c1486c3ed1ba6 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Fri, 27 Nov 2020 13:26:33 -0700
Subject: [PATCH 282/555] silly mistake...now it works on small debug case

---
 apf/apfConvertTags.h  |  2 +-
 phasta/phPartition.cc | 14 +++++++-------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/apf/apfConvertTags.h b/apf/apfConvertTags.h
index bf99c54cb..3d5a0e1ea 100644
--- a/apf/apfConvertTags.h
+++ b/apf/apfConvertTags.h
@@ -18,7 +18,7 @@ namespace {
   template <class T> inline
   apf::MeshTag* createTag(apf::Mesh*,
       const char*, const int) {
-    exit(EXIT_FAILURE); 
+    exit(EXIT_FAILURE);
     return 0;
   }
   
diff --git a/phasta/phPartition.cc b/phasta/phPartition.cc
index 7411a752d..6c8ab423f 100644
--- a/phasta/phPartition.cc
+++ b/phasta/phPartition.cc
@@ -26,14 +26,14 @@ void setWeight(apf::Mesh* m, apf::MeshTag* tag, int dim) {
   apf::MeshIterator* it = m->begin(dim);
   int nverts =1;
   apf::Downward verts;
-  while ((e = m->iterate(it)))
-// copied from function defn: int getDimension(Mesh* m, MeshEntity* e);
-//  if(getDimension(m,e)==3) {
-// No-OP    if(0==1){
-//      nverts = m->getDownward(e, 0, verts);
-//      w=1.0*nverts;
-// No-OP   }
+  while ((e = m->iterate(it))){
+    int dimEnt=getDimension(m,e);
+    if(dimEnt==3) {
+      nverts = m->getDownward(e, 0, verts);
+      if(nverts==8) w=6.0;
+    }
     m->setDoubleTag(e, tag, &w);
+  }
   m->end(it);
 }
 

From 7bc29a3d2aba20bb04dd006e52cf25e52310491f Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Thu, 3 Dec 2020 15:36:11 -0500
Subject: [PATCH 283/555] Adds residual based error estimation test

---
 pumi-meshes                          |  1 -
 test/CMakeLists.txt                  |  2 ++
 test/residualErrorEstimation_test.cc | 51 ++++++++++++++++++++++++++++
 3 files changed, 53 insertions(+), 1 deletion(-)
 delete mode 160000 pumi-meshes
 create mode 100644 test/residualErrorEstimation_test.cc

diff --git a/pumi-meshes b/pumi-meshes
deleted file mode 160000
index 94870b0ce..000000000
--- a/pumi-meshes
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 94870b0ce0f9d953e985ed95c47c5b65246ae361
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 027f43ed4..c282995cb 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -185,6 +185,8 @@ test_exe_func(create_mis create_mis.cc)
 test_exe_func(fieldReduce fieldReduce.cc)
 test_exe_func(test_integrator test_integrator.cc)
 test_exe_func(test_matrix_gradient test_matrix_grad.cc)
+test_exe_func(residualErrorEstimation_test residualErrorEstimation_test.cc)
+
 if(ENABLE_DSP)
   test_exe_func(graphdist graphdist.cc)
   test_exe_func(moving moving.cc)
diff --git a/test/residualErrorEstimation_test.cc b/test/residualErrorEstimation_test.cc
new file mode 100644
index 000000000..3d382be06
--- /dev/null
+++ b/test/residualErrorEstimation_test.cc
@@ -0,0 +1,51 @@
+#include <em.h>
+#include <apf.h>
+#include <gmi_mesh.h>
+#include <apfMDS.h>
+#include <apfShape.h>
+#include <PCU.h>
+#include <lionPrint.h>
+#ifdef HAVE_SIMMETRIX
+#include <gmi_sim.h>
+#include <SimUtil.h>
+#include <MeshSim.h>
+#include <SimModel.h>
+#endif
+#include <pcu_util.h>
+
+
+int main(int argc, char** argv)
+{
+  PCU_ALWAYS_ASSERT(argc==3);
+  const char* modelFile = argv[1];
+  const char* meshFile = argv[2];
+  MPI_Init(&argc,&argv);
+  PCU_Comm_Init();
+  lion_set_verbosity(1);
+#ifdef HAVE_SIMMETRIX
+  MS_init();
+  SimModel_start();
+  Sim_readLicenseFile(0);
+  gmi_sim_start();
+  gmi_register_sim();
+#endif
+  gmi_register_mesh();
+  ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile);
+  m->verify();
+
+  apf::Field* electric_field = m->getField(0);
+  apf::Field* implicit_error_field = em::estimateError(electric_field);
+
+  m->destroyNative();
+  apf::destroyMesh(m);
+#ifdef HAVE_SIMMETRIX
+  gmi_sim_stop();
+  Sim_unregisterAllKeys();
+  SimModel_stop();
+  MS_exit();
+#endif
+  PCU_Comm_Free();
+  MPI_Finalize();
+}
+
+

From 6f2a3a34a6bf0df45fcf2bfdce1678a0617c5eec Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Thu, 3 Dec 2020 17:28:44 -0500
Subject: [PATCH 284/555] Adds residualErrorEstimation test in testing.cmake

---
 test/residualErrorEstimation_test.cc | 2 ++
 test/testing.cmake                   | 4 ++++
 2 files changed, 6 insertions(+)

diff --git a/test/residualErrorEstimation_test.cc b/test/residualErrorEstimation_test.cc
index 3d382be06..865255992 100644
--- a/test/residualErrorEstimation_test.cc
+++ b/test/residualErrorEstimation_test.cc
@@ -1,3 +1,4 @@
+#include "ma.h"
 #include <em.h>
 #include <apf.h>
 #include <gmi_mesh.h>
@@ -35,6 +36,7 @@ int main(int argc, char** argv)
 
   apf::Field* electric_field = m->getField(0);
   apf::Field* implicit_error_field = em::estimateError(electric_field);
+  apf::destroyField(implicit_error_field);
 
   m->destroyNative();
   apf::destroyMesh(m);
diff --git a/test/testing.cmake b/test/testing.cmake
index 5841957fd..d8c97ab8b 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -294,6 +294,10 @@ mpi_test(tet_serial 1
   "${MDIR}/pipe.${GXT}"
   "pipe.smb"
   "tet.smb")
+mpi_test(residualErrorEstimation_test 1
+  ./residualErrorEstimation_test
+  "${MESHES}/fichera/fichera.x_t"
+  "${MESHES}/fichera/fichera.smb")
 if(PCU_COMPRESS)
   set(MESHFILE "bz2:pipe_2_.smb")
 else()

From e8e74f244c00e0aa65b464a6db8a32665083a26e Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Thu, 3 Dec 2020 18:04:39 -0500
Subject: [PATCH 285/555] Cleans up the error estimation files.

---
 em/em.h                     | 19 +-------
 em/emCorrectedFlux.cc       | 42 +++-------------
 em/emEstimateError.cc       | 17 +++----
 em/emFluxCorrection.cc      |  6 +--
 em/emResidualFunctionals.cc | 97 ++-----------------------------------
 em/emSizeField.cc           |  8 +--
 6 files changed, 18 insertions(+), 171 deletions(-)

diff --git a/em/em.h b/em/em.h
index 656e69356..a64c1b578 100644
--- a/em/em.h
+++ b/em/em.h
@@ -9,18 +9,12 @@
 #define EM_H
 
 
-/** \file em.h
- *  \brief The Elegtromagnetics Equilibrated Residual error estimator inteface
- */
 #include "apf.h"
 #include <apfMesh.h>
-
 #include <PCU.h>
 #include <lionPrint.h>
-
 #include "apfShape.h"
 #include "apfField.h"
-
 #include <mthQR.h>
 #include <mth.h>
 #include <mth_def.h>
@@ -28,7 +22,7 @@
 namespace em {
 
 /*
- * Computes nodal size field // TODO currently only element size field
+ * Computes nodal size field
  */
 apf::Field* getTargetEMSizeField(
     apf::Field* ef,
@@ -78,15 +72,4 @@ apf::Vector3 computeFaceOutwardNormal(apf::Mesh* m,
 
 
 }
-
-
-
-
-
-
-
-
-
-
-
 #endif
diff --git a/em/emCorrectedFlux.cc b/em/emCorrectedFlux.cc
index b196ead69..d5f0b4082 100644
--- a/em/emCorrectedFlux.cc
+++ b/em/emCorrectedFlux.cc
@@ -4,16 +4,12 @@
  * This work is open source software, licensed under the terms of the
  * BSD license as described in the LICENSE file in the top-level directory.
  */
-#include <iostream>
-#include <cstdlib>
-
 #include <apfCavityOp.h>
 #include "apfElement.h"
 #include "crv.h"
 #include "crvShape.h"
-
 #include "em.h"
-using namespace std;
+
 namespace em {
 
 enum {VISITED};
@@ -64,7 +60,7 @@ static void setupCorrectFlux(
   while ((tet = cf->mesh->iterate(it))) {
     apf::MeshElement* me = apf::createMeshElement(cf->mesh, tet);
     int orderp1 = cf->order+1;
-    int np = apf::countIntPoints(me, 2*orderp1-1); // TODO 2*order
+    int np = apf::countIntPoints(me, 2*orderp1-1);
 
     for (int i = 0; i < np; i++) {
       apf::setComponents(cf->correctedFlux, tet, i, zeros);
@@ -166,8 +162,8 @@ static void computeCorrectedFlux(FaceCavity* fc)
   theta_coeffs(1) = components[1];
   theta_coeffs(2) = components[2];
 
-//////
-  // 4. Evaluate and save corrected flux vector in an auxiliary field
+
+  // 5. Evaluate and save corrected flux vector in an auxiliary field
   int ftype = fc->mesh->getType(face);
   PCU_ALWAYS_ASSERT(ftype == apf::Mesh::TRIANGLE);
   int nfdofs = apf::countElementNodes(fc->correctflux->ef->getShape(), ftype);
@@ -175,7 +171,7 @@ static void computeCorrectedFlux(FaceCavity* fc)
 
   apf::MeshElement* fme = apf::createMeshElement(fc->mesh, face);
   apf::Element* fel = apf::createElement(fc->correctflux->ef, fme);
-  int int_order = 2*fc->correctflux->orderp1-1; // TODO 2*order
+  int int_order = 2*fc->correctflux->orderp1-1;
   int np = apf::countIntPoints(fme, int_order);
   int nc = apf::countComponents(fc->correctflux->correctedFlux);
 
@@ -288,7 +284,7 @@ apf::Field* computeCorrectedFlux(apf::Field* ef, apf::Field* theta)
 {
   int dim = apf::getMesh(ef)->getDimension();
   int order = ef->getShape()->getOrder() + 1; // local BVPs require p+1
-  int int_order = 2*order-1; // TODO 2*order TODO update IPField to handle
+  int int_order = 2*order-1;
   int nc = 4*3; // 1 flux vector per face
   apf::Field* correctedFlux = createPackedField(
       apf::getMesh(ef), "correctedFlux", nc, apf::getIPShape(dim, int_order));
@@ -297,33 +293,7 @@ apf::Field* computeCorrectedFlux(apf::Field* ef, apf::Field* theta)
   setupCorrectFlux(&correctflux, ef, theta, correctedFlux);
   FaceCavityOp op (&correctflux);
   op.applyToDimension(2);
-  // TODO remove tag
   return correctedFlux;
 
-  /* debug
-  // testing initial corrected flux field
-  cout << "testing initial field" << endl;
-    int elemNo = 0;
-    apf::MeshEntity* ent;
-    apf::MeshIterator* itr = apf::getMesh(ef)->begin(3);
-    while ((ent = apf::getMesh(ef)->iterate(itr)))
-    {
-      cout << "at tet " << elemNo++ << endl;
-      apf::MeshElement* me = apf::createMeshElement(apf::getMesh(ef), ent);
-      int np = apf::countIntPoints(me, int_order);
-      for (int i = 0; i < np; i++) {
-        cout << " at point " << i << endl;
-        double components[nc];
-        apf::getComponents(correctflux.correctedFlux, ent, i, components);
-        for (int j = 0; j < nc; j++) {
-          cout << components[j] << " ";
-        }
-        cout << endl;
-      }
-      cout << "==================================================" << endl;
-    }*/
 }
-
-
-
 }
diff --git a/em/emEstimateError.cc b/em/emEstimateError.cc
index 1b3e94654..412c97522 100644
--- a/em/emEstimateError.cc
+++ b/em/emEstimateError.cc
@@ -4,16 +4,13 @@
  * This work is open source software, licensed under the terms of the
  * BSD license as described in the LICENSE file in the top-level directory.
  */
-#include <iostream>
-#include <cstdlib>
 
 #include <apfCavityOp.h>
 #include "apfElement.h"
 #include "crv.h"
 #include "crvShape.h"
-
 #include "em.h"
-using namespace std;
+
 namespace em {
 
 static void computeResidualBLF(apf::Mesh* mesh, apf::MeshEntity* e,
@@ -52,7 +49,7 @@ static void computeResidualBLF(apf::Mesh* mesh, apf::MeshEntity* e,
 
     apf::getCurl(fel, p, curl);
 
-    // get curlshape values // TODO CLEAN use getCurlShapeValues
+    // get curlshape values
     fp1el->getShape()->getLocalVectorCurls(mesh, e, p, curlshape);
     phys_curlshape.zero();
     for (int i = 0; i < nd; i++)
@@ -147,14 +144,14 @@ static void computeLambdaVector(
     apf::NewArray<apf::Vector3> vectorshape(nfdofs);
 
     apf::MeshElement* fme = apf::createMeshElement(mesh, face);
-    int np = apf::countIntPoints(fme, 2*order-1); // TODO 2*order
+    int np = apf::countIntPoints(fme, 2*order-1);
 
     // 4. Compute integral on the face
     apf::Vector3 p;
     for (int n = 0; n < np; n++) {
 
-      apf::getIntPoint(fme, 2*order-1, n, p); // TODO 2*order
-      double weight = apf::getIntWeight(fme, 2*order-1, n); // TODO 2*order
+      apf::getIntPoint(fme, 2*order-1, n, p);
+      double weight = apf::getIntWeight(fme, 2*order-1, n);
       apf::Matrix3x3 fJ;
       apf::getJacobian(fme, p, fJ);
       double jdet = apf::getJacobianDeterminant(
@@ -442,7 +439,7 @@ apf::Field* estimateError(apf::Field* f)
   apf::Field* theta = em::computeFluxCorrection(f, g);
   lion_eprint(1,"2/4: flux corrections computed \n");
   apf::destroyField(g);
-  PCU_Barrier(); // TODO remove
+  PCU_Barrier();
 
   apf::Field* correctedFlux = em::computeCorrectedFlux(f, theta);
   lion_eprint(1,"3/4: corrected flux field computed\n");
@@ -457,7 +454,5 @@ apf::Field* estimateError(apf::Field* f)
     lion_eprint(1,"EM: Error estimated in %f seconds\n",t1-t0);
 
   return error_field;
-  //return correctedFlux;
 }
-
 }
diff --git a/em/emFluxCorrection.cc b/em/emFluxCorrection.cc
index ad5fa58a3..1dac4b02a 100644
--- a/em/emFluxCorrection.cc
+++ b/em/emFluxCorrection.cc
@@ -4,15 +4,11 @@
  * This work is open source software, licensed under the terms of the
  * BSD license as described in the LICENSE file in the top-level directory.
  */
-#include <iostream>
-#include <cstdlib>
-
 #include "crv.h"
 #include "crvShape.h"
 #include "apfElement.h"
-
 #include "em.h"
-using namespace std;
+
 namespace em {
 
 struct QRDecomp {
diff --git a/em/emResidualFunctionals.cc b/em/emResidualFunctionals.cc
index 31a6c39aa..015dc5a00 100644
--- a/em/emResidualFunctionals.cc
+++ b/em/emResidualFunctionals.cc
@@ -4,19 +4,13 @@
  * This work is open source software, licensed under the terms of the
  * BSD license as described in the LICENSE file in the top-level directory.
  */
-#include <iostream>
-#include <cstdlib>
 #include <algorithm>
-
 #include <apfCavityOp.h>
 #include "apfElement.h"
 #include "crv.h"
 #include "crvShape.h"
-
 #include "em.h"
 
-using namespace std;
-
 namespace em {
 
 enum {VISITED};
@@ -173,7 +167,6 @@ static void assembleEdgePatchLHS(EdgePatch* ep)
 
 /*
  * Performs Curl Curl integration using curl vector Nedelec shapes
- * TODO Add 2D curl curl integration. not needed right now.
  */
 void assembleCurlCurlElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
     apf::Field* f, mth::Matrix<double>& elmat)
@@ -213,15 +206,6 @@ void assembleCurlCurlElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
           for (int l = 0; l < dim; l++)
             phys_curlshape(j,k) += curlshape[j][l] * J[l][k];
     }
-    else {
-      /*
-      2D in apfNedelec.cc
-      el->getShape()->getLocalVectorCurls(mesh, e, p, curlshape);
-      phys_curlshape.zero();
-      for (int i = 0; i < nd; i++)
-        for (int j = 0; j < dimc; j++)
-        */
-    }
     mth::Matrix<double> phys_curlshapeT;
     mth::transpose(phys_curlshape, phys_curlshapeT);
 
@@ -311,8 +295,6 @@ static double getLocalEdgeBLF(EdgePatch* ep, apf::MeshEntity* tet)
   apf::Downward e;
   int ne = ep->mesh->getDownward(tet, 1, e);
   int ei = apf::findIn(e, ne, ep->entity);
-  if (PCU_Comm_Self() == 0)
-    cout << "ei " << ei << endl;
   // get Element Dofs
   apf::MeshElement* me = apf::createMeshElement(ep->mesh, tet);
   apf::Element* el = apf::createElement(ep->equilibration->ef, me);
@@ -352,8 +334,9 @@ static double getLocalEdgeBLF(EdgePatch* ep, apf::MeshEntity* tet)
   return blf_integrals(ei);
 }
 
-
-// TODO QUESTION redo this to allow user access from outside
+/*
+ * Capability can be added in the future to allow the user
+ * to define a custom  function */
 void pumiUserFunction(apf::Mesh* mesh, apf::MeshEntity* e,
     const apf::Vector3& x, mth::Vector<double>& f)
 {
@@ -662,13 +645,6 @@ static void assembleEdgePatchRHS(EdgePatch* p)
     double blfIntegral = getLocalEdgeBLF(p, tet);
     double lfIntegral = getLocalEdgeLF(p, tet);
     double fluxIntegral = getLocalFluxIntegral(p, tet);
-    if (PCU_Comm_Self() == 0) {
-      apf::Vector3 center = apf::getLinearCentroid(p->mesh, tet);
-      cout << center << endl;
-      cout << "bilinear integral " << blfIntegral << endl;
-      cout << "linear integral " << lfIntegral << endl;
-      cout << "flux integral " << fluxIntegral << endl;
-    }
     if(p->isOnBdry)
       p->b(nf+i) = blfIntegral - lfIntegral - fluxIntegral;
     else
@@ -808,15 +784,6 @@ void getClockwiseTetsandFaces(EdgePatch* p)
         std::reverse(p->tets.begin(), p->tets.end());
         std::reverse(p->faces.begin(), p->faces.begin());
       }
-      else if ((vFirst * normal < 0)  && (vLast * normal > 0)) {
-        cout << "already clockwise" << endl;
-      }
-      else if ((vFirst * normal < 0)  && (vLast * normal < 0)) {
-        cout << "failed clockwise. ABORT" << endl;
-      }
-      else if ((vFirst * normal > 0)  && (vLast * normal > 0)) {
-        cout << "failed clockwise. ABORT" << endl;
-      }
     }
   }
   else {
@@ -864,14 +831,6 @@ void getClockwiseTetsandFaces(EdgePatch* p)
       std::reverse(p->tets.begin(), p->tets.end());
       std::reverse(p->faces.begin(), p->faces.begin());
     }
-    else if ((vFirst * normal < 0)  && (vLast * normal > 0)) {
-    }
-    else if ((vFirst * normal < 0)  && (vLast * normal < 0)) {
-      cout << "failed clockwise. ABORT" << endl;
-    }
-    else if ((vFirst * normal > 0)  && (vLast * normal > 0)) {
-      cout << "failed clockwise. ABORT" << endl;
-    }
   }
 }
 
@@ -905,56 +864,6 @@ static void runErm(EdgePatch* ep)
     components[ei] = ep->x(i);
     apf::setComponents(ep->equilibration->g, face, 0, components);
   }
-  // debug
-  if (PCU_Comm_Self() == 0) {
-    cout << "Part " << PCU_Comm_Self() << ": ";
-    apf::Vector3 center = apf::getLinearCentroid(ep->mesh, ep->entity);
-    cout << center << endl;
-
-    cout << "LHS: ";
-    for (size_t i = 0; i < ep->T.rows(); i++) {
-      for (size_t j = 0; j < ep->T.cols(); j++) {
-        cout << ep->T(i,j) << " ";
-      }
-      cout << endl;
-    }
-
-    cout << "RHS: ";
-    for (size_t i = 0; i < ep->b.size(); i++)
-      cout << ep->b[i] << " ";
-    cout << endl;
-
-    cout << "gs: ";
-    for (size_t i = 0; i < ep->x.size(); i++)
-      cout << ep->x[i] << " ";
-    cout << endl;
-
-    cout << "Ordered tets" << endl;
-    for (size_t i = 0; i < ep->tets.size(); i++) {
-      apf::Vector3 center = apf::getLinearCentroid(ep->mesh, ep->tets[i]);
-      cout << center << endl;
-    }
-    cout << "Ordered faces" << endl;
-    for (size_t i = 0; i < ep->faces.size(); i++) {
-      apf::Vector3 center = apf::getLinearCentroid(ep->mesh, ep->faces[i]);
-      cout << center << endl;
-    }
-
-
-    cout << "==================" << endl;
-
-
-  }
-  /*if (PCU_Comm_Self() == 1) {
-    cout << "Part " << PCU_Comm_Self() << ": ";
-    apf::Vector3 center = apf::getLinearCentroid(ep->mesh, ep->entity);
-    cout << center << endl;
-
-    cout << "RHS: ";
-    for (size_t i = 0; i < ep->faces.size(); i++)
-      cout << ep->b[i] << " ";
-    cout << endl;
-  }*/
 }
 
 
diff --git a/em/emSizeField.cc b/em/emSizeField.cc
index fbffadcae..15132809e 100644
--- a/em/emSizeField.cc
+++ b/em/emSizeField.cc
@@ -4,16 +4,12 @@
  * This work is open source software, licensed under the terms of the
  * BSD license as described in the LICENSE file in the top-level directory.
  */
-#include <iostream>
-#include <cstdlib>
-
 #include "apfElement.h"
 #include "crv.h"
 #include "crvShape.h"
 #include <apfCavityOp.h>
-
 #include "em.h"
-using namespace std;
+
 namespace em {
 
 
@@ -103,7 +99,6 @@ static void getElementSizeField(Sizefield* sz)
   while ((entity = sz->mesh->iterate(elements))) {
     double h = getDesiredSize(sz, entity);
     apf::setScalar(eSize, entity, 0, h);
-    cout << "h " << h << endl;
   }
   sz->mesh->end(elements);
   sz->element_size = eSize;
@@ -175,5 +170,4 @@ apf::Field* getTargetEMSizeField(
   return sz.size;
 }
 
-
 }

From 18afbd6d07c5b76ee09b59e4a68ea99341484b0f Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Thu, 3 Dec 2020 18:40:31 -0500
Subject: [PATCH 286/555] Changes the name of the subdirectory 'em' to 'ree'.

ree (short for residual-based error estimation)
---
 ree/.nfs00000000000286a50000bdda     | Bin 0 -> 57344 bytes
 ree/.nfs000000000002a1270000bfa3     | Bin 0 -> 20480 bytes
 ree/CMakeLists.txt                   |  32 +
 ree/cmake/Dependencies.cmake         |   3 +
 ree/pkg_tribits.cmake                |  23 +
 ree/ree.h                            |  73 +++
 ree/reeCorrectedFlux.cc              | 299 +++++++++
 ree/reeEstimateError.cc              | 458 +++++++++++++
 ree/reeFluxCorrection.cc             |  73 +++
 ree/reeResidualFunctionals.cc        | 918 +++++++++++++++++++++++++++
 ree/reeSizeField.cc                  | 173 +++++
 test/residualErrorEstimation_test.cc |   4 +-
 12 files changed, 2054 insertions(+), 2 deletions(-)
 create mode 100644 ree/.nfs00000000000286a50000bdda
 create mode 100644 ree/.nfs000000000002a1270000bfa3
 create mode 100644 ree/CMakeLists.txt
 create mode 100644 ree/cmake/Dependencies.cmake
 create mode 100644 ree/pkg_tribits.cmake
 create mode 100644 ree/ree.h
 create mode 100644 ree/reeCorrectedFlux.cc
 create mode 100644 ree/reeEstimateError.cc
 create mode 100644 ree/reeFluxCorrection.cc
 create mode 100644 ree/reeResidualFunctionals.cc
 create mode 100644 ree/reeSizeField.cc

diff --git a/ree/.nfs00000000000286a50000bdda b/ree/.nfs00000000000286a50000bdda
new file mode 100644
index 0000000000000000000000000000000000000000..a7d66c4b90f11a9164288cd908cd8e194ae85164
GIT binary patch
literal 57344
zcmeI537A|*b?=*<uoy6eXO=ul#>hRR){JH(Z;~Z5mPR0<&C*!L2s7#FzCCKIr@Ph5
zNFy0+2sjRyJ?!}7#Wn;U`M{8{#fGrN@Pctz92N&&0zCKv7zj&9cr5Yzol{l!*6n3x
zq}aU2OZTV0divhFRj1CW?VM9}?#!C~+jg3BM%HEcJTjBH?<?Ln{mbj_o7uiKlbOp`
z%7>cca%a&`qc+#Bw3{Q%Lah-MMjG|<NK|ZhKQdb3{;11!v=J2#=3A@hO3~cvVl*FB
zYV}dcU^JTJ-`=QMF1GWPt?g=|RjyU@mF7sH(4FVt%76m{4lIKM?N({^xew2*9UogG
zsh@oM3iIewFJ8ujgCqkE3^*|0z<>h-4h%Rj;J|<b0}c#0@c#=3TJ?uzuB0;ebrrh8
zecz|!`(f_)<J@<(<Nib4?|Juqb;tdOxZf@J{i}}qzvF%%ci(kKfT!cZ?)p*peOJf*
z2f6DN_g!!m^3OleUH9dGw&VW&-Ti6zz4?Cj5C8nb-F08ipLg7UfV=PM{BIri-OtPh
zm+q|{_ub{e*MI{94h%Rj;J|<b0}c#0FyO#|0|O2WI56PAfCKjm2lB;C<{>D#j}?6s
z|G(G&fBlF|=A+=l;0Ra+&IFGJ8F1GrnauaV=fQ`-+rg{AE5Y-@Rp1b~92@}80%w4S
zgI}Ob-v&MkUJL#RtOr@}a|{h111|(kfO#TQ15X0K2kyqua3{D8Tn}CY{ungCrC=4v
zfX`!axE*{DydS&|+yGt-4udL~1LuKr!BfDg;3pU&J_+6k4uL)30x%371Ad8u{|)d#
z@D^}2I09CK?_eDGHux6!Fn9$hfiu99!7njF`~v&~cr!Q-4uXroPOt;40pG=#@F8#|
zm<BoU9Sjuj1=oSsffs@2gTr7BRKPm05<D9G5P5zd+y%r|aXq*e%z_I*4rIY;KxF?T
z>>=L;9{_IxRj?0S1onV6Kt9b@zEv)m`C7SX^2Oq0wN-AFqvlj?Prg++n9XFKn=yQ%
z`sz)ssMQ>)x0?s2XYz$Z*=T43x5}lg+?$J<2Ul;JjapL+^@v+$<3^LOm)5V}$&K~v
zrzWR{;(SW^LUckI2?2Z@C6#7m8d0m=sG3r~A{Uci%}8jn3FGUnMtgaw^kJ=%DK{^y
zwX4N^V__@QX6JI0B~(WT%9jlpsY|KW$d;=uQ#KpTnhkuN?~+vcv1-+jIZmQE<;lhM
zY~(Ufo+)E4uG{n*GBdSWB`9#RI2&bS-VGbdj(@3m7KRNB#;_LDD%-A)RE6ndL*@cQ
zjU%&O*kc(IlTpf<c0ENyyWT~B);KP*HvuI}!0>2a_IC;;5lbi-$8u%5ZA0bew(90e
zt=VowyGz-*S~03jN3bMk?B61XA={XYV~nKnoJkTP7n@_LTrKWQ(qzYTX1Lr^E;uo^
zNZX8BBQw!#xtbk9fGxAaUW%$kd%=s;m}aZEem%+0NBo=CuP?WvM!r>R@Q^gV{zOSF
zVxn3#B`F(8hg)#OjE<Ue)wB*qreHZ>Y9*7eF34BwV2y^a$?2aXS`z8oT$Bg2-tldp
z4OE&|%@htsg+p}JFr^SvZjz1k;A(5yGOk7Ub<Kzd=4BS8`<!i5)YSJpX>YdfWG0L(
z(0N*6w)t5nGwCuxarxPpBMctOlBxV`)(WnJN6Mt0)Q)Rw>{FGsL!M06AEeG{DzwtB
zwZvq|K-QlQseSsBY8ES{J=rlJol1VieQQ#MPj}YuTxN85sj-03G)Kp`%pMy*J2!UL
zSZ?gx^KxV7jq}a#HRrC8|IXsSwer_FXXVa1J9p07+}Y>m#?Q-*ugk5Kr=Gh(`|jv)
zifo}8fgB%OvraQOmkiF*T;#vA^*_0B7MYIa*2(R$+&O3GGR9oAcS}^L&DG7}hEO-x
zV%Xmw73dh}7i;aAO0>xw?RR4)zOggkYLt)0PfTqvwDBoOni!sFnN?=YXWfx-Li?T;
z+IdIvwv4vq_+hu>s?2VZ)Jb4>^}=FfK`!)>mfYf#)p^0JH1hnA$v0%$pr4hSM*F)J
ztx;x^oXZ6+MQoNFF?my$N!Ao<l?t5V2{-#@j2Ru)&Kl<7@;-8~%rnyG^=Z;h9&VQ_
z<rzjHHdfM_!k?)cFqYGRHZbWo+J%;xWDL-0UM3Ui`NgPYN4I$^xX!9?N-+}Ghc?7s
zT;D6@_PTLVTzcGK+)Nf7_R9zvbKL-Kr*fr(rn#B+Rjb0KW>!+q+DSEM*5u5ws8MU`
z2mya8k;$rHXgEo>cA$<V>ml1U<N9rLWB9O){NXcpMi|$O<J6O`+luI=p0<5k5)Q1r
zOghk-5vWqYB9bV%+?bX0@}t%$Mhz~MbcL|*V4l%tNdMj~cbIcrLPSC;|EpM@<7ZMQ
zMyoC5xv1Ji^2swIWs)huGM@1>TO!BfspTFFZP!z6C!6{uEy=Atww9wxaoDhY&^+DG
z*&){gQWu3(10$=(){#BHji`y-iskC8%tpo9k?N6rqo^IEY3H^`d!_3p!=}*YnQE)D
zVAh&uAz#Ti%zVDmj+%6^h$hOm;-aZlO_ZlAh!`l79#d<#>g|>(3AOa9YAau^N})=E
zZaa3_$n+T!9Vvs!p<I;=T#;S*?4pOzQ>j6zDGNZ;tX105>5N=v<SMwbwY$qCqbv}P
zkaxXSS*X_L%K3`p470k=v00}Qw)*lIVs;OyDEVncoTEhQ+b{?7)nY}ewiY(PCM{5F
zB*O0O=iz`rUl&fzvuLTX?5t9aIUP7;DN$-yOtIaNj8mhC<e#bLAx7zp=48ZZHzi$i
zm6?RL{^WACP-zz<b9yv4a`5!<LZLCQ4P7oL$+()nKs6>S5jAe5uAk5K&G~s&OS|hM
z2R8*7pP#SH)*9v3!MROIzULR9hfy)SRIV}dMR}4b!-ma;`?i=$nGu;)RK977QM1q}
z&(OBqGHl1T&6B(KO(Io>6nCvvtFNxe{&u9q_ITRj0?ldIOv#vkq}E`eMWe1qRb<s}
zAk}8A)S|t!f^|g|4RZv&U&e3-o;lwK$VD>RT&piM%CiSsCcAmajIS9RGy4i<*ikOg
zz+?<;i_%7`V^e*UZ_oucb2n-r*o^4^x1rbGf$l5%KN8W;SJ3g_3SI(U1fCDB0tK)c
zJPw=&?gzerPJbgf0_MT9!Q+7F`5yp(4&DQ90`CS_gBOA;z_Y+6Ao~Af!KvVz==uKu
zZU!#{M?o1p6HI}P;5@Jbh)v*u;C|q~;0x&fp9Aj%ZLl8<g9n3eQ^z~Om%*377lG9K
z3t;dWaA3fJ0S5*g7;s>~fdK~w9Qdu}z;mHIFm<a6U8|4e>-EaQR4q||$1vPtEfSN7
z*fnG8#_l@H@$AZ!LZj++4+S`#3j0FN%xc<{W}H@zRN*yFp-j~6oH>2=^qg7ibY^aB
zu2pfjU>L<-shNf<s2D;EC?_pUGKE7?QFZJrDtnkHsYB!s)HbcJz^UXVuad{LSAM9K
zN9eTnlt#Nc*_eYR-T<W39OwRgxzXa3uJZ>8hqGd9caK@G%f4r9+hgnOZ2Y^Q;aT7=
z?47*m;>mqe_9|xQtXW}Zuu&Cbs)Dj}YAti-VffQ%^-g8|O(BuH+bt}PC|FTzQcW_6
z+r<7BM@dY*mA0>Oih$^F%m61wuV|{BNpx_@C#Gw6z1vbUv_4bE%vmn1M`ndezS#`Z
z<BhR*t=Yk6?^<?$D;KV{MCPb`X``3eTdh4z={FcU{DlKiSqxF)n>(5<yW857v?2Or
z>byIV5}d6oC}{0vYT+_w&*qD#Cw5#iap}J4iGBMf_fBQ2Xwvwk)|!<oO;i?_%4#|(
zoOBj=@trA6;-cAB6)zc&<dkaBcVowh*hDlly(RKcQ`r`0Nm5&1&W!P7SW|!7V5QSS
zwJb4^_~s#|BgZAR<Ko>D@_P98s-?iKr{suTPS~ptjx@{1FyLu9R+{#I>p2w(kh&Gk
z)ni$R`6w$-O{aW_lA}6WU51G-(b-cun%^o*zm${{Ez7K<bRsgRe9`~q6z~sZqdWS)
z-~awC^!ks1Yd{&C10DcAhhG1u;22l{o56+P6mTcH`;FiRFbg(<2Z7ts%WnnZ)4v4l
z2B(0pqLbeNUInJXCEy~k6KnyG1>ZwAzX`k>Tm{yFyRcWh6+8#zz$)-i@I`#^?*e}W
zwu5b882k+1`)9#BK^4q_ePA4jPhR}-?*h*Q&j3#bKgZ|(Z{QQ)<KPwGG9bSARp7_?
z*#8E+23!U%0jGge!5#S1#ozvBa2XI^`$}*+FyKkx*Z9+a34Q^70A2?+%iqZ17H~88
zOYi~ke(*l<7r=dt;ScG*SRc<n-<)o;yCyFg(h)?>NmVn0JA&)nOnlRr6$!BrKeBo6
z{yvXjW5ZDy`?TF?DJ@9)Qdfryb5)<5!6~=imhK<Y4j4P^lB==Pj?7&j);`%IPdpC&
zGZpvSa&k3OQL}|1lI<5M>$aQY;xU`Kpgv?566p>aGq9A$%7@*!QY#!fQf@{gX5zx#
zd#9A=xFjle;8I_x%xT&At>s3ug~6^WCe(`AWY!EB8S1k0JMx`3yf4rX3-?&5=l3gB
zZlp`?PkP@>q)zEfod3Il1+>wOvPjtufX;WGGo4r5h}`|04s+pSdO;oT@iVOFTuB&D
z;i)*nakSwSFIqpIj_W5T4$qimO0bT>wZ_VAqha=LO+~HU_4@v((b|lwVzsHSvGUhB
zB!0j)0UR{a6#6|vC9J7fY_`_5hWmghD`YATky^bjwc=Kz)eL4TUy5sAZmulg=xS!`
zW2@`9z3St9!7Z(e>(1IU_tcoAbY66|P%l~Cda{guTyhq<Li|>yLKR+W;_W$Ox*VC&
zSWC!r?n1T?6%mWBX{j8R_U3`H%Pc8=CD2WEI{g}#UaBc{rv#upVBW8f+nZr#e)^X-
zYAdA$r*aST^yziQ9Mfy_MTAZY$V=?@(&+nfJ>7h?(DWr*R9LW=b5|l;wp2Qnl}i%{
zEzBug+oW@|!b)1Gl@-;JWZL?q6^f9eqv#>Ues$|;=vujsYkg|hovKIme4=I4&nHc_
z#vT>abbV{sy=X0!$LvZ~hH@M~!k^BStGk?Dy*h?VhiI~polgH&GUT3<m#iVBsB|8_
zQgJ377kcHAE{P0#b^=FD&anL{q%0t(I-M%5Urs)eX61h<!C_O3ds8Y!y6k?-az~#f
zfRV9zfGWwiTV}5w(`C9F3&KY43kx1yvuIkO+a@lF+z!~(I#V9cc*WOG+WtFOs4Z{N
zfEplMs7`F23g=!=iHrp&xO7;o?7s89pChHbo@M7N*eZ$($!e)%nRHU6_N%e(Q96kg
z$H}cZWL3SKRfp_1JCS9e=V?ZHnx4n}%A?61s%kMA#nNk!j>*q0K$lp3Xmw6>{n1g<
za#tYnM0LVz5L<ESjopioUj3YesexafOf5rtk&>GFyA~o#Khe1q>EKf5VkDiR9}t(Z
z9C7*jCweVNRH{9eB%=Sn9NqmkAo@S*vVLwx*M9?eF35vbU>N+urTsMDe<RNfJ_8O6
zI56PAfCB>#3^*|0z<>h-4h%Rj;J|<b0}lKqIbe+>y^fKo8(mz3fkjX)k7ABlAvelR
zdwdJemDqAPr7<6wiPrF38&lt`p4`A+=p5>LcPrkQi}tF+@o;10BI^<5uf6uK8Rur#
zV%XXd&2gY@((fhN`}BQ0+56s(<lgA0Y`yT_MQl7xyjWtl;53tcn)`bCClU)#c(>b6
zYY7q#*uT_z2??VApNX6vjqh3Xe;*(3Hgx?@flq?J2A=@01hV&k0!S=?hXRQMa65Pp
z5IcgL`M(7015X2!U<;T4j{;w3Kfu?(D?tI|!A0Py;Bnw<><jn|`0wBvFb&QIr-8p^
zKfrr|>=$??sDV8o2foI>06FJ>BanRp&jy>olfh}=*X$4YH}F&NJ@EJ7ufSWtF|ZR1
zffe8hK+gGp6?_Q10yMy3kOijzIp_aL@GkIDPy<t78yEqP0H=VzW-q}F;5s1t3yy)K
z;0Tbt2HU_o@L2F|_7!{pTm$|POo0c2_pzry&iAhXKVt8|KZD!BYrq`X3LXIN559pM
z-wv(>mw?9rk?jM)azC~!g<b*c9qcqVsxznaV&cu|uy*xG*u$X{r{d_6+BT)ZYgZAf
zy4;I#@|1Cp-q2y&lU}Xa^>yr|jc?8!n)cEz?!4-#uJ<fBLSRde+WJJxtE^AGbX_&&
zB2`_|i#48dVcG3hrSdq#p^Cumzlyo9qmuZrI<MH@DH)!eBdbqFoh4e=jBHX#Z}^Jo
zQYTdWOX-eVmu}O#hkRO{CRr~#$Eh?0&abst`?eL6@f$m|Z%JL<!Iu)G_U^2{nF~cW
zeM#n)ID45L(~~AURtOg%ZZ3B(m^ax?qcK7HPFeBn$HB&-a9@p<8pH{yMn`p%fD?=N
zp~W&KI)+aYN2idn!C#~<5MhS^sT)TK<$$7;ujlh+&)`wHZ~u@aOo`m3K&mDmVRHvX
zq>tN)EsHA@MI4PNW|?(jWi$`|bLIF{i6n%5{wA(s&121FK1jt94AJw-S$RkUcxWsG
zDRBt~b+r{wK;jPa&58|;$qo<487ZM>7Q(-!S?(KV&a$|J!Tzq}a9270ekUiphcV9;
zzgZ+?N>C{m)P#yklMh54r|D>Yz7+<n@x-tN)Xd=AJmlFY(hWA<34>h2Q$cG$80|%5
zCYw5C*a@P-NEL#;qdnLj6KP8f>nPF+dWCY5$T1SN#d%nd^M|cXQm3RTD22%YBVK6I
z+fPTT<u>R3LP|Clg;j88xrJ>(O&lBIx(G?#IS0?iy*0SDbUSr*%=r`LmWrScbT^qG
z|G3PK_U&?Yw_U&i&rSva-@avV%}C9bYL=2FC^sIz<a+PkBNbV?`koL8(9tyv`kSsA
zUFG-+`Htb6QCC8rkx%GbQnx&PclIr1>2{KC@jbW^u#NJA3?;-s*7hH}b&Sw4W<}7E
zCzBaO8-%fQp_Nsk!pUkIQ9*?t|Cav7$8MpuJGX4`k6gz!&1WdJC$*1$7c<nJ+wT?c
zNrnM`Xg%JbYO)MBg~E}cV0&R4fP`2<JftVnRi|v~ESPMZl;iAq0vC|T6!ciVUXe58
z;b|M)_2@?yAJeJn_Q~6YgOcxe@S~1$;)VEBn`E!Q^2$b^^3LpRNuIWw^=-h89ztc=
zBF74Z(K7n#BXW{}uu~G&p<K1`hI$RR>w>YiqhwYHY<?=)Qe&oDNfO+md0-u7TKUFz
zdxyUwr#MCYe2;4!KXC!px{~)@(+K;ix$r#LGmVYG>}$^nM>tEOn{5ziv$gkNQ!cYD
z+-^}8{eLBj;ZJ05Ir@LaDTyCMx4#<5*?;l>4}pgRiTQU6kobPHK=%Ki3S{s9SHL^K
zwLtXw$AeSB1HcE--(Lk@3?#1Ke()gh3v~BC12=*jz$?H45S{*;=<Tlub+8d+!GpoK
z(9i!D_zZX-h`?#`8@>E`Fb8&lG4KfRUG(s8fUg75*RKO_2G0Y}1ZRSupnu;C-U!|R
zt^`}bQ^DyV1MWr_zY}~Bd<5JAZU)za7lSsq5R8G<;0$m-@Dud*AA&o;2Y|%zdjTkc
zN$_8R#Ok{Ryb#pEIxqwzF5eyCCU6xv2IfH<OoJgHviq6HZ&{yIGr6!*Vz2U!lC+hu
znQ(naDb-L?k8Q2Ak80EEcucN$JSF{cMcT&Zo2y@l$V}60#YL{#Xl5#<ZA7S;CFTg^
zB|mHzx9NRZBrj2{Pah_6jhz$_FcTl8QB_}lG6D1}iDJ8Rjkozp-8IW<Y1N5m!^9FY
zBqO=Gb5dcMP)DcVQ6rv>)YVroTbIK-UK?@GB`gY_k_l_a)}HCZYA40W0~dBa(KQlL
zIE7FHndp)eP+bezY7`8uj?R>z34}Nz;tK9bMWq4txaEZD@5rjTDrc%7#7@EfNR_o#
zRGm_*vvG3EPE8`a5eWfhXKX9X5yB41sVrABw|7)mJF(zT_;F&8br7f-O0VF0T7gc*
z*6d|87WwGW%tVLwQ*6*cgr%0wRBLkmL>7X+DTVuKiDlDj)E0KiX*U~9CtI5HmN#t!
zE=7Dds<@S)EsRwwM1<KJip;hSapGmZDFLg7`0Gfdk!ps`<;BRwKh&*hazL=%lAt-Z
z+uL<82Tb)z#O%&NNIp$Dtmyl>l14wVODl~Rv|mF%WlK?oy@9#Zla6AygxD1wnP@cf
z3lb{GN1t1}DfIw~y5D>6+7(9>?PSir+G@*jV;Auvo2)zNnjA!mOOZxuLnJ2iKhaJ1
zZkyQkv>lVa3K%DS^=&PUR@$=w4sz7D-EaqrQtaZM=#Nojb!3HR^6bga&vGtFq%q|4
z@FH>*?;-5D$S<;GIO-+vs1l|Sf*f6lb!=pGT>G*-th!0XN}p4P<J@08HWD+%<!%MG
zma$Bdc%JHEl~R>;mlO~d=mou*>se^gQE>cdxlc+b7>;pOg52f2v~R%T`RO>y8gYKR
z2P1+`AxrL#{DQmj2{doeor(u)VGCms1pB(qxQMY+HAm4i%fa(hUqYHme+n|ALC1WQ
zF(grtoJAqkI+TUJ34|gINuG<mv812allzjP$W>%E=9_QWdF&luLwoe7ROP^pJelOv
zAxrMVnlUfSj&y+}<KqwZ<olA6?8qsow#cy~x8QCd2d}KgY`2v9DxQE4eNLeagRZX}
z%1AS@{U@HEGJ44k>Gp3r2n4u6Lz)Oj+eLrfr6&t}P~TzDNE%Ux$&x{Qxmm|1>_zgH
zCJMh)6?486Z{_UMdXs_55)wA9ED;j*Un~nDSERTMPApAsJjn}>?=?D-R)D~h9-c^~
z7teZi?vhw^ILxb|5$(=y;>k7iaF#n2=o-uRB&0+V{lcG>O-XMk=B!(j$t>-gI+N`7
ze4#c|&R4fY5{Ir_mEaT^TZ-?}K8dUzbowoAk2BRYHs0#mqiaoRyB|VaX>M`g<^sxN
zp>l~X7;J9LbkiCk5z~D-Q=8U#?WEu^Y0}|h0|2ITiNaka3U{^6cM`2xn-p5VOhK(f
ztR8eOwuFX_QC1dy(BeCuh}>A`CNOJAqnB4|HNuK8H~STi2IQ;DsY@p;{(=_IY+YPz
ztG2XCtSnM>a&=d7;?a@iPW-UO1zBQrw5vPO|DT55C}+<_|M&0LyAxgiE#NvJF#yB{
zumd~=$QgjIfOmq|0*U*7HFzQT10c2mu?w6A9s|CQ{{LQZ6L=G7foZ_&$TJTIcVP$k
z2zUp0D|iEV9e6E}*#F~T4UqVLw__K0EqDPaf=M8;0wl)YBf&pmANVYg*#DmaVn4VT
z>;q>2*$?oq*a|)d{usO(TnVOu#1mKvP6gk`ZXouA+rZnv%fOZ3IH&=!F;qYqYyeLJ
zPXKpdOZYr^6?i3Rf$iWN@DL!e20jd`K;r#B9XtU%9!R_au`9d@%mIlvAh8A>0KSBc
z;632&;17V<4=x3x;E5mu?#5p5Dezu!92@{6U^VzD@_j#e84y`-0RPdC?fCk??b(U$
zfO(8SEAgUEmR-H4*I*EIty=m4upON&WcgTXsykN=lU`-hA~lSSF8U=Ase3N!^kGfq
zAErm1HNv#P3cg2&RKJX@pVbiW6IAR<O6VgviL0BpV~R7;?yO<amINZ{Hff7qN{Ex7
zvZ|^2;W+KK4!E;;xDpWpby_!qFF``1SUW8y+hSMnT$g8XBAH`v{ur5|UMU|?dYaBi
zIf<;bro+M08%?!csnlB<(z{1WKl%Am#PY9s_HyO~E>1iA2HvP{zQ(EjnolTCLRiQm
zDOeW;jCGNOQUbuo9Ha=`P+fHol2Iv*uq{P-yNKgK;I=|aYe&Se1Hn3VGkN%cuHL6(
zWh1<H6JwWVEmjaUwd;)5t~SfF67NR)4s^H|OeEwuV@6<*fetf163k0+gmCM961jA9
zxhqF$%vMYDE2bbx^74-ob|T;83HdK?M39E65|M>W)4xl>_91&oLpv}P%d@f{MMM<b
zk(FdTp$87sROlTo%4*eC%hzGZHr!2tyrhK*_td16Y74O=0zp}afL26Xg1=`2Zo|?s
zD_W(}N2=yIvQ9-{55&67+c}*`d$K)6(3FHaAydqkZV-|>*_WmSF}qzEw6ty$Vqi#$
zSjTj=YRTQZkG#`IUM(Rj9*JEo!ImZTuv>RGX+~Hn)#MP5mK=%2qEs%lXqT;8r=O@Q
z47W{2)?(FRECeb8F^nB8Pt{sS%fnS0Bjq?rl|*kc@CkU3WD;!SXjummP1VD=Vb?k?
zm$`ghIWkpyTBSCVuf!`=a!EuRlqGZ}EV8v<VuIr(ED}JS2dz)Zu1sq-K#1-mgYfRc
zxR}(#Q!q~zouJ*Nv?7#{=Eh@J3=bjvsbnu4MA3qUE}bk0fsGioUAlvzPMdJSMr^og
zHiUL?U*r`iTu$ed8ry}oh%026qiQi9n6E0%x@K#oY!FJxJ18x!=*CqSw_C~&Ssk%;
za$&Z5R6(D$778=Vs(p4m2(sr1;^IcL-EHtm(KIW1=XO2+?-|si;W-PcK8Y}5<;dwq
zRyDG<&^K9#TBKT$y%sL07F$ob=B8pL;yW)TD9~+rolnioL=vbSbD;Qs+l_htoFiM`
zizV_Q<JM+pewBgCM=r7P+VzU3JGVS{Ck-LRt3<4!chmDYkgjV>DM|M9D9<MotjzOT
znqP{|V}*^X^R!1zrNgB4E%%x&O4a#bR_vbf$VulojqYLpwu?8JEqIt#(BU?w_G`58
zq<<&##Js*VM4r`*JD+6@$+OXX;B$_j<?1rs&7S|kTqltYL*|FKXqa1(yW3WZLO;xF
znuz&duh*Kq1Vcg&$yhL3o+n6sBpU#TSZ^H<yp}|gK22}s?Q&USEfHl<roXbhq=W~=
zE=~?ET#N)|@*#1Sv3;Pmh_DectQKM0R7(iEwS=&_FT!5eQ`ma5Q(%m02#mse2%D=i
z)Si4p=@>~*U~j>q!Vabc-T(i)D1yHy`XKuMai<i113mxi;CApj@Eq_ga48rD4+h^s
zx4!|r4v6kQ1<nRpuo654oB{p`9ske30yqL>&;NFi1NR3%L9hQ1cmdc3&IM<J$AVMA
z?*NGtcpbPFTm!BKo4_gHVL)R0e-gX_RKOM>@AW$eJP62p{XP#q0p0^{0`p)iFyP1N
z?mq(m2tEQ{4W0$IfHS}k(A~cTz6kyVTmzm1B!-}Wzu!;M;lBWUEWzjS_a1O2kp2Cq
zf=7XWMW6pNkbMMy2R;wp46XuKfc4;9@W0XNKMQUFF9$^+`vuMfKS!tk0{AF+F}M_5
z49)_wXW&~v_6@uTyb-(<yac=${2|x@RsfOv6G6|<q62|W4E+WIStoXk1CqYyI6&IY
z)nUcHUuD#JHzUsD6nH<Fk4ri_np|Q0T|G9O{FWYP*fOBkM}}=qGDnmuW!4pLA;6F~
zQ9);tg>0Y}LQ1H_=ZwBQi4lfpbEZ+_JsQ@zCw<!8vsc*zDzD{}ghGK`*+K=*l9;w(
zS;lvGoYb7j9s+rY>8W!wXul_dDa)<%wZ_(;PW^k^L8zN@(vDTm!@Q+7$7^qPY<1>B
zJ1cka1mb#~tg^WY=`11qF(+d)G&{B$n-{lrzKd6}exmMaGm)f=SVUAptk$d;bly)U
zNJq(%CJ<?Hy|mkVK+O`(iX9a9l7hu6eAs3v4P|aAJxf`$#ER72jt#})(1$U$@mk?Z
z1KuxubuAzcCo4U&)=;Awy_&Zz<;;F<SH|p*U03mAwk5>xY1^N2tK;PlGRt0`JH#}e
z^1>gu{IUawVwZosj1$K=rjA2c@#PIj<`8ClB{fcQxlM7o*!hba%;i2ouUCzQL<oI3
z$<%TL3Q=!|H<}}s74`O9`Qm2O*xIh@Yogr#ZdPxzs!}RL95+-DT{@>z0BrIwrRCIG
zmys<_vtY=w_mt;uk3;V*4$6ZgRK3S;3pZRW!?A<g$TC2r=t=OJ05w+cj(6_p{lvB7
zJUgziMf^G`zXoobkQLN;oAZEZaSRRK{e#!vddy-;%jn|FUgFV-U#V)Faoi$djgBz=
z94#q&lo&}KmrNnO?a^9)*O6q;0SWa|+ELBIJk$tv5$^2?i|Ti1Y<A3pe$Cf66P;~n
zoN)zvkJQq9vtF8xC2qTu+ux$OE@MNC-46p9UbfOig-7J>D~;f`_*anmt`YMR?ZES9
z@eMTHx|rQc)5GXxDR4V&M7>orx*E!YiK5F{kb0S3O1JXi@sU^eizx|CZl_Uu<jGHb
zy(d!Ts|Z?l9EcR-3zN<;!cJ`&aqXyUN)qkg*#eDxN?8tFv5^YX;wp29w;1G6xb$20
zQ}Ir<tiN~^YlQ1RJVkKfWj-}#pNjq8j*^5FI|{|5B&!TpN_au3UpBF#Y3k=hYB?U)
z(u@wDL`7vasI67}FzQ-gZu@m@-ZGqhr;X?^GT1r2XPdCd-PNm%6C>*;&z8%hqv;!B
z=seDKiOh((E7hM@nxlQ19B*pS19n|gPu!jsYhbL;rgm@HZS*}0+jj3leJ_eu&-xwh
zo;Ul|j)<A4Ve$o5>COO%Ij~tSMyLkTBHUV>oujr=$?s{{r+aldnC=Jg(NPknZ_pfB
zsdrtL+<1*@ykd6+{Kl8;u&5f%+85<vp|F1w2VmD01Y2@;vU-^pBU=*)ujIo&C{dZk
zn89Ewf%bX9p<PBK>nN!ii>S_e->b@i<)(B?n5zw>-%3#Y|Bn&nPjtYJ^Z%bl$A24m
zJrMu@#b6RlfR*4uK;8*(e{di0m+1QM2NkdpYyjf_{~0>IoC~-DOo1`*6mSOk7j*mo
z3FO@W^`HTE0C}(96T!oPoDsMaybH)X{p4K%7XW#u-;dGle*|s;F9DZ>N$|ViG2ol%
z_FmtADSwNve+PIfSOfg~10KoWVgvYR@I&x!a20qS*aQ9x_!_$YSHWAsYk-^|m<16i
zf;<=oa_;{j;8W=Ow}Pv|vw@uTe<JuXy8Tyy`1vmZ<KSP=>HiMg4BiNigIVxYAm;<*
z%>S)G-YIYlYy<Kh0Xg^ob!7KB@F8$LXo77(^!g_Nk?q}m&IhEMN2eyBlV7~#k1D+R
zlSLuDA(w8?I_vb9DbuOqML%85XUbHn-XufTyvf-d;kBIcHP`R7T<Jq|OtsVcO9qYj
zGG=Yw`7Xr8mphk3JJaup*2KERcdH-KUyHn9HYA3DkW%E+LkriPqA@vIf4>f}OS7H)
zbnLW+_fe@kC`(sS$Z~4B8&{MEX*!Q)l*oQmCfrF;xf!MjPJ+fHM>`WFG|C;_>e-7z
zk)}O#QC#fQ0BV{n-l<ft1|^xq?I^0~I?q*Mi{x)NYwM5~FJ|rZyU7&e2W7rrY#r({
zP8N*WA&k6!XSY1<)|$O72ErC4!BN&<A?Vk#f>rCXl>|=<l)7EVG#{4Y#!-P`MyD|$
zf0-Ltluxpc$LF2ip7PsZjFIy!_mVYchPS9M+a42-R_PudUru)_y&Mc;_P5j6lB6Sz
z+L5NT&EOJmhvyP~T^gYqJo*huRwF@^7sZssb`)L4I1xUWl&ueHuk9Gsk|bd{ei^J?
zr9<ONXCNBaT%J7%x&-#MN)KI{i*%NL>tOUOUC3&}e(Ryd=R>O*#WLF|9o?9nfn_K?
zW7@S~Y$OY;TvIu-Q`yDift=l-m##ep$XumUl@_Rlv>N#;uSw^fDvOljis&Vi@Tn<Y
zErmveK~~03Eki06COJ8?oaB1X+vOxzS2Gpnz%<4{ed4Au<=uX5lxvL|lG4Kt?La4h
z-(p)?YTWdF$o27feGu{?efq1fu}fMY)1nN6c1n!RUD_^%Z8n8(pIhPtZs?5DEw&Cx
z9VlC_J<MEjg<GDaEoFP{sk6A?d(T{!9=>`H-nQL!uacEviw)x@iEOYZYKSYO=^Rb|
zOEQ&~D>ez;W^Z@0*wM`Fii%MsD%dl2&Dis(<(|Z>;0<@npA_l`7n;*DBR1tQBZlnM
z?~|Aqf>FC;W=Ln<RTND@^#9GMi06o&i2gs%mwY7V-^anLK?Br54UB*s$bcWB+kXUH
z3l0K_2XGdUIDd}>_W^gK<NrJOIQS@dIk*Zu4?Gtf0~dh%f`3QX|6A}Ga1(enD1ohD
z9e6bO6*~Wy!P|h?0wf0DIpA(|{a=Dx!8?KM189M#gA2i_K;r)03f>8>0e;W_M*e<0
z$bes==id%w-~Xq<r@#xr(}C;-_!WBn?cg?WBX|*z7y_%n{lGWT@4p1T2;L4P)?Xg{
zK3ECF_x~Ai9T1<t#11$HE(IgtXXyI>2>uFO0}g{acs6)E_%-_e--EvZ*MeQ(>0l#}
zm;sLmUqO~PgV%$@pagy&h>Ww~q&~79NPl`QD$)(kz0uX%la1~taGAu8o4`fKkXZ<v
zbbYq>CJLftl`b*0loAtxj?TIO%MR(^y7kizKAWWvdjEuav*KQ<gQR3XzJ-xQlsPwS
zj1&qPxsF+PI3r)N)!YvRa#22M!tC-18%A7&_%th*VLsl@SK!^0XJh7w9|^Bk#d|m_
z-hr7))z=Ri<P@+u<q7R1^d9D5r>#9Bga@w4(cmDsGn4<PaZT!!^BZ1&u~}MQeWztF
zX`Z-8{FQyyU`r1lYM_%bL3^W8+-J=ecdEV%Hpzy*6!?8E;620Zjh<ruzq(xee`Gx-
zgQ4IAmFi$MrVgeFtf-8cA`tsuunrqxXR&PVVF6Y&4UGd)tsT(~r145lNHF?e@F1=!
z@vLa3i?8CeQ58AO#0#sNk&DpGl1l7X=|IiO_GD&~1XI~^?(!iqG~2TXsWCPjKM8CS
z?<9$b>Tm;si+o}=X{E{AyVVLjVF~hYW1e!ejV4UgdLb!J>5}NRmXB0YE#<1#Qm8wy
z@Z_m~D$ltVd3k2L!d@Z5RkvjktOG9w!3o*CE9`W4G3T4?#P9^iNmfzsHJVO~Aw=cs
zv7EY*<#2}baKc`cJK|pUeDZv>&y{l;6GxtL4{>FcKHyGy1oKjgZL7zEog41q)P$r3
z#MNi<T(!p;bQbOzuuYzDDi?pT-A@kZxzKISoTH-G%F;MjRZDT*lzMqBx$IO6Z^TTJ
zn|ec1bUzp0>c=c;a%EDv`>sHk6<>`xX4%05Bu^MD)$`X*hQ)ENHcKDGqi3sjuQX38
z-zA!<L^MLB<(eJfO2J_~vBw7P5UGO`OyZ7@4`S)QQGb4ovSqN2Hwa>Z_Ahx9YvQ62
zbR@KT$`TwOabm-u>~lvgl>MNxovkMEMcAro^~dbk*vQz}xg5ci@AK^U`1p_xU3;Ta
z#5w)~@21`nHIGGy%$8{GQ0+*$a4hG}Rmk9_j;(|B)%jw+-eS9atav&<daGe$)X9Qr
zh3<SRx61iSOfdY)ncC2h_rsSs-NAe6<2vj1)0uX;QcNC+|B{zkyPYSSHFn?bI{W*F
zQjUE~01U<vrdw0WQFE%cC*LX@)Uxx%%h|zOwSlL6uhJ#;guA06wSv@xa={b#Shj1a
zMo5UR3brg%^K<1w5X(ml2wI7S;Z5-a)>`ZM?f6G~g5hR`sjc3`ZjLr*)>K1al*q13
z8fI>cW1?wZ_ma`_7@n|rA!M)~8r9CCy-!arN<gY<_MdHZTiZA&NxA#%sV1|Ejl*s$
zNw1O@XA5STCg-e*b}#;?(Z5Cif1Fbe??%5D{r{uj9pL3a-UWCZ90TWr^+5IkJPO>6
zuKz#4C&8`YML=Q!Tm-}>@L=!*%D4-B4`_KOrHBq58gO91fdK~w92jt5z<~h=1{@f0
zV8DR^2maG?py@swPAwNOU6hNxRCufFTXT<EO_Ja|)`W=lETz@E(=$DS%)|yDYnnO9
zjQn<lM6FJd9;?Q^G}}PEm%s+PtxJga@su%5CZV4sgScwe@@kC~s%s}YJ`Z6~u^h5B
zi(ldpur0n8TLYAAEKiMAOlsQB)a%SoK(r->(lyeB(sLL)PI<=dUVyB_o-VOu3Y~|o
zQ8TdrSSq4cyWW?I$uL_?LWdzj^#3yY`=0>O|8ZgKM|A$TgFgi~f*Zi~K-0N2gRcPx
z1{@f0V8DR^2L>D%aA3fJ0S5*g7;s>~fdK~w9JuE=U<`T54lm-Bx8&7*r<?4q$xDWe
eX!^39t5!8L#m0ht&0jymM`Gw0ieLX{GXDps#)W49

literal 0
HcmV?d00001

diff --git a/ree/.nfs000000000002a1270000bfa3 b/ree/.nfs000000000002a1270000bfa3
new file mode 100644
index 0000000000000000000000000000000000000000..fc89809479486b99aeaa5827cead8ad7ceb811ac
GIT binary patch
literal 20480
zcmeI2ZEPGz8ON8jw1p;ZX+9wM0F$dId*^$xU7*Nz45@R5ywxG+6d`EMdUwvZ&Aw%K
z){gB)r2;{PfT~}LAgUB0nj#_bffp5(wu&k~R4qsmNQe(q5efnX-;0o_%Kw>}-Mu?U
zkb)4ZG^hM>zIlD-|IEzpJiDut_Z>RM?rwj`!sj;2I@Nz}<*^I9tiy|z6*_@`Dx0R9
zfB8w{Fc0#qow;$!-FBMzZSLhet8@b7cXX?rlzRh5w8B0QTOMEML7a4`fesJxujg;^
zMV|*=+jV!gWKxU*Mu9h_KrZ^NkM6PVnx2}ZdUw?JvAwsSd{a%B!bSn3fKk9GU=%P4
z7zK<1MggP1TU!B<TxUIuA?(t_IHSMcHuinJ{(YnVerPQJI{o`D{r$*TzW!;xi~>dh
zqkvJsC}0#Y3K#{90!9I&fKk9GU=;X2DByUO^%DC1IRyal{J%K=zkaJ_T?EgA4KM*d
z2=)RCym5<V{R#XATm(;p3*ZUx2uQ(ckbwKaVQ>TZ`yR{sAvg~{1Mo=FdLOt6Jii-l
zfG5CLz!A^^f4SMRo&t}9Z-B3Z)8GMc7_>kQ+y>rwr)9kkeh;1j-vnO(4+9T;9NZ56
z_72N>8GH>q3Nmmv_z2hsZUL{|WLbX%e*jm&cfohSgP;dm;2#KxTn0Y|-vbxGQ{eO9
z0q|b%*BdYf@GI~GunHV73vLFlzTL8Z30?s&gLB|pAOwq`25tnuMiAya$iY$YNzelC
z0=I%&z#i}-f-x7tli-VB2^;{s!DaZFeEl?d6g&hX5Q0T;5X=Dv=)>57b)GtFd?~T%
zos2%clhI(=R(-}w^3}k1XRK|@4V)~iO0pTI@|ctKXJ!_hbzf{QC6;v-?a8E!i=NAv
z8|Q+}&9PcG@^_8WGWw54UNDPF|0X+Qo&AglJmitc+G}VWrrLE?Gh-9`Rj-ucI)Rhg
zyx%O#f7ZXh$tGptew;Ep6W+{Bh6Yvy^Y>5AGM~YaosF}J3BL~8ycjxgmZ2zqg{^sX
zbHIDhabb)NnmX;E5A!lVj_i8Xx$IU-QdB9sqSj>2F`f+?EM(OvC<S9c<!x&{_C4k(
zCoRYK@l^0LwyY=LpJ^}}>+tXO;y1O@jRbsyLXoJ$-Q=Kbt2M8OyFxOcUru_GJ(L%s
z8cIZ-U5FwFK~;ynt8}J{7_ICDDmU3c7A{pApmpeVkY_$#2%^$<wpCP+MK=aM)U`=T
zSc>A64ZgtC6uh#dUT51dqg3sXED)y{pZ4EK{Z+o^N46)sDeDX=g8pdGek7PTQjsjK
zyh&x~QyA7{H4JBq*R<}vg#$+qoTz#!{6eEev$AJ5<rqm%@I>a?aBL<V;k!)w7aexh
z2&j@OLVxwk<*0m@a*?MI8>~cpLz>=aw*0~7rFpAdIE_JF70^RcX``jF{NQOPNCrEK
z>qIKAXTIuSKN~RG7+Ui*_LaWlia4F69b_yG-6rt{+}xTMPKuGO@HCCnPBG`?uO_pn
z+pI(XHxf3%yn0;<crCurW+i%jaq4kpNMSEB(CtE=rkGSDqcn!*^~lWA#YgNyRp~F(
z*INmVLz*nTQZ%4;l+ua1DbyOC=_4N0lVxmCVhUni(%LFqt7aRm8@W-1n87eR<W82S
z+zw%Yx5hJUYDzG<rIBjoRu=Kuv8}6YEXqTk`tC~LhrY<>l-2VyGof=v?WWXFM3Yu+
za|h#%Xv0ad6EvrqvfpZ!7kSDavbxh?slPTr3-Jbn-F}#dGM5EBS`!22XG{z@gUNWI
z&}p~ZtkJ0!LZ#ub=gM`99r9`tYW0da2-D>@Exo2zRI$mG>9inFj=sojDk>*E(&TqJ
zsfVX8GDR$$Kuu<vxl~W>mAK5|QsC7aERy<EIkb!VLME#HbGVJ2mn~^EEtadyHO8)C
zr$k?RkJM@o_Xla(G~iOwYWwsB##fm=ym4O3iY!|k-(0o4LpuigDIHNQE2PRFD{{jt
zydqaNW%cDnw>NMST=nJOOSI_KoGr|*3ep;*ylr$}?qcL_x}28@($`IyBB#CjQw~MS
zkBbVqLYZ`x%)Uw+UF0LXI_`q<q|CRm&P(}q4<rnw;bZ5SAFVNm1u^bniYq?01vkdU
ze-k@^7{ne6b9cZtxO0l9SWHgDddE-HX&uS+N6RbaSsn;#i&|$=(Pq&O_gEkA^U8Ht
zY<yKQG%_;&p#?-%Vu3qM@Gyx}2Nubeu}j4%b>^u{yzDe%j;Jol(l|XIi$$k@>h)Io
zWO&8t9#xX^1zenAPl@|js6o+z%)%X_TAvJi+$yv|89lyCsiS+O!Il@VvEi00PrC$4
z`d$ua$ceysCj8LBa7W}$(3$Q@ms7gLtHSXk=<W1zm%{iN?&i`f<Rd>!aBJ<0al!k=
zps7rh;O;q#15}G6>cbV-ytV9O%~PX`-D6aIoh$ojQtNADIX99x*o@-P$C%XOf=pC(
zXD|11k9yR-Nf(x@99uC~*evFHn!AGaAgLPjHf}K^2bl@(ju{Vex29%wm57VltaXPU
zxk2u6+(?D(K@G2~7726cq&QEs2lG`%%}v+k20-P@;*>g?fcjlEZO0mR(R1Lh4mCOF
z#~1;3!g)(7)?oMa4l?)!xAB}g8BJ+7^;h9TKaw&XJ+#ohx7URzxMkI8B2HQXz2D9}
z+~U$r(PkU%vE>0ScN=kfiY|L`!Xt>Br;sI!`vMnL+&1g3JeEh;&14UlDj|wXub>rb
zXdzBE)pgZgsI%$GsVUZTeY#!lV>T?rA*KoLBr!!!@Qgbt_J;-J^3>`yEIR*R!g=_o
zI6Kq%zj%Is1!w$A;6?BPcouvWd>O2RE|><p!D~3zQ!L;L_yu?goCBL62Xy|w2Yd|7
zgZF@UgI93oe+E1W&Vg@(Gr$9%0CV8|;CiqN{28%;--7etOW+jff(ft}T*4S$0MCJ+
zfoB1Yi@wdrC}0#Y3K#{90!9I&fKk9G@YYq}?Afz0ekdxKZMEs(s?MbW?)6uAf>XD!
zr<%3yu^D*;TclVq9=XtYKpj~=&}SLK2a%V3E~+)fR7<GGAU>kT`;IT1T*2WXT<P(k
zZ`X&ObciXOvnhKSF>n=Qkw-rqheR50%6<_2YU@+4jIXJ<pn6I{=RsBIPCQ2<n-Le)
zhq9e{V?h#BL5Pzc-wXv`ScK3Y1*@AfYA4&QPMz5n^-%=8CdZ85MqZgKLw}vdem32n
zY;<;pgAn4NDB1Lf8=e0yG9(3)A~b09%Q6t&Jys20xi?PQPLc$h%W?I*LHVeBEQhlr
z#Fj&*NLL1NF+$nCdZ=*JSxcR0O-o)x!{{MJC|xu9n4-z1(AMZ<rOK*ORQU*SIABFF
zwejd4tB(FCUqk`O87kOG!k5X)bof<1=aJ(<_+{+5)==r;*+sq}Kd$?!pm>aWCPu>9
z<U`e)4xld`Ap_{66HatLb+X%AR;e`g*}jyY&M}#fQlt$Lc>7LeRC7u$gQ2>;BZrPx
z6{Y9{CsR7uL&nmd<<Qi$Qcc2Da@7u7zmyBCQUn69Dx?_dao&i+#jvOz)0)|ns^zj@
znioTX$s%r2My8QT=l@UR4F4FQ^FJO}%g<%}eii%<`~>_6d>_bix{UcU3K#{90!9I&
rfKk9GU=%P47zK<1MggOMQQ$vP;D7nLwR|C$Uj9YoLB5KcA;|Jiz1fW!

literal 0
HcmV?d00001

diff --git a/ree/CMakeLists.txt b/ree/CMakeLists.txt
new file mode 100644
index 000000000..c574820e8
--- /dev/null
+++ b/ree/CMakeLists.txt
@@ -0,0 +1,32 @@
+if(DEFINED TRIBITS_PACKAGE)
+  include(pkg_tribits.cmake)
+  return()
+endif()
+
+# Package sources
+set(SOURCES
+  reeResidualFunctionals.cc
+  reeFluxCorrection.cc
+  reeCorrectedFlux.cc
+  reeEstimateError.cc
+  reeSizeField.cc)
+
+# Package headers
+set(HEADERS
+  ree.h)
+
+# Add the ree library
+add_library(ree ${SOURCES})
+
+# Include directories
+target_include_directories(ree INTERFACE
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+    $<INSTALL_INTERFACE:include>
+    )
+
+# Link this package to these libraries
+target_link_libraries(ree PUBLIC apf pcu crv)
+
+scorec_export_library(ree)
+
+bob_end_subdir()
diff --git a/ree/cmake/Dependencies.cmake b/ree/cmake/Dependencies.cmake
new file mode 100644
index 000000000..9d9456a31
--- /dev/null
+++ b/ree/cmake/Dependencies.cmake
@@ -0,0 +1,3 @@
+TRIBITS_PACKAGE_DEFINE_DEPENDENCIES(
+  LIB_REQUIRED_PACKAGES SCORECapf
+  )
diff --git a/ree/pkg_tribits.cmake b/ree/pkg_tribits.cmake
new file mode 100644
index 000000000..7819acdfa
--- /dev/null
+++ b/ree/pkg_tribits.cmake
@@ -0,0 +1,23 @@
+tribits_package(SCORECree)
+
+# THIS IS WHERE TRIBITS GETS HEADERS
+include_directories(${CMAKE_CURRENT_SOURCE_DIR})
+
+#Sources & Headers
+set(SOURCES
+  reeResidualFunctionals.cc
+  reeFluxCorrection.cc
+  reeCorrectedFlux.cc
+  reeEstimateError.cc
+  reeSizeField.cc)
+
+set(HEADERS
+  ree.h)
+
+#Library
+tribits_add_library(
+  ree
+  HEADERS ${HEADERS}
+  SOURCES ${SOURCES})
+
+tribits_package_postprocess()
diff --git a/ree/ree.h b/ree/ree.h
new file mode 100644
index 000000000..8278c5424
--- /dev/null
+++ b/ree/ree.h
@@ -0,0 +1,73 @@
+/*
+ * Copyright (C) 2011 Scientific Computation Research Center
+ *
+ * This work is open source software, licensed under the terms of the
+ * BSD license as described in the LICENSE file in the top-level directory.
+ */
+
+#ifndef REE_H
+#define REE_H
+
+
+#include "apf.h"
+#include <apfMesh.h>
+#include <PCU.h>
+#include <lionPrint.h>
+#include "apfShape.h"
+#include "apfField.h"
+#include <mthQR.h>
+#include <mth.h>
+#include <mth_def.h>
+
+namespace ree {
+
+/*
+ * Computes nodal size field
+ */
+apf::Field* getTargetEMSizeField(
+    apf::Field* ef,
+    apf::Field* error_field,
+    int n,
+    double alpha = 0.25,
+    double beta = 2.0);
+/*
+ * Takes the solution electric field and computes edge equilibrations.
+ */
+apf::Field* equilibrateResiduals(apf::Field* f);
+
+/*
+ * Takes the solution electric field and equilibrated field (of face vectors)
+ * and computes the 'correction' to the flux vectors on each face.
+ */
+apf::Field* computeFluxCorrection(apf::Field* ef, apf::Field* g);
+
+/* Takes the solution electric field and correctiion to the flux vectors on
+ * each face and computes the 'corrected' flux vectors on each face
+ */
+apf::Field* computeCorrectedFlux(apf::Field* ef, apf::Field* theta);
+
+/* Takes the solution electric field and corrected flux field and solves
+ * local element level BVPs to estimate the error.
+ * Returns a per-element scalar error field.
+ */
+apf::Field* computeErrorField(apf::Field* ef, apf::Field* correctedFlux);
+
+apf::Field* estimateError(apf::Field* f);
+
+
+void assembleCurlCurlElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
+    apf::Field* f, mth::Matrix<double>& elmat);
+
+void assembleVectorMassElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
+    apf::Field* f, mth::Matrix<double>& elmat);
+
+void assembleElementMatrix(apf::Mesh* mesh, apf::MeshEntity*e,
+    apf::Field* f, mth::Matrix<double>& elmat);
+
+void assembleDomainLFElementVector(apf::Mesh* mesh, apf::MeshEntity* e,
+    apf::Field* f, mth::Vector<double>& elvect);
+
+apf::Vector3 computeFaceOutwardNormal(apf::Mesh* m,
+    apf::MeshEntity* t, apf::MeshEntity* f, apf::Vector3 const& p);
+}
+#endif
diff --git a/ree/reeCorrectedFlux.cc b/ree/reeCorrectedFlux.cc
new file mode 100644
index 000000000..d700857c8
--- /dev/null
+++ b/ree/reeCorrectedFlux.cc
@@ -0,0 +1,299 @@
+/*
+ * Copyright (C) 2011 Scientific Computation Research Center
+ *
+ * This work is open source software, licensed under the terms of the
+ * BSD license as described in the LICENSE file in the top-level directory.
+ */
+#include <apfCavityOp.h>
+#include "apfElement.h"
+#include "crv.h"
+#include "crvShape.h"
+#include "ree.h"
+
+namespace ree {
+
+enum {VISITED};
+
+/* overall information useful during computation of corrected flux */
+struct CorrectFlux
+{
+  apf::Mesh* mesh;
+  /* mesh dimension, so far handling 3 only */
+  int dim;
+  /* polynomial order of nedelec space */
+  int order;
+  /* polynomial order of p+1 nedelec space */
+  int orderp1;
+  /* input nedelec field containing scalar dofs for electric field */
+  apf::Field* ef;
+  /* input theta field for flux correction
+   * (3 scalar dofs on each face, 1 per edge) */
+  apf::Field* theta;
+  /* tags each face once it has been visited */
+  apf::MeshTag* tag;
+  /* output per-element field containing correctedFlux vectors,
+   * which are computed using theta and electric fields */
+  apf::Field* correctedFlux;
+};
+
+static void setupCorrectFlux(
+    CorrectFlux* cf,
+    apf::Field* f,
+    apf::Field* theta,
+    apf::Field* correctedFlux)
+{
+  cf->mesh = apf::getMesh(f);
+  cf->dim = cf->mesh->getDimension();
+  cf->order = f->getShape()->getOrder();
+  cf->orderp1 = cf->order+1;
+  cf->ef = f;
+  cf->theta = theta;
+  cf->tag = cf->mesh->createIntTag("isVisited", 1);
+
+  cf->correctedFlux = correctedFlux;
+  int nc = apf::countComponents(cf->correctedFlux);
+  double zeros[nc];
+  for (int i = 0; i < nc; i++)
+    zeros[i] = 0.;
+  apf::MeshEntity* tet;
+  apf::MeshIterator* it = cf->mesh->begin(3);
+  while ((tet = cf->mesh->iterate(it))) {
+    apf::MeshElement* me = apf::createMeshElement(cf->mesh, tet);
+    int orderp1 = cf->order+1;
+    int np = apf::countIntPoints(me, 2*orderp1-1);
+
+    for (int i = 0; i < np; i++) {
+      apf::setComponents(cf->correctedFlux, tet, i, zeros);
+    }
+  }
+  cf->mesh->end(it);
+}
+
+typedef std::vector<apf::MeshEntity*> EntityVector;
+struct FaceCavity
+{
+  apf::Mesh* mesh;
+  apf::MeshEntity* entity;
+  CorrectFlux* correctflux;
+  EntityVector tets;
+};
+
+static void setupFaceCavity(FaceCavity* fc, CorrectFlux* cf)
+{
+  fc->mesh = cf->mesh;
+  fc->correctflux = cf;
+  fc->entity = 0;
+}
+
+static void startFaceCavity(FaceCavity* fc, apf::MeshEntity* f)
+{
+  fc->entity = f;
+  fc->tets.clear();
+}
+
+static void addEntityToCavity(FaceCavity* fc, apf::MeshEntity* e)
+{
+  PCU_ALWAYS_ASSERT(fc->mesh->getType(e) == apf::Mesh::TET);
+  fc->tets.push_back(e);
+}
+
+static void addEntitiesToCavity(
+    FaceCavity* fc, apf::DynamicArray<apf::MeshEntity*>& es)
+{
+  for (std::size_t i=0; i < es.getSize(); ++i)
+    addEntityToCavity(fc, es[i]);
+}
+
+static bool getInitialFaceCavity(FaceCavity* fc, apf::CavityOp* o)
+{
+  if (! o->requestLocality(&fc->entity, 1))
+    return false;
+
+  apf::DynamicArray<apf::MeshEntity*> adjacent;
+  fc->mesh->getAdjacent(fc->entity, 3, adjacent);
+  addEntitiesToCavity(fc, adjacent);
+  return true;
+}
+
+static bool buildFaceCavity(FaceCavity* fc, apf::CavityOp* o)
+{
+  if (!getInitialFaceCavity(fc, o)) return false;
+  return true;
+}
+
+
+static void computeCorrectedFlux(FaceCavity* fc)
+{
+  apf::MeshEntity* face = fc->entity;
+
+  // 1. get upward tets of the face
+  apf::Up up;
+  fc->mesh->getUp(face, up);
+  if (crv::isBoundaryEntity(fc->mesh, face))
+    PCU_ALWAYS_ASSERT(up.n == 1);
+  else
+    PCU_ALWAYS_ASSERT(up.n == 2);
+
+  apf::MeshEntity* firstTet = up.e[0];
+  apf::MeshEntity* secondTet = nullptr;
+  if (up.n == 2)
+    secondTet  = up.e[1];
+
+  // 2. get positions of the face in downward faces of upward tets
+  apf::Downward tet1_faces, tet2_faces;
+  int nf = fc->mesh->getDownward(firstTet, 2, tet1_faces);
+  if (up.n == 2)
+    fc->mesh->getDownward(secondTet, 2, tet2_faces);
+  int tet1_pos = -1;
+  int tet2_pos = -1;
+  tet1_pos = apf::findIn(tet1_faces, nf, face);
+  if (up.n == 2)
+    tet2_pos = apf::findIn(tet2_faces, nf, face);
+
+  // 3. get downward edges of the face
+  apf::Downward edges;
+  int nedges = fc->mesh->getDownward(face, 1, edges);
+
+  // 4. get theta coeffs on the face
+  double components[3];
+  apf::getComponents(fc->correctflux->theta, face, 0, components);
+  mth::Vector<double> theta_coeffs(3);
+  theta_coeffs(0) = components[0];
+  theta_coeffs(1) = components[1];
+  theta_coeffs(2) = components[2];
+
+
+  // 5. Evaluate and save corrected flux vector in an auxiliary field
+  int ftype = fc->mesh->getType(face);
+  PCU_ALWAYS_ASSERT(ftype == apf::Mesh::TRIANGLE);
+  int nfdofs = apf::countElementNodes(fc->correctflux->ef->getShape(), ftype);
+  apf::NewArray<apf::Vector3> vectorshapes(nfdofs);
+
+  apf::MeshElement* fme = apf::createMeshElement(fc->mesh, face);
+  apf::Element* fel = apf::createElement(fc->correctflux->ef, fme);
+  int int_order = 2*fc->correctflux->orderp1-1;
+  int np = apf::countIntPoints(fme, int_order);
+  int nc = apf::countComponents(fc->correctflux->correctedFlux);
+
+  apf::Vector3 p, tet1xi, tet2xi, curl1, curl2, curl,
+      fnormal1, fnormal2, tk, tk1, tk2, vshape;
+  for (int n = 0; n < np; n++) {
+
+    apf::getIntPoint(fme, int_order, n, p);
+
+    // evaluate theta vector using theta coeffs
+    apf::Vector3 theta_vector, theta_vector1, theta_vector2;
+    theta_vector.zero(); theta_vector1.zero(); theta_vector2.zero();
+    apf::NewArray<apf::Vector3> triVectorShapes (nfdofs);
+    apf::getVectorShapeValues(fel, p, triVectorShapes);
+    for (int i = 0; i < nedges; i++) {
+      apf::Vector3 v = triVectorShapes[i];
+      v = v * theta_coeffs[i];
+      theta_vector += v;
+    }
+
+    // orient face outward normals wrt tets and theta vectors
+    fnormal1 = computeFaceOutwardNormal(fc->mesh, firstTet, face, p);
+    fnormal2 = apf::Vector3(0.,0.,0.);
+    theta_vector1 = theta_vector;
+    if (up.n == 2) {
+      fnormal2 = computeFaceOutwardNormal(fc->mesh, secondTet, face, p);
+      theta_vector2 = theta_vector * -1.;
+    }
+
+    curl.zero();
+    // compute curl1
+    tet1xi = apf::boundaryToElementXi(fc->mesh, face, firstTet, p);
+    apf::MeshElement* me1 = apf::createMeshElement(fc->mesh, firstTet);
+    apf::Element* el1 = apf::createElement(fc->correctflux->ef, me1);
+    apf::getCurl(el1, tet1xi, curl1);
+    apf::Vector3 temp1 = apf::cross(fnormal1, curl1);
+    curl += temp1;
+    apf::destroyElement(el1);
+    apf::destroyMeshElement(me1);
+
+    // compute curl2
+    if (up.n == 2) {
+      tet2xi = apf::boundaryToElementXi(fc->mesh, face, secondTet, p);
+      apf::MeshElement* me2 = apf::createMeshElement(fc->mesh, secondTet);
+      apf::Element* el2 = apf::createElement(fc->correctflux->ef, me2);
+      apf::getCurl(el2, tet2xi, curl2);
+      apf::Vector3 temp2 = apf::cross(fnormal2, curl2);
+      curl += (temp2 * -1.);
+      curl = curl * 1./2.;
+      apf::destroyElement(el2);
+      apf::destroyMeshElement(me2);
+    }
+
+    tk = curl;
+    tk1 = tk;
+    apf::Vector3 theta_plus_tk1 = theta_vector1 + tk1;
+
+    // get and set components in the auxiliary field
+    double comp1[nc];
+    apf::getComponents(fc->correctflux->correctedFlux, firstTet, n, comp1);
+    int id1 = tet1_pos * 3;
+    comp1[id1] = theta_plus_tk1[0];
+    comp1[id1+1] = theta_plus_tk1[1];
+    comp1[id1+2] = theta_plus_tk1[2];
+    apf::setComponents(fc->correctflux->correctedFlux, firstTet, n, comp1);
+
+    if (up.n == 2) {
+      tk2 = tk * -1.;
+      apf::Vector3 theta_plus_tk2 = theta_vector2 + tk2;
+      double comp2[nc];
+      apf::getComponents(fc->correctflux->correctedFlux, secondTet, n, comp2);
+      int id2 = tet2_pos * 3;
+      comp2[id2] = theta_plus_tk2[0];
+      comp2[id2+1] = theta_plus_tk2[1];
+      comp2[id2+2] = theta_plus_tk2[2];
+      apf::setComponents(fc->correctflux->correctedFlux, secondTet, n, comp2);
+    }
+  }
+}
+
+class FaceCavityOp : public apf::CavityOp
+{
+public:
+  FaceCavityOp(CorrectFlux* cf):
+    apf::CavityOp(cf->mesh)
+  {
+    setupFaceCavity(&face_cavity, cf);
+  }
+  virtual Outcome setEntity(apf::MeshEntity* e)
+  {
+    if (face_cavity.mesh->hasTag(e, face_cavity.correctflux->tag))
+      return SKIP;
+    startFaceCavity(&face_cavity, e);
+    if ( ! buildFaceCavity(&face_cavity, this)) {
+      return REQUEST;
+    }
+    return OK;
+  }
+  virtual void apply()
+  {
+    computeCorrectedFlux(&face_cavity);
+    int n = VISITED;
+    face_cavity.mesh->setIntTag(
+        face_cavity.entity, face_cavity.correctflux->tag, &n);
+  }
+  FaceCavity face_cavity;
+};
+
+apf::Field* computeCorrectedFlux(apf::Field* ef, apf::Field* theta)
+{
+  int dim = apf::getMesh(ef)->getDimension();
+  int order = ef->getShape()->getOrder() + 1; // local BVPs require p+1
+  int int_order = 2*order-1;
+  int nc = 4*3; // 1 flux vector per face
+  apf::Field* correctedFlux = createPackedField(
+      apf::getMesh(ef), "correctedFlux", nc, apf::getIPShape(dim, int_order));
+
+  CorrectFlux correctflux;
+  setupCorrectFlux(&correctflux, ef, theta, correctedFlux);
+  FaceCavityOp op (&correctflux);
+  op.applyToDimension(2);
+  return correctedFlux;
+
+}
+}
diff --git a/ree/reeEstimateError.cc b/ree/reeEstimateError.cc
new file mode 100644
index 000000000..f137e4f1a
--- /dev/null
+++ b/ree/reeEstimateError.cc
@@ -0,0 +1,458 @@
+/*
+ * Copyright (C) 2011 Scientific Computation Research Center
+ *
+ * This work is open source software, licensed under the terms of the
+ * BSD license as described in the LICENSE file in the top-level directory.
+ */
+
+#include <apfCavityOp.h>
+#include "apfElement.h"
+#include "crv.h"
+#include "crvShape.h"
+#include "ree.h"
+
+namespace ree {
+
+static void computeResidualBLF(apf::Mesh* mesh, apf::MeshEntity* e,
+  apf::Field* f, apf::Field* fp1, mth::Vector<double>& blf)
+{
+  apf::FieldShape* fp1s = fp1->getShape();
+  int type = mesh->getType(e);
+  PCU_ALWAYS_ASSERT(type == apf::Mesh::TET);
+  int nd = apf::countElementNodes(fp1s, type);
+  int dim = apf::getDimension(mesh, e);
+  double w;
+
+  apf::NewArray<apf::Vector3> curlshape(nd);
+  apf::NewArray<apf::Vector3> vectorshape(nd);
+  mth::Matrix<double> phys_curlshape(nd, dim);
+  mth::Matrix<double> vectorShape(nd, dim);
+  mth::Vector<double> curlcurl_vec (nd);
+  mth::Vector<double> mass_vec (nd);
+  blf.resize(nd);
+
+  apf::MeshElement* me = apf::createMeshElement(mesh, e);
+  apf::Element* fp1el = apf::createElement(fp1, me);
+  apf::Element* fel = apf::createElement(f, me);
+  int int_order = 2 * fp1s->getOrder();
+  int np = apf::countIntPoints(me, int_order);
+
+  // 1. Compute Curl Curl Integration
+  curlcurl_vec.zero();
+  apf::Vector3 p, curl;
+  for (int i = 0; i < np; i++) {
+    apf::getIntPoint(me, int_order, i, p);
+    double weight = apf::getIntWeight(me, int_order, i);
+    apf::Matrix3x3 J;
+    apf::getJacobian(me, p, J);
+    w = weight;
+
+    apf::getCurl(fel, p, curl);
+
+    // get curlshape values
+    fp1el->getShape()->getLocalVectorCurls(mesh, e, p, curlshape);
+    phys_curlshape.zero();
+    for (int i = 0; i < nd; i++)
+      for (int j = 0; j < dim; j++)
+        for (int k = 0; k < dim; k++)
+          phys_curlshape(i,j) += curlshape[i][k] * J[k][j];
+
+    // multiply
+    mth::Vector<double> V (nd);
+    V.zero();
+    mth::Vector<double> c (dim);
+    c(0) = curl[0]; c(1) = curl[1]; c(2) = curl[2];
+    mth::multiply(phys_curlshape, c, V);
+    V *= w;
+
+    curlcurl_vec += V;
+  }
+
+  // 2. Compute Vector Mass Integration
+  int_order = 2 * fp1s->getOrder();
+  np = apf::countIntPoints(me, int_order);
+
+  mass_vec.zero();
+  apf::Vector3 vvalue;
+  for (int i = 0; i < np; i++) {
+    apf::getIntPoint(me, int_order, i, p);
+    double weight = apf::getIntWeight(me, int_order, i);
+    apf::Matrix3x3 J;
+    apf::getJacobian(me, p, J);
+    double jdet = apf::getJacobianDeterminant(J, dim);
+
+    w = weight * jdet;
+
+    apf::getVector(fel, p, vvalue);
+
+    apf::getVectorShapeValues(fp1el, p, vectorshape);
+    vectorShape.zero();
+    for (int i = 0; i < nd; i++)
+      for (int j = 0; j < dim; j++)
+        vectorShape(i,j) = vectorshape[i][j];
+
+
+    mth::Vector<double> V (nd);
+    V.zero();
+    mth::Vector<double> v (dim);
+    v(0) = vvalue[0]; v(1) = vvalue[1]; v(2) = vvalue[2];
+    mth::multiply(vectorShape, v, V);
+    V *= w;
+
+    mass_vec += V;
+  }
+
+  // 3. get result
+  blf.zero();
+  blf += curlcurl_vec;
+  blf += mass_vec;
+
+  apf::destroyMeshElement(me);
+  apf::destroyElement(fp1el);
+  apf::destroyElement(fel);
+}
+
+static void computeLambdaVector(
+    apf::Mesh* mesh,
+    apf::MeshEntity* e,
+    apf::Field* f,
+    apf::Field* fp1,
+    apf::Field* flux_field,
+    mth::Vector<double>& lambda)
+{
+  apf::FieldShape* fp1s = fp1->getShape();
+  int order = fp1s->getOrder();
+  int etype = mesh->getType(e);
+  PCU_ALWAYS_ASSERT(etype == apf::Mesh::TET);
+  int nedofs = apf::countElementNodes(fp1s, etype);
+  lambda.resize(nedofs);
+
+  int nc = apf::countComponents(flux_field);
+
+  // get the downward faces of the element
+  apf::Downward faces;
+  int nf = mesh->getDownward(e, 2, faces);
+
+  lambda.zero();
+  // assemble lambda vector LOOP OVER DOWNWARD FACES
+  for (int ii = 0; ii < nf; ii++) {
+    apf::MeshEntity* face = faces[ii];
+
+    int ftype = mesh->getType(face);
+    PCU_ALWAYS_ASSERT(ftype == apf::Mesh::TRIANGLE);
+    int nfdofs = apf::countElementNodes(f->getShape(), ftype);
+    apf::NewArray<apf::Vector3> vectorshape(nfdofs);
+
+    apf::MeshElement* fme = apf::createMeshElement(mesh, face);
+    int np = apf::countIntPoints(fme, 2*order-1);
+
+    // 4. Compute integral on the face
+    apf::Vector3 p;
+    for (int n = 0; n < np; n++) {
+
+      apf::getIntPoint(fme, 2*order-1, n, p);
+      double weight = apf::getIntWeight(fme, 2*order-1, n);
+      apf::Matrix3x3 fJ;
+      apf::getJacobian(fme, p, fJ);
+      double jdet = apf::getJacobianDeterminant(
+          fJ, apf::getDimension(mesh, face));
+
+      // obtain corrected flux vector
+      double comp[nc];
+      apf::getComponents(flux_field, e, n, comp);
+      apf::Vector3 theta_plus_tk;
+      int index = ii*3;
+      theta_plus_tk[0] = comp[index];
+      theta_plus_tk[1] = comp[index+1];
+      theta_plus_tk[2] = comp[index+2];
+
+      // compute p+1 order 3D vector shapes
+      apf::NewArray<apf::Vector3> tetVectorShapes (nedofs);
+      apf::MeshElement* me = apf::createMeshElement(mesh, e);
+      apf::Element* el = apf::createElement(fp1, me);
+      apf::Vector3 tetxi = apf::boundaryToElementXi(mesh, face, e, p);
+      apf::getVectorShapeValues(el, tetxi, tetVectorShapes);
+
+      apf::destroyElement(el);
+      apf::destroyMeshElement(me);
+
+      // compute integral
+      double w = weight * jdet;
+      theta_plus_tk = theta_plus_tk * w;
+
+      // matrix vector multiplication
+      for (int i = 0; i < nedofs; i++)
+        lambda(i) += theta_plus_tk * tetVectorShapes[i];
+
+    } // end integral loop
+    apf::destroyMeshElement(fme);
+  } // end face loop
+}
+
+static void getEssentialElementNDDofs(apf::Mesh* mesh, apf::MeshEntity* e,
+    apf::Field* f, apf::NewArray<int>& ess_dofs, apf::NewArray<int>& uness_dofs)
+{
+  apf::FieldShape* fs = f->getShape();
+  int etype = mesh->getType(e);
+  PCU_ALWAYS_ASSERT(etype == apf::Mesh::TET);
+  int nedofs = apf::countElementNodes(fs, etype);
+
+  apf::NewArray<int> marker_list (nedofs);
+  for (int i = 0; i < nedofs; i++)
+    marker_list[i] = 0;
+
+  // populate marker list by iterating over downward edges and faces
+  apf::Downward edges;
+  int nedges = mesh->getDownward(e, 1, edges);
+  PCU_ALWAYS_ASSERT(nedges == 6);
+  for (int i = 0; i < nedges; i++) {
+    int nodesOnEdge = fs->countNodesOn(mesh->getType(edges[i]));
+    if ( crv::isBoundaryEntity(mesh, edges[i]) ) {
+      for (int n = 0; n < nodesOnEdge; n++) {
+        marker_list[(i*nodesOnEdge)+n] = -1;
+      }
+    }
+  }
+  int nodesOnEdges = fs->countNodesOn(apf::Mesh::EDGE) * nedges;
+
+  apf::Downward faces;
+  int nfaces = mesh->getDownward(e, 2, faces);
+  PCU_ALWAYS_ASSERT(nfaces == 4);
+  for (int i = 0; i < nfaces; i++) {
+    int nodesOnFace = fs->countNodesOn(mesh->getType(faces[i]));
+    if ( crv::isBoundaryEntity(mesh, faces[i]) ) {
+      for (int n = 0; n < nodesOnFace; n++) {
+        marker_list[(nodesOnEdges + i*nodesOnFace)+n] = -1;
+      }
+    }
+  }
+
+  int num_marked = 0;
+  for (int i = 0; i < nedofs; i++) {
+    if (marker_list[i])
+      num_marked++;
+  }
+
+  // use marker list to get ess_dofs list
+  ess_dofs.allocated() ? ess_dofs.resize(num_marked)
+    : ess_dofs.allocate(num_marked);
+  uness_dofs.allocated() ? uness_dofs.resize(nedofs-num_marked)
+    : uness_dofs.allocate(nedofs-num_marked);
+  int ess_dof_counter = 0;
+  int uness_dof_counter = 0;
+  for (int i = 0; i < nedofs; i++) {
+    if(marker_list[i])
+      ess_dofs[ess_dof_counter++] = i;
+    else
+      uness_dofs[uness_dof_counter++] = i;
+  }
+}
+
+/**
+ * Inputs: Matrix A, Vector X, Vector B, essential dofs, other dofs.
+ * Output: reduced matrix A, reduced rhs B.
+ */
+static void eliminateDBCs(
+    mth::Matrix<double> const &A,
+    mth::Vector<double> const &X,
+    mth::Vector<double> const &B,
+    apf::NewArray<int>  const &ess_dofs,
+    apf::NewArray<int>  const &uness_dofs,
+    mth::Matrix<double> &Anew,
+    mth::Vector<double> &Bnew)
+{
+  int num_ess_dofs = ess_dofs.size();
+  int num_uness_dofs = uness_dofs.size();
+
+  // 1. Remove rows of A corresponding to
+	// ess_dofs by copying into Anew
+  Anew.resize(num_uness_dofs, num_uness_dofs);
+	for(int rr = 0; rr < num_uness_dofs; rr++) {
+    int i = uness_dofs[rr];
+    for(int cc = 0; cc < num_uness_dofs; cc++) {
+      int j = uness_dofs[cc];
+      Anew(rr,cc) = A(i,j);
+    }
+  }
+
+	// 2. Assemble new B
+	Bnew.resize(num_uness_dofs);
+  for(int i = 0; i < num_uness_dofs; i++) {
+    Bnew(i) = B(uness_dofs[i]);
+  }
+
+  if (num_ess_dofs > 0) {
+	  // 3. Subtract from Bnew: (Bnew -= Ae*Xe)
+    mth::Matrix<double> Ae(num_uness_dofs, num_ess_dofs);
+    for(int rr = 0; rr < num_uness_dofs; rr++) {
+      int i = uness_dofs[rr];
+      for(int cc = 0; cc < num_ess_dofs; cc++) {
+        int j = ess_dofs[cc];
+        Ae(rr,cc) = A(i,j);
+      }
+    }
+
+    mth::Vector<double> Xe(num_ess_dofs);
+    for(int i = 0; i < num_ess_dofs; i++) {
+      Xe(i) = X(ess_dofs[i]);
+    }
+
+    mth::Vector<double> temp;
+    mth::multiply(Ae, Xe, temp);
+    Bnew -= temp;
+  }
+}
+
+static double computeL2Error(apf::Mesh* mesh, apf::MeshEntity* e,
+	apf::Field* f, mth::Vector<double> const error_dofs)
+{
+	double error = 0.0;
+
+  apf::FieldShape* fs = f->getShape();
+  int type = mesh->getType(e);
+	PCU_ALWAYS_ASSERT(type == apf::Mesh::TET);
+  int nd = apf::countElementNodes(fs, type);
+  int dim = apf::getDimension(mesh, e);
+  double w;
+
+  apf::MeshElement* me = apf::createMeshElement(mesh, e);
+  apf::Element* el = apf::createElement(f, me);
+  int order = 2 * fs->getOrder();
+  int np = apf::countIntPoints(me, order);
+
+  apf::NewArray<apf::Vector3> vectorshape(nd);
+
+  apf::Vector3 p;
+  for (int i = 0; i < np; i++) {
+    apf::getIntPoint(me, order, i, p);
+    double weight = apf::getIntWeight(me, order, i);
+    apf::Matrix3x3 J;
+    apf::getJacobian(me, p, J);
+    double jdet = apf::getJacobianDeterminant(J, dim);
+    w = weight * jdet;
+
+    apf::getVectorShapeValues(el, p, vectorshape);
+    mth::Matrix<double> vectorShape (nd, dim);
+    for (int j = 0; j < nd; j++)
+      for (int k = 0; k < dim; k++)
+        vectorShape(j,k) = vectorshape[j][k];
+
+    mth::Matrix<double> vectorShapeT (dim, nd);
+    mth::transpose(vectorShape, vectorShapeT);
+
+		mth::Vector<double> err_func;
+		mth::multiply(vectorShapeT, error_dofs, err_func);
+
+		error += w * (err_func * err_func);
+	}
+	if (error < 0.0)
+		error = -error;
+
+	return sqrt(error);
+}
+
+apf::Field* computeErrorField(apf::Field* ef, apf::Field* correctedFlux)
+{
+
+	// 1. Create per-element SCALAR error field
+  apf::Field* error_field = apf::createIPField(
+		apf::getMesh(ef), "residual_error_field", apf::SCALAR, 1);
+
+  // 2. Create p+1 order tet ND field
+  int order = ef->getShape()->getOrder();
+  int orderp1 = order+1;
+  apf::Field* efp1 = apf::createField(apf::getMesh(ef),
+	 "orderp1_nedelec_field", apf::SCALAR, apf::getNedelec(orderp1));
+  apf::zeroField(efp1);
+
+  // 2. iterate over all elements of the mesh
+  apf::MeshEntity* el;
+  apf::MeshIterator* it = apf::getMesh(ef)->begin(3);
+  while ((el = apf::getMesh(ef)->iterate(it))) {
+
+    // 2(a). Assemble LHS element matrix
+    mth::Matrix<double> A;
+    assembleElementMatrix( apf::getMesh(ef), el, efp1, A);
+
+    // 2(b). Compute Bilinear Form Vector
+    mth::Vector<double> blf;
+    computeResidualBLF(apf::getMesh(efp1), el, ef, efp1, blf);
+
+    // 2(c). Compute Linear Form Vector
+    mth::Vector<double> lf;
+    assembleDomainLFElementVector(apf::getMesh(efp1), el, efp1, lf);
+
+    // 2(d). Compute Lambda Vector
+    mth::Vector<double> lambda;
+    computeLambdaVector(
+        apf::getMesh(ef), el, ef, efp1, correctedFlux, lambda);
+
+    // 2(e). Assemble RHS element vector = blf - lf - lambda
+    mth::Vector<double> B(blf.size());
+    B.zero();
+    B += blf; B -= lf; B -= lambda;
+
+    // 2(f). Get List of Essential Dofs
+    apf::NewArray<int> ess_dofs, uness_dofs;
+    getEssentialElementNDDofs(
+        apf::getMesh(efp1), el, efp1, ess_dofs, uness_dofs);
+
+    // 2(g). eliminate Dirichlet (Essential) Boundary Conditions
+		mth::Vector<double> X, Bnew;
+    mth::Matrix<double> Anew;
+		X.resize(B.size());
+		X.zero(); // initialize X with exact DBC (e = 0.0)
+		eliminateDBCs(A, X, B, ess_dofs, uness_dofs, Anew, Bnew);
+
+		// 2(h). Solve the reduced system
+	  mth::Matrix<double> Q, R;
+		mth::decomposeQR(Anew, Q, R);
+		mth::Vector<double> Xnew;
+		mth::solveFromQR(Q, R, Bnew, Xnew);
+
+		// 2(i). Recover the solution
+		mth::Vector<double> error_dofs(B.size());
+    for(unsigned int i = 0; i < ess_dofs.size(); i++) {
+    	int index = ess_dofs[i];
+      error_dofs(index) = X(index);
+    }
+    for(unsigned int i = 0; i < uness_dofs.size(); i++) {
+      int index = uness_dofs[i];
+      error_dofs(index) = Xnew(i);
+    }
+
+		// 2(j). Compute L2 Norm Error
+		double l2_error = computeL2Error(apf::getMesh(ef), el, efp1, error_dofs);
+		apf::setScalar(error_field, el, 0, l2_error);
+  }
+  apf::getMesh(ef)->end(it);
+	apf::destroyField(efp1);
+
+	return error_field;
+}
+
+apf::Field* estimateError(apf::Field* f)
+{
+  double t0 = PCU_Time();
+  apf::Field* g = ree::equilibrateResiduals(f);
+  lion_eprint(1,"1/4: residuals equilibrated \n");
+  apf::Field* theta = ree::computeFluxCorrection(f, g);
+  lion_eprint(1,"2/4: flux corrections computed \n");
+  apf::destroyField(g);
+  PCU_Barrier();
+
+  apf::Field* correctedFlux = ree::computeCorrectedFlux(f, theta);
+  lion_eprint(1,"3/4: corrected flux field computed\n");
+  apf::destroyField(theta);
+
+  apf::Field* error_field = ree::computeErrorField(f, correctedFlux);
+  lion_eprint(1,"4/4: error computed \n");
+  apf::destroyField(correctedFlux);
+
+  double t1 = PCU_Time();
+  if (!PCU_Comm_Self())
+    lion_eprint(1,"REE: Error estimated in %f seconds\n",t1-t0);
+
+  return error_field;
+}
+}
diff --git a/ree/reeFluxCorrection.cc b/ree/reeFluxCorrection.cc
new file mode 100644
index 000000000..a369cb5ba
--- /dev/null
+++ b/ree/reeFluxCorrection.cc
@@ -0,0 +1,73 @@
+/*
+ * Copyright (C) 2011 Scientific Computation Research Center
+ *
+ * This work is open source software, licensed under the terms of the
+ * BSD license as described in the LICENSE file in the top-level directory.
+ */
+#include "crv.h"
+#include "crvShape.h"
+#include "apfElement.h"
+#include "ree.h"
+
+namespace ree {
+
+struct QRDecomp {
+  mth::Matrix<double> Q;
+  mth::Matrix<double> R;
+};
+
+/*
+ * This function takes the NedelecField and g parameters field
+ * and returns the THETA field. It computes the 3 scalar parameters
+ * per face for the flux correction vectors.
+ */
+apf::Field* computeFluxCorrection(apf::Field* ef, apf::Field* g)
+{
+  apf::Mesh* mesh = apf::getMesh(ef);
+  apf::Field* THETA_Field =  createPackedField(
+      mesh, "theta_field", 3, apf::getConstant(2));
+
+  apf::MeshEntity* face;
+  apf::MeshIterator* it = mesh->begin(2);
+  while ((face = mesh->iterate(it))) {
+
+    // 1. assemble RHS vector
+    double components[3];
+    apf::getComponents(g, face, 0, components);
+
+    mth::Vector<double> rhs(3);
+    apf::Downward edges;
+    int ne = mesh->getDownward(face, 1, edges);
+    for (int i = 0; i < ne; i++) {
+      int which, rotate; bool flip;
+      apf::getAlignment(mesh, face, edges[i], which, flip, rotate);
+      if (flip)
+        rhs(i) = -1. * components[i];
+      else
+        rhs(i) = components[i];
+    }
+
+    // 2. assemble LHS face mass matrix
+    mth::Matrix<double> M;
+    assembleVectorMassElementMatrix(mesh, face, ef, M);
+
+    // 3. solve the system
+    QRDecomp qr;
+    mth::decomposeQR(M, qr.Q, qr.R);
+    mth::Vector<double> theta;
+    mth::solveFromQR(qr.Q, qr.R, rhs, theta);
+
+    // set solution vector on face field
+    components[0] = theta(0);
+    components[1] = theta(1);
+    components[2] = theta(2);
+    apf::setComponents(THETA_Field, face, 0, components);
+  }
+  mesh->end(it);
+
+  return THETA_Field;
+}
+
+}
+
+
diff --git a/ree/reeResidualFunctionals.cc b/ree/reeResidualFunctionals.cc
new file mode 100644
index 000000000..08685b384
--- /dev/null
+++ b/ree/reeResidualFunctionals.cc
@@ -0,0 +1,918 @@
+/*
+ * Copyright (C) 2011 Scientific Computation Research Center
+ *
+ * This work is open source software, licensed under the terms of the
+ * BSD license as described in the LICENSE file in the top-level directory.
+ */
+#include <algorithm>
+#include <apfCavityOp.h>
+#include "apfElement.h"
+#include "crv.h"
+#include "crvShape.h"
+#include "ree.h"
+
+namespace ree {
+
+enum {VISITED};
+
+/* overall information useful during equilibration */
+struct Equilibration {
+  apf::Mesh* mesh;
+  /* mesh dimension, so far handling 3 only */
+  int dim;
+  /* polynomial order of Nedelec space */
+  int order;
+  /* input scalar field containing Nedelec dofs for solution electric field */
+  apf::Field* ef;
+  /* tags each edge once it has been visited during the equilibration process */
+  apf::MeshTag* tag;
+  /* output field containing correction values.
+   * currently 3 scalar values are stored on each face
+   * in order corresponding to downward edges of the face */
+  apf::Field* g;
+};
+
+static void setupEquilibration(Equilibration* eq, apf::Field* f, apf::Field* g)
+{
+  eq->mesh = apf::getMesh(f);
+  eq->tag = eq->mesh->createIntTag("isVisited", 1);
+  eq->dim = eq->mesh->getDimension();
+  eq->ef = f;
+  eq->order = f->getShape()->getOrder();
+  eq->g = g;
+
+  double zeros[3] = {0., 0., 0.};
+  apf::MeshEntity* face;
+  apf::MeshIterator* it = eq->mesh->begin(2);
+  while ((face = eq->mesh->iterate(it))) {
+    apf::setComponents(eq->g, face, 0, zeros);
+  }
+  eq->mesh->end(it);
+}
+
+struct QRDecomp {
+  mth::Matrix<double> Q;
+  mth::Matrix<double> R;
+};
+
+typedef std::vector<apf::MeshEntity*> EntityVector;
+
+struct EdgePatch {
+  apf::Mesh* mesh;
+  Equilibration* equilibration;
+  /* the edge entity around which the patch
+     is centered. a patch collects entities
+     (faces & tets) around this edge entity */
+  apf::MeshEntity* entity;
+  bool isOnBdry;
+  EntityVector tets;
+  EntityVector faces;
+  mth::Matrix<double> A;
+  mth::Matrix<double> At;
+  mth::Matrix<double> T; // T = A*At + 1
+  mth::Vector<double> b;
+  mth::Vector<double> x;
+  QRDecomp qr;
+};
+
+static void setupEdgePatch(EdgePatch* ep, Equilibration* eq)
+{
+  ep->mesh = eq->mesh;
+  ep->equilibration = eq;
+  ep->entity = 0;
+}
+
+static void startEdgePatch(EdgePatch* ep, apf::MeshEntity* e)
+{
+  ep->tets.clear();
+  ep->faces.clear();
+  ep->entity = e;
+  ep->isOnBdry = crv::isBoundaryEntity(ep->mesh, ep->entity);
+}
+
+static void addEntityToPatch(EdgePatch* ep, apf::MeshEntity* e)
+{
+  if(ep->mesh->getType(e) == apf::Mesh::TRIANGLE)
+    ep->faces.push_back(e);
+  if(ep->mesh->getType(e) == apf::Mesh::TET)
+    ep->tets.push_back(e);
+}
+
+static void addEntitiesToPatch(
+    EdgePatch* ep, apf::DynamicArray<apf::MeshEntity*>& es)
+{
+  for (std::size_t i=0; i < es.getSize(); ++i)
+    addEntityToPatch(ep, es[i]);
+}
+
+static bool getInitialEdgePatch(EdgePatch* ep, apf::CavityOp* o)
+{
+  if ( ! o->requestLocality(&ep->entity,1))
+    return false;
+  apf::DynamicArray<apf::MeshEntity*> adjacent;
+  ep->mesh->getAdjacent(ep->entity, 3, adjacent);
+  addEntitiesToPatch(ep, adjacent);
+
+  ep->mesh->getAdjacent(ep->entity, 2, adjacent);
+  addEntitiesToPatch(ep, adjacent);
+
+  return true;
+}
+
+static bool buildEdgePatch(EdgePatch* ep, apf::CavityOp* o)
+{
+  if (!getInitialEdgePatch(ep, o)) return false;
+  return true;
+}
+
+/*
+ * Reference: Leszek Demkowicz, Computing with hp-adaptive
+ * finite elements, vol2, equation (11.118, 11.119, 11.122).
+ */
+static void assembleEdgePatchLHS(EdgePatch* ep)
+{
+  int ne = ep->tets.size();
+  int nf = ep->faces.size();
+  if( crv::isBoundaryEntity(ep->mesh, ep->entity) ) {
+    ep->T.resize(ne+nf, ne+nf);
+    ep->T.zero();
+    for (int i = 0; i < nf; i++)
+      ep->T(i,i) = 2.;
+    for (int i = 0; i < ne-1; i++) {
+      ep->T(i+nf,i) = 1.; ep->T(i+nf,i+1) = -1.;
+      ep->T(i,i+nf) = 1.; ep->T(i+1,i+nf) = -1.;
+    }
+    ep->T(ne+nf-1, ne-1) = 1.; ep->T(ne+nf-1, ne) = 1.;
+    ep->T(ne-1, ne+nf-1) = 1.; ep->T(ne, ne+nf-1) = 1.;
+  }
+  else if( ! crv::isBoundaryEntity(ep->mesh, ep->entity) ) {
+    ep->A.resize(ne, nf);
+    ep->A.zero();
+    for (int i = 0; i < ne-1; i++) {
+      ep->A(i,i) = 1.;  ep->A(i,i+1) = -1.;
+    }
+    ep->A(ne-1,0) = -1.; ep->A(ne-1,ne-1) = 1.;
+    // A is singular so do (A*At) + 1.0
+    // to pick a particular solution
+    ep->At.resize(nf,ne);
+    mth::transpose(ep->A, ep->At);
+    mth::multiply(ep->A, ep->At, ep->T);
+
+    for (int i = 0; i < ne; i++)
+      for (int j = 0; j < ne; j++)
+        ep->T(i,j) += 1.;
+  }
+  mth::decomposeQR(ep->T, ep->qr.Q, ep->qr.R);
+}
+
+/*
+ * Performs Curl Curl integration using curl vector Nedelec shapes
+ */
+void assembleCurlCurlElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
+    apf::Field* f, mth::Matrix<double>& elmat)
+{
+  apf::FieldShape* fs = f->getShape();
+  int type = mesh->getType(e);
+  PCU_ALWAYS_ASSERT(type == apf::Mesh::TET);
+  int nd = apf::countElementNodes(fs, type);
+  int dim = apf::getDimension(mesh, e);
+  int dimc = (dim == 3) ? 3 : 1;
+  double w;
+
+  apf::NewArray<apf::Vector3> curlshape(nd);
+  mth::Matrix<double> phys_curlshape(nd, dimc);
+  elmat.resize(nd,nd);
+
+  apf::MeshElement* me = apf::createMeshElement(mesh, e);
+  apf::Element* el = apf::createElement(f, me);
+  int int_order = 2 * fs->getOrder() - 2;
+  int np = apf::countIntPoints(me, int_order);
+
+  elmat.zero();
+  apf::Vector3 p;
+  for (int i = 0; i < np; i++) {
+    apf::getIntPoint(me, int_order, i, p);
+    double weight = apf::getIntWeight(me, int_order, i);
+    apf::Matrix3x3 J;
+    apf::getJacobian(me, p, J);
+    double jdet = apf::getJacobianDeterminant(J, dim);
+    w = weight / jdet;
+
+    if (dim == 3) {
+      el->getShape()->getLocalVectorCurls(mesh, e, p, curlshape);
+      phys_curlshape.zero();
+      for (int j = 0; j < nd; j++)
+        for (int k = 0; k < dim; k++)
+          for (int l = 0; l < dim; l++)
+            phys_curlshape(j,k) += curlshape[j][l] * J[l][k];
+    }
+    mth::Matrix<double> phys_curlshapeT;
+    mth::transpose(phys_curlshape, phys_curlshapeT);
+
+    mth::Matrix<double> M (nd, nd);
+    M.zero();
+    mth::multiply(phys_curlshape, phys_curlshapeT, M);
+    M *= w;
+    elmat += M;
+  }
+  apf::destroyElement(el);
+  apf::destroyMeshElement(me);
+}
+
+/*
+ * Performs Vector Vector Mass integration using vector Nedelec shapes
+ */
+void assembleVectorMassElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
+    apf::Field* f, mth::Matrix<double>& elmat)
+{
+  apf::FieldShape* fs = f->getShape();
+  int type = mesh->getType(e);
+  PCU_ALWAYS_ASSERT(type == apf::Mesh::TET || type == apf::Mesh::TRIANGLE);
+  int nd = apf::countElementNodes(fs, type);
+  int dim = apf::getDimension(mesh, e);
+  int sdim = mesh->getDimension();
+  double w;
+
+  apf::NewArray<apf::Vector3> vectorshapes(nd);
+  elmat.resize(nd,nd);
+
+  apf::MeshElement* me = apf::createMeshElement(mesh, e);
+  apf::Element* el = apf::createElement(f, me);
+  int int_order = 2 * fs->getOrder();
+  int np = apf::countIntPoints(me, int_order);
+
+  elmat.zero();
+  apf::Vector3 p;
+  for (int i = 0; i < np; i++) {
+    apf::getIntPoint(me, int_order, i, p);
+    double weight = apf::getIntWeight(me, int_order, i);
+    apf::Matrix3x3 J;
+    apf::getJacobian(me, p, J);
+    double jdet = apf::getJacobianDeterminant(J, dim);
+    w = weight * jdet;
+
+    apf::getVectorShapeValues(el, p, vectorshapes);
+    mth::Matrix<double> vectorShapes (nd, sdim);
+    for (int j = 0; j < nd; j++)
+      for (int k = 0; k < sdim; k++)
+        vectorShapes(j,k) = vectorshapes[j][k];
+
+    mth::Matrix<double> vectorShapesT (sdim, nd);
+    mth::transpose(vectorShapes, vectorShapesT);
+
+    mth::Matrix<double> M (nd,nd);
+    M.zero();
+    mth::multiply(vectorShapes, vectorShapesT, M);
+    M *= w;
+    elmat += M;
+  }
+
+  apf::destroyElement(el);
+  apf::destroyMeshElement(me);
+}
+
+void assembleElementMatrix(apf::Mesh* mesh, apf::MeshEntity*e,
+    apf::Field* f, mth::Matrix<double>& elmat)
+{
+  mth::Matrix<double> curl_elmat, mass_elmat;
+  assembleCurlCurlElementMatrix(mesh, e, f, curl_elmat);
+  assembleVectorMassElementMatrix(mesh, e, f, mass_elmat);
+
+  elmat.resize(curl_elmat.rows(), curl_elmat.cols());
+  elmat.zero();
+  elmat += curl_elmat;
+  elmat += mass_elmat;
+}
+
+/*
+ * computes local bilinear form integral restricted to
+ * an edge of a tet element
+ */
+static double getLocalEdgeBLF(EdgePatch* ep, apf::MeshEntity* tet)
+{
+  PCU_ALWAYS_ASSERT(ep->mesh->getType(tet) == apf::Mesh::TET);
+  // findIn edge in downward edges of tet
+  apf::Downward e;
+  int ne = ep->mesh->getDownward(tet, 1, e);
+  int ei = apf::findIn(e, ne, ep->entity);
+  // get Element Dofs
+  apf::MeshElement* me = apf::createMeshElement(ep->mesh, tet);
+  apf::Element* el = apf::createElement(ep->equilibration->ef, me);
+  int type = ep->mesh->getType(tet);
+  int nd = apf::countElementNodes(el->getFieldShape(), type);
+  apf::NewArray<double> d (nd);
+  el->getElementDofs(d);
+  mth::Vector<double> dofs (nd);
+  for (int i = 0; i < nd; i++)
+    dofs(i) = d[i];
+  // assemble curl curl element matrix
+  mth::Matrix<double> curl_elmat;
+  assembleCurlCurlElementMatrix(ep->mesh, tet,
+      ep->equilibration->ef, curl_elmat);
+  // assemble vector mass element matrix
+  mth::Matrix<double> mass_elmat;
+  assembleVectorMassElementMatrix(ep->mesh, tet,
+      ep->equilibration->ef, mass_elmat);
+  // add element matrices
+  mth::Matrix<double> elmat(nd, nd);
+  elmat.zero();
+  elmat += curl_elmat;
+  elmat += mass_elmat;
+  // multiply element matrix with element dofs
+  mth::Vector<double> blf_integrals (nd);
+  mth::multiply(elmat, dofs, blf_integrals);
+
+  apf::destroyElement(el);
+  apf::destroyMeshElement(me);
+
+  // pick edge index from the resulting vector
+  // negation of negative ND dofs
+  int which, rotate; bool flip;
+  apf::getAlignment(ep->mesh, tet, ep->entity, which, flip, rotate);
+  if (flip)
+    blf_integrals(ei) = -1*blf_integrals(ei);
+  return blf_integrals(ei);
+}
+
+/*
+ * Capability can be added in the future to allow the user
+ * to define a custom  function */
+void pumiUserFunction(apf::Mesh* mesh, apf::MeshEntity* e,
+    const apf::Vector3& x, mth::Vector<double>& f)
+{
+  double freq = 1.;
+  double kappa = freq * M_PI;
+  int dim = apf::getDimension(mesh, e);
+  if (dim == 3) {
+      f(0) = (1. + kappa * kappa) * sin(kappa * x[1]);
+      f(1) = (1. + kappa * kappa) * sin(kappa * x[2]);
+      f(2) = (1. + kappa * kappa) * sin(kappa * x[0]);
+      /*f(0) = 0.;
+      f(1) = 0.;
+      f(2) = 0.;*/
+  }
+  else {
+     f(0) = (1. + kappa * kappa) * sin(kappa * x[1]);
+     f(1) = (1. + kappa * kappa) * sin(kappa * x[0]);
+     f(2) = 0.0;
+  }
+}
+void assembleDomainLFElementVector(apf::Mesh* mesh, apf::MeshEntity* e,
+    apf::Field* f, mth::Vector<double>& elvect)
+{
+  apf::FieldShape* fs = f->getShape();
+  int type = mesh->getType(e);
+  PCU_ALWAYS_ASSERT(type == apf::Mesh::TET);
+  int nd = apf::countElementNodes(fs, type);
+  int dim = apf::getDimension(mesh, e);
+  double w;
+
+  apf::NewArray<apf::Vector3> vectorshapes(nd);
+  elvect.resize(nd);
+  mth::Vector<double> val (dim);
+  val.zero();
+  apf::Vector3 p;
+
+  apf::MeshElement* me = apf::createMeshElement(mesh, e);
+  apf::Element* el = apf::createElement(f, me);
+  int int_order = 2 * fs->getOrder();
+  int np = apf::countIntPoints(me, int_order);
+
+  elvect.zero();
+  for (int i = 0; i < np; i++) {
+    apf::getIntPoint(me, int_order, i, p);
+    double weight = apf::getIntWeight(me, int_order, i);
+    apf::Matrix3x3 J;
+    apf::getJacobian(me, p, J);
+    double jdet = apf::getJacobianDeterminant(J, dim);
+    w = weight * jdet;
+
+    apf::getVectorShapeValues(el, p, vectorshapes);
+    apf::Vector3 global;
+    apf::mapLocalToGlobal(me, p, global);
+    pumiUserFunction(mesh, e, global, val);
+    val *= w;
+
+    mth::Matrix<double> vectorShapes (nd, dim);
+    for (int j = 0; j < nd; j++)
+      for (int k = 0; k < dim; k++)
+        vectorShapes(j,k) = vectorshapes[j][k];
+    mth::Vector<double> V (nd);
+    V.zero();
+    mth::multiply(vectorShapes, val, V);
+    elvect += V;
+  }
+
+  apf::destroyElement(el);
+  apf::destroyMeshElement(me);
+}
+
+/*
+ * computes local linear form integral restricted to
+ * an edge of a tet element
+ */
+static double getLocalEdgeLF(EdgePatch* ep, apf::MeshEntity* tet)
+{
+  PCU_ALWAYS_ASSERT(ep->mesh->getType(tet) == apf::Mesh::TET);
+  // findIn edge in downward edges of tet
+  apf::Downward e;
+  int ne = ep->mesh->getDownward(tet, 1, e);
+  int ei = apf::findIn(e, ne, ep->entity);
+  // assemble Domain LF Vector
+  mth::Vector<double> elvect;
+  assembleDomainLFElementVector(ep->mesh, tet,
+      ep->equilibration->ef, elvect);
+  int which, rotate; bool flip;
+  apf::getAlignment(ep->mesh, tet, ep->entity, which, flip, rotate);
+  if (flip)
+    elvect(ei) = -1*elvect(ei);
+  return elvect(ei);
+}
+
+/*
+ * Given a tet and one of its faces, the vertex of the tet
+ * opposite to the given face is returned.
+ */
+static apf::MeshEntity* getTetOppVert(
+    apf::Mesh* m, apf::MeshEntity* t, apf::MeshEntity* f)
+{
+  apf::Downward fvs;
+  int fnv = m->getDownward(f, 0, fvs);
+  apf::Downward tvs;
+  int tnv = m->getDownward(t, 0, tvs);
+  PCU_ALWAYS_ASSERT(tnv == 4 && fnv == 3);
+  for (int i = 0; i < tnv; i++) {
+    if (apf::findIn(fvs, fnv, tvs[i]) == -1)
+      return tvs[i];
+  }
+  return 0;
+}
+
+/*
+ * Given a face and one of its edges, the vertex of the face
+ * opposite to the given edge is returned.
+ */
+static apf::MeshEntity* getFaceOppVert(
+    apf::Mesh* m, apf::MeshEntity* f, apf::MeshEntity* e)
+{
+  apf::Downward evs;
+  int env = m->getDownward(e, 0, evs);
+  apf::Downward fvs;
+  int fnv = m->getDownward(f, 0, fvs);
+  PCU_ALWAYS_ASSERT(env == 2 && fnv == 3);
+  for (int i = 0; i < fnv; i++) {
+    if (apf::findIn(evs, env, fvs[i]) == -1)
+      return fvs[i];
+  }
+  return 0;
+}
+
+static apf::Vector3 computeFaceNormal(
+    apf::Mesh* m, apf::MeshEntity* f, apf::Vector3 const& p)
+{
+  // Compute face normal using face Jacobian,
+  // so it can also be used for curved mesh elements
+  apf::MeshElement* me = apf::createMeshElement(m, f);
+  apf::Matrix3x3 J;
+  apf::getJacobian(me, p, J);
+  apf::destroyMeshElement(me);
+
+  apf::Vector3 g1 = J[0];
+  apf::Vector3 g2 = J[1];
+  apf::Vector3 n = apf::cross( g1, g2 );
+  return n.normalize();
+}
+
+apf::Vector3 computeFaceOutwardNormal(apf::Mesh* m,
+    apf::MeshEntity* t, apf::MeshEntity* f, apf::Vector3 const& p)
+{
+  apf::Vector3 n = computeFaceNormal(m, f, p);
+
+  // orient the normal outwards from the tet
+  apf::MeshEntity* oppVert = getTetOppVert(m, t, f);
+  apf::Vector3 vxi = apf::Vector3(0.,0.,0.);
+
+  apf::Vector3 txi;
+  m->getPoint(oppVert, 0, txi);
+
+  apf::MeshElement* fme = apf::createMeshElement(m, f);
+  apf::Vector3 pxi;
+  apf::mapLocalToGlobal(fme, p, pxi);
+  apf::destroyMeshElement(fme);
+
+  apf::Vector3 pxiTotxi = txi - pxi;
+  if (pxiTotxi*n > 0) {
+    n = n*-1.;
+  }
+  return n;
+}
+
+/*
+ * computes local flux integral restricted to
+ * an edge of a tet element
+ */
+static double getLocalFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
+{
+  PCU_ALWAYS_ASSERT(ep->mesh->getType(tet) == apf::Mesh::TET);
+
+  double fluxIntegral = 0.0;
+  // 1. get faces of the tet in the patch
+  apf::Downward f;
+  int nf = ep->mesh->getDownward(tet, 2, f);
+  std::vector<apf::MeshEntity*> patchFaces;
+  for (int i = 0; i < nf; i++) {
+    if(std::find(ep->faces.begin(), ep->faces.end(), f[i]) != ep->faces.end())
+      patchFaces.push_back(f[i]);
+  }
+  PCU_ALWAYS_ASSERT(patchFaces.size() == 2);
+
+  //  2. loop over the patch faces
+  for (unsigned int i = 0; i < patchFaces.size(); i++) {
+    double fluxFaceIntegral = 0.0;
+    // 3. get upward tets of the current face
+    apf::Up up;
+    apf::MeshEntity* currentFace = patchFaces[i];
+    ep->mesh->getUp(currentFace, up);
+    if (crv::isBoundaryEntity(ep->mesh, currentFace))
+      PCU_ALWAYS_ASSERT( up.n == 1);
+    else
+      PCU_ALWAYS_ASSERT( up.n == 2);
+
+    apf::MeshEntity* firstTet = up.e[0];
+    apf::MeshEntity* secondTet = nullptr;
+    if (up.n == 2)
+      secondTet  = up.e[1];
+
+    // 4. findIn edge in downward edges of current face
+    apf::Downward e;
+    int ne = ep->mesh->getDownward(currentFace, 1, e);
+    PCU_ALWAYS_ASSERT(ne == 3);
+    int ei = apf::findIn(e, ne, ep->entity);
+
+    // 5. count integration points for flux face integral
+    apf::FieldShape* fs = ep->equilibration->ef->getShape();
+    int int_order = 2 * fs->getOrder();
+    apf::MeshElement* fme = apf::createMeshElement(ep->mesh, currentFace);
+    apf::Element* fel = apf::createElement(ep->equilibration->ef, fme);
+    int np = apf::countIntPoints(fme, int_order);
+
+    // loop over integration points
+    apf::Vector3 p, tet1xi, tet2xi, curl1, curl2, curl,
+      fn1, fn2, tk, vshape;
+    for (int n = 0; n < np; n++) {
+      apf::getIntPoint(fme, int_order, n, p);
+      double weight = apf::getIntWeight(fme, int_order, n);
+      apf::Matrix3x3 fJ;
+      apf::getJacobian(fme, p, fJ);
+      double jdet = apf::getJacobianDeterminant(
+          fJ, apf::getDimension(ep->mesh, currentFace));
+
+      // compute face outward normals wrt tets
+      if (tet == firstTet) {
+        fn1 = computeFaceOutwardNormal(ep->mesh, firstTet, currentFace, p);
+        fn2 = apf::Vector3(0.,0.,0.);
+      }
+      else {
+        fn1 = computeFaceOutwardNormal(ep->mesh, secondTet, currentFace, p);
+        fn2 = apf::Vector3(0.,0.,0.);
+      }
+      if (up.n == 2) {
+        if (tet == firstTet)
+          fn2 = computeFaceOutwardNormal(ep->mesh, secondTet, currentFace, p);
+        else
+          fn2 = computeFaceOutwardNormal(ep->mesh, firstTet, currentFace, p);
+      }
+
+      curl.zero();
+      // compute curl1
+      tet1xi = apf::boundaryToElementXi(ep->mesh, currentFace, firstTet, p);
+      apf::MeshElement* me1 = apf::createMeshElement(ep->mesh, firstTet);
+      apf::Element* el1 = apf::createElement(ep->equilibration->ef, me1);
+      apf::getCurl(el1, tet1xi, curl1);
+      apf::Vector3 temp1 = apf::cross(fn1, curl1);
+      curl += temp1;
+      apf::destroyElement(el1);
+      apf::destroyMeshElement(me1);
+
+      // compute curl2
+      if (up.n == 2) {
+        tet2xi = apf::boundaryToElementXi(ep->mesh, currentFace, secondTet, p);
+        apf::MeshElement* me2 = apf::createMeshElement(ep->mesh, secondTet);
+        apf::Element* el2 = apf::createElement(ep->equilibration->ef, me2);
+        apf::getCurl(el2, tet2xi, curl2);
+        apf::Vector3 temp2 = apf::cross(fn2, curl2);
+        curl += (temp2 * -1.);
+        curl = curl * 1./2.;
+        apf::destroyElement(el2);
+        apf::destroyMeshElement(me2);
+      }
+
+      // compute tk (inter-element averaged flux)
+      tk = curl;
+
+      // compute vector shape
+      int type = apf::Mesh::TRIANGLE;
+      int nd = apf::countElementNodes(fs, type);
+      apf::NewArray<apf::Vector3> vectorshapes (nd);
+      apf::getVectorShapeValues(fel, p, vectorshapes);
+      vshape = vectorshapes[ei];
+
+      // compute integral
+      fluxFaceIntegral += (tk * vshape) * weight * jdet;
+    }
+    apf::destroyElement(fel);
+    apf::destroyMeshElement(fme);
+
+    fluxIntegral += fluxFaceIntegral;
+  }
+  return fluxIntegral;
+}
+
+static void assembleEdgePatchRHS(EdgePatch* p)
+{
+  if (p->isOnBdry) {
+    p->b.resize(p->tets.size() + p->faces.size());
+    p->b.zero();
+  }
+  else {
+    p->b.resize(p->tets.size());
+    p->b.zero();
+  }
+  int ne = p->tets.size();
+  int nf = p->faces.size();
+  for (int i = 0; i < ne; i++) {
+    apf::MeshEntity* tet = p->tets[i];
+    double blfIntegral = getLocalEdgeBLF(p, tet);
+    double lfIntegral = getLocalEdgeLF(p, tet);
+    double fluxIntegral = getLocalFluxIntegral(p, tet);
+    if(p->isOnBdry)
+      p->b(nf+i) = blfIntegral - lfIntegral - fluxIntegral;
+    else
+      p->b(i) = blfIntegral - lfIntegral - fluxIntegral;
+  }
+}
+
+static apf::MeshEntity* getTetOppFaceSharingEdge(
+    apf::Mesh* m, apf::MeshEntity* t, apf::MeshEntity* f, apf::MeshEntity* e)
+{
+  apf::MeshEntity* fs[4];
+  m->getDownward(t, 2, fs);
+  for (int i = 0; i < 4; i++) {
+    if (fs[i] == f) continue;
+    apf::MeshEntity* es[3];
+    m->getDownward(fs[i], 1, es);
+    if (apf::findIn(es, 3, e) > -1)
+      return fs[i];
+  }
+  return 0;
+}
+
+/*
+ * Orders tets and faces in an edge cavity in a
+ * clockwise (or ccw) direction around the edge.
+ */
+static void getOrderedTetsandFaces(apf::Mesh* mesh, apf::MeshEntity* edge,
+     EntityVector& tets, EntityVector& faces)
+{
+  tets.clear();
+  faces.clear();
+  if( ! crv::isBoundaryEntity(mesh, edge) ) {
+    apf::MeshEntity* currentFace = mesh->getUpward(edge, 0);
+    apf::Up up;
+    mesh->getUp(currentFace, up);
+    PCU_ALWAYS_ASSERT(up.n == 2);
+    apf::MeshEntity* firstTet = up.e[0];
+    apf::MeshEntity* nextTet  = up.e[1];
+    tets.push_back(firstTet);
+    apf::MeshEntity* firstFace = getTetOppFaceSharingEdge(mesh, firstTet,
+        currentFace, edge);
+    faces.push_back(firstFace);
+
+    while (nextTet != firstTet) {
+      tets.push_back(nextTet);
+      faces.push_back(currentFace);
+      currentFace = getTetOppFaceSharingEdge(mesh, nextTet, currentFace, edge);
+      PCU_ALWAYS_ASSERT(currentFace);
+      apf::Up up;
+      mesh->getUp(currentFace, up);
+      PCU_ALWAYS_ASSERT(up.n == 2);
+      if (nextTet != up.e[0])
+        nextTet = up.e[0];
+      else
+        nextTet = up.e[1];
+    }
+  }
+  else {
+    apf::Up up;
+    mesh->getUp(edge, up);
+    apf::MeshEntity* firstFace;
+    for (int i = 0; i < up.n; i++) {
+      if ( crv::isBoundaryEntity(mesh, up.e[i]) ) {
+        firstFace = up.e[i]; break;
+      }
+    }
+    faces.push_back(firstFace);
+    mesh->getUp(firstFace, up);
+    PCU_ALWAYS_ASSERT(up.n == 1);
+    apf::MeshEntity* firstTet = up.e[0];
+    tets.push_back(firstTet);
+
+    apf::MeshEntity* nextFace = getTetOppFaceSharingEdge(mesh, firstTet,
+        firstFace, edge);
+    apf::MeshEntity* nextTet = firstTet;
+    mesh->getUp(nextFace, up);
+    while( up.n == 2) {
+      faces.push_back(nextFace);
+      if (nextTet != up.e[0])
+        nextTet = up.e[0];
+      else
+        nextTet = up.e[1];
+      tets.push_back(nextTet);
+
+      nextFace = getTetOppFaceSharingEdge(mesh, nextTet, nextFace, edge);
+      mesh->getUp(nextFace, up);
+    }
+    faces.push_back(nextFace);
+  }
+}
+
+void getClockwiseTetsandFaces(EdgePatch* p)
+{
+  if (p->isOnBdry) {
+    if (p->tets.size() > 1) {
+      apf::MeshEntity* secondFace = p->faces[1];
+      apf::MeshEntity* firstTet = p->tets[0];
+      apf::MeshEntity* secondTet = p->tets[1];
+
+      apf::Downward firstTetFaces, secondTetFaces;
+      int n_firstTetFaces = p->mesh->getDownward(firstTet, 2, firstTetFaces);
+      int n_secondTetFaces = p->mesh->getDownward(secondTet, 2, secondTetFaces);
+      int fi1 = apf::findIn(firstTetFaces, n_firstTetFaces, secondFace);
+      int fi2 = apf::findIn(secondTetFaces, n_secondTetFaces, secondFace);
+      PCU_ALWAYS_ASSERT(fi1 != -1 && fi2 != -1);
+
+      // first tet opp vertex crd
+      apf::MeshEntity* firstTetOppVert = getTetOppVert(
+        p->mesh, firstTet, secondFace);
+      apf::Vector3 firstTetOppVertCrd;
+      p->mesh->getPoint(firstTetOppVert, 0, firstTetOppVertCrd);
+
+      // second tet opp vertex crd
+      apf::MeshEntity* secondTetOppVert = getTetOppVert(
+        p->mesh, secondTet, secondFace);
+      apf::Vector3 secondTetOppVertCrd;
+      p->mesh->getPoint(secondTetOppVert, 0, secondTetOppVertCrd);
+
+      // normal to the face
+      apf::Downward edge_vertices;
+      p->mesh->getDownward(p->entity, 0, edge_vertices);
+      apf::Vector3 p0, p1, p2;
+      p->mesh->getPoint(edge_vertices[0], 0, p0);
+      p->mesh->getPoint(edge_vertices[1], 0, p1);
+      apf::MeshEntity* faceOppVert = getFaceOppVert(
+        p->mesh, secondFace, p->entity);
+      p->mesh->getPoint(faceOppVert, 0, p2);
+
+      apf::Vector3 normal = apf::cross(p1-p0, p2-p0);
+
+      // direction vectors from p0 to opp tet verts
+      apf::Vector3 vFirst = firstTetOppVertCrd - p0;
+      apf::Vector3 vLast = secondTetOppVertCrd - p0;
+
+      if ((vFirst * normal > 0)  && (vLast * normal < 0)) {
+        // reverse list of tets and faces
+        std::reverse(p->tets.begin(), p->tets.end());
+        std::reverse(p->faces.begin(), p->faces.begin());
+      }
+    }
+  }
+  else {
+    apf::MeshEntity* firstFace = p->faces[0];
+    apf::MeshEntity* firstTet = p->tets[0];
+    apf::MeshEntity* lastTet = p->tets[p->tets.size()-1];
+
+    apf::Downward firstTetFaces, lastTetFaces;
+    int n_firstTetFaces = p->mesh->getDownward(firstTet, 2, firstTetFaces);
+    int n_lastTetFaces = p->mesh->getDownward(lastTet, 2, lastTetFaces);
+    int fi1 = apf::findIn(firstTetFaces, n_firstTetFaces, firstFace);
+    int filast = apf::findIn(lastTetFaces, n_lastTetFaces, firstFace);
+    PCU_ALWAYS_ASSERT(fi1 != -1 && filast != -1);
+
+    // first tet opp vertex crd
+    apf::MeshEntity* firstTetOppVert = getTetOppVert(
+        p->mesh, firstTet, firstFace);
+    apf::Vector3 firstTetOppVertCrd;
+    p->mesh->getPoint(firstTetOppVert, 0, firstTetOppVertCrd);
+
+    // last tet opp vertex crd
+    apf::MeshEntity* lastTetOppVert = getTetOppVert(
+        p->mesh, lastTet, firstFace);
+    apf::Vector3 lastTetOppVertCrd;
+    p->mesh->getPoint(lastTetOppVert, 0, lastTetOppVertCrd);
+
+    // normal to the face
+    apf::Downward edge_vertices;
+    p->mesh->getDownward(p->entity, 0, edge_vertices);
+    apf::Vector3 p0, p1, p2;
+    p->mesh->getPoint(edge_vertices[0], 0, p0);
+    p->mesh->getPoint(edge_vertices[1], 0, p1);
+    apf::MeshEntity* faceOppVert = getFaceOppVert(
+        p->mesh, firstFace, p->entity);
+    p->mesh->getPoint(faceOppVert, 0, p2);
+
+    apf::Vector3 normal = apf::cross(p1-p0, p2-p0);
+
+    // direction vectors from p0 to opp tet verts
+    apf::Vector3 vFirst = firstTetOppVertCrd - p0;
+    apf::Vector3 vLast = lastTetOppVertCrd - p0;
+
+    if ((vFirst * normal > 0)  && (vLast * normal < 0)) {
+      // reverse list of tets and faces
+      std::reverse(p->tets.begin(), p->tets.end());
+      std::reverse(p->faces.begin(), p->faces.begin());
+    }
+  }
+}
+
+static void runErm(EdgePatch* ep)
+{
+  getOrderedTetsandFaces(ep->mesh, ep->entity, ep->tets, ep->faces);
+  //getClockwiseTetsandFaces(ep);
+  assembleEdgePatchLHS(ep);
+  assembleEdgePatchRHS(ep);
+  mth::solveFromQR(ep->qr.Q, ep->qr.R, ep->b, ep->x);
+
+  if (!ep->isOnBdry) { // solve At*mu = g for g
+    mth::Vector<double> temp(ep->tets.size());
+    mth::multiply(ep->At, ep->x, temp);
+    for (size_t i = 0; i < ep->tets.size(); i++) {
+      ep->x(i) = temp(i);
+    }
+  }
+
+  int nf = ep->faces.size();
+  for(int i = 0; i < nf; i++) {
+    apf::MeshEntity* face = ep->faces[i];
+
+    apf::Downward e;
+    int ned = ep->mesh->getDownward(face, 1, e);
+    int ei = apf::findIn(e, ned, ep->entity);
+    PCU_ALWAYS_ASSERT(ned == 3 && ei != -1);
+
+    double components[3];
+    apf::getComponents(ep->equilibration->g, face, 0, components);
+    components[ei] = ep->x(i);
+    apf::setComponents(ep->equilibration->g, face, 0, components);
+  }
+}
+
+
+class EdgePatchOp : public apf::CavityOp
+{
+public:
+  EdgePatchOp(Equilibration* eq):
+    apf::CavityOp(eq->mesh)
+  {
+    setupEdgePatch(&edgePatch, eq);
+  }
+  virtual Outcome setEntity(apf::MeshEntity* e)
+  {
+    if (edgePatch.mesh->hasTag(e, edgePatch.equilibration->tag))
+      return SKIP;
+    startEdgePatch(&edgePatch, e);
+    if ( ! buildEdgePatch(&edgePatch, this))
+      return REQUEST;
+    return OK;
+  }
+  virtual void apply()
+  {
+    runErm(&edgePatch);
+    int n = VISITED;
+    edgePatch.mesh->setIntTag(
+        edgePatch.entity, edgePatch.equilibration->tag, &n);
+  }
+  EdgePatch edgePatch;
+};
+
+apf::Field* equilibrateResiduals(apf::Field* f)
+{
+  apf::Field* g = createPackedField(
+      apf::getMesh(f), "g", 3, apf::getConstant(2) );
+  Equilibration equilibration;
+  setupEquilibration(&equilibration, f, g);
+  EdgePatchOp op(&equilibration);
+  op.applyToDimension(1); // edges
+
+  apf::MeshEntity* ent;
+  apf::MeshIterator* it = apf::getMesh(f)->begin(1);
+  while ((ent = apf::getMesh(f)->iterate(it))) {
+    apf::getMesh(f)->removeTag(ent, equilibration.tag);
+  }
+  apf::getMesh(f)->end(it);
+  apf::getMesh(f)->destroyTag(equilibration.tag);
+
+  return g;
+}
+
+
+}
diff --git a/ree/reeSizeField.cc b/ree/reeSizeField.cc
new file mode 100644
index 000000000..fdd382925
--- /dev/null
+++ b/ree/reeSizeField.cc
@@ -0,0 +1,173 @@
+/*
+ * Copyright (C) 2011 Scientific Computation Research Center
+ *
+ * This work is open source software, licensed under the terms of the
+ * BSD license as described in the LICENSE file in the top-level directory.
+ */
+#include "apfElement.h"
+#include "crv.h"
+#include "crvShape.h"
+#include <apfCavityOp.h>
+#include "ree.h"
+
+namespace ree {
+
+
+struct Sizefield {
+  apf::Mesh* mesh;
+  /* the polynomial order of the solution electric Nedelec field */
+  int order;
+  /* the input solution electric field obtained after the FEM solve */
+  apf::Field* ef;
+  /* the per element error field obtained after executing the implicit
+     residual error estimator */
+  apf::Field* errorField;
+  double size_factor;
+  /* target error = sum of all element erros / total number of elements */
+  double target_error;
+  double alpha;
+  double beta;
+  /* a temporary field storing desired sizes at elements */
+  apf::Field* element_size;
+  /* the resulting size field, recovered from the element_size field
+     (using a local average recovery method much weaker than SPR) */
+  apf::Field* size;
+};
+
+static void setupSizefield(
+    Sizefield* sz,
+    apf::Field* ef,
+    apf::Field* error_field,
+    int n,
+    double alpha,
+    double beta)
+{
+  sz->mesh = ef->getMesh();
+  sz->order = ef->getShape()->getOrder();
+  sz->errorField = error_field;
+  sz->size_factor = 0;
+
+  apf::MeshEntity* entity;
+  int d = sz->mesh->getDimension();
+  apf::MeshIterator* it = sz->mesh->begin(d);
+  double total_error = 0.;
+  while ((entity = sz->mesh->iterate(it))) {
+    total_error += apf::getScalar(sz->errorField, entity, 0);
+  }
+  sz->mesh->end(it);
+  sz->target_error = total_error / (sz->mesh->count(d)* n);
+
+  sz->alpha = alpha;
+  sz->beta = beta;
+  sz->element_size = 0;
+  sz->size = 0;
+}
+
+static double getCurrentSize(apf::Mesh* m, apf::MeshEntity* e)
+{
+  /* right now minimum edge length is the formula... */
+  apf::Downward edges;
+  int ne = m->getDownward(e,1,edges);
+  double h = std::numeric_limits<double>::max();
+  for (int i=0; i < ne; ++i)
+    h = std::min(h, measure(m, edges[i]));
+  return h;
+}
+
+static double getDesiredSize(Sizefield* sz, apf::MeshEntity* entity)
+{
+  double element_error = apf::getScalar(sz->errorField, entity, 0);
+  double h = getCurrentSize(sz->mesh, entity);
+
+  double p = sz->order;
+  int d = sz->mesh->getDimension();
+  sz->size_factor = pow(sz->target_error/element_error, (2. / (2.*p + d)));
+
+  double h_new = h * sz->size_factor;
+  if (h_new < sz->alpha*h) h_new = sz->alpha*h;
+  if (h_new > sz->beta*h) h_new = sz->beta*h;
+  return h_new;
+}
+
+static void getElementSizeField(Sizefield* sz)
+{
+  apf::Field* eSize = apf::createStepField(
+      sz->mesh, "esize", apf::SCALAR);
+  int d = sz->mesh->getDimension();
+  apf::MeshEntity* entity;
+  apf::MeshIterator* elements = sz->mesh->begin(d);
+  while ((entity = sz->mesh->iterate(elements))) {
+    double h = getDesiredSize(sz, entity);
+    apf::setScalar(eSize, entity, 0, h);
+  }
+  sz->mesh->end(elements);
+  sz->element_size = eSize;
+}
+
+
+void averageToVertex(apf::Field* ef, apf::Field* vf, apf::MeshEntity* ent)
+{
+  apf::Mesh* m = apf::getMesh(ef);
+  apf::Adjacent elements;
+  m->getAdjacent(ent, m->getDimension(), elements);
+  double s=0;
+  for (std::size_t i=0; i < elements.getSize(); ++i)
+    s += apf::getScalar(ef, elements[i], 0);
+  s /= elements.getSize();
+  apf::setScalar(vf, ent, 0, s);
+}
+
+class AverageOp : public apf::CavityOp
+{
+public:
+  AverageOp(Sizefield* sz):
+    apf::CavityOp(sz->mesh),
+    sizefield(sz),
+    entity(0)
+  {
+  }
+  virtual Outcome setEntity(apf::MeshEntity* e)
+  {
+    entity = e;
+    if (apf::hasEntity(sizefield->size, entity))
+      return SKIP;
+    if ( !requestLocality(&entity,1))
+      return REQUEST;
+    return OK;
+  }
+  virtual void apply()
+  {
+    averageToVertex(sizefield->element_size,
+        sizefield->size, entity);
+  }
+  Sizefield* sizefield;
+  apf::MeshEntity* entity;
+};
+
+void averageSizeField(Sizefield* sz)
+{
+  sz->size = apf::createLagrangeField(sz->mesh, "size", apf::SCALAR, 1);
+  AverageOp op(sz);
+  op.applyToDimension(0);
+}
+
+apf::Field* getTargetEMSizeField(
+    apf::Field* ef,
+    apf::Field* error_field,
+    int n,
+    double alpha /*= 0.25*/,
+    double beta /*= 2.0*/)
+{
+  double t0 = PCU_Time();
+  Sizefield sz;
+  setupSizefield(&sz, ef, error_field, n, alpha, beta);
+  getElementSizeField(&sz);
+  averageSizeField(&sz);
+  apf::destroyField(sz.element_size);
+  double t1 = PCU_Time();
+  if (!PCU_Comm_Self())
+    lion_eprint(1,"EM: SizeField computed in %f seconds\n",t1-t0);
+  return sz.size;
+}
+
+}
diff --git a/test/residualErrorEstimation_test.cc b/test/residualErrorEstimation_test.cc
index 865255992..4ce4bdb41 100644
--- a/test/residualErrorEstimation_test.cc
+++ b/test/residualErrorEstimation_test.cc
@@ -1,5 +1,5 @@
 #include "ma.h"
-#include <em.h>
+#include <ree.h>
 #include <apf.h>
 #include <gmi_mesh.h>
 #include <apfMDS.h>
@@ -35,7 +35,7 @@ int main(int argc, char** argv)
   m->verify();
 
   apf::Field* electric_field = m->getField(0);
-  apf::Field* implicit_error_field = em::estimateError(electric_field);
+  apf::Field* implicit_error_field = ree::estimateError(electric_field);
   apf::destroyField(implicit_error_field);
 
   m->destroyNative();

From 5422a1cfc87543734d8f4e590aaf1125de7abb12 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <39503305+Samiullah-Malik@users.noreply.github.com>
Date: Thu, 3 Dec 2020 18:43:34 -0500
Subject: [PATCH 287/555] Delete .nfs00000000000286a50000bdda

---
 ree/.nfs00000000000286a50000bdda | Bin 57344 -> 0 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 ree/.nfs00000000000286a50000bdda

diff --git a/ree/.nfs00000000000286a50000bdda b/ree/.nfs00000000000286a50000bdda
deleted file mode 100644
index a7d66c4b90f11a9164288cd908cd8e194ae85164..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 57344
zcmeI537A|*b?=*<uoy6eXO=ul#>hRR){JH(Z;~Z5mPR0<&C*!L2s7#FzCCKIr@Ph5
zNFy0+2sjRyJ?!}7#Wn;U`M{8{#fGrN@Pctz92N&&0zCKv7zj&9cr5Yzol{l!*6n3x
zq}aU2OZTV0divhFRj1CW?VM9}?#!C~+jg3BM%HEcJTjBH?<?Ln{mbj_o7uiKlbOp`
z%7>cca%a&`qc+#Bw3{Q%Lah-MMjG|<NK|ZhKQdb3{;11!v=J2#=3A@hO3~cvVl*FB
zYV}dcU^JTJ-`=QMF1GWPt?g=|RjyU@mF7sH(4FVt%76m{4lIKM?N({^xew2*9UogG
zsh@oM3iIewFJ8ujgCqkE3^*|0z<>h-4h%Rj;J|<b0}c#0@c#=3TJ?uzuB0;ebrrh8
zecz|!`(f_)<J@<(<Nib4?|Juqb;tdOxZf@J{i}}qzvF%%ci(kKfT!cZ?)p*peOJf*
z2f6DN_g!!m^3OleUH9dGw&VW&-Ti6zz4?Cj5C8nb-F08ipLg7UfV=PM{BIri-OtPh
zm+q|{_ub{e*MI{94h%Rj;J|<b0}c#0FyO#|0|O2WI56PAfCKjm2lB;C<{>D#j}?6s
z|G(G&fBlF|=A+=l;0Ra+&IFGJ8F1GrnauaV=fQ`-+rg{AE5Y-@Rp1b~92@}80%w4S
zgI}Ob-v&MkUJL#RtOr@}a|{h111|(kfO#TQ15X0K2kyqua3{D8Tn}CY{ungCrC=4v
zfX`!axE*{DydS&|+yGt-4udL~1LuKr!BfDg;3pU&J_+6k4uL)30x%371Ad8u{|)d#
z@D^}2I09CK?_eDGHux6!Fn9$hfiu99!7njF`~v&~cr!Q-4uXroPOt;40pG=#@F8#|
zm<BoU9Sjuj1=oSsffs@2gTr7BRKPm05<D9G5P5zd+y%r|aXq*e%z_I*4rIY;KxF?T
z>>=L;9{_IxRj?0S1onV6Kt9b@zEv)m`C7SX^2Oq0wN-AFqvlj?Prg++n9XFKn=yQ%
z`sz)ssMQ>)x0?s2XYz$Z*=T43x5}lg+?$J<2Ul;JjapL+^@v+$<3^LOm)5V}$&K~v
zrzWR{;(SW^LUckI2?2Z@C6#7m8d0m=sG3r~A{Uci%}8jn3FGUnMtgaw^kJ=%DK{^y
zwX4N^V__@QX6JI0B~(WT%9jlpsY|KW$d;=uQ#KpTnhkuN?~+vcv1-+jIZmQE<;lhM
zY~(Ufo+)E4uG{n*GBdSWB`9#RI2&bS-VGbdj(@3m7KRNB#;_LDD%-A)RE6ndL*@cQ
zjU%&O*kc(IlTpf<c0ENyyWT~B);KP*HvuI}!0>2a_IC;;5lbi-$8u%5ZA0bew(90e
zt=VowyGz-*S~03jN3bMk?B61XA={XYV~nKnoJkTP7n@_LTrKWQ(qzYTX1Lr^E;uo^
zNZX8BBQw!#xtbk9fGxAaUW%$kd%=s;m}aZEem%+0NBo=CuP?WvM!r>R@Q^gV{zOSF
zVxn3#B`F(8hg)#OjE<Ue)wB*qreHZ>Y9*7eF34BwV2y^a$?2aXS`z8oT$Bg2-tldp
z4OE&|%@htsg+p}JFr^SvZjz1k;A(5yGOk7Ub<Kzd=4BS8`<!i5)YSJpX>YdfWG0L(
z(0N*6w)t5nGwCuxarxPpBMctOlBxV`)(WnJN6Mt0)Q)Rw>{FGsL!M06AEeG{DzwtB
zwZvq|K-QlQseSsBY8ES{J=rlJol1VieQQ#MPj}YuTxN85sj-03G)Kp`%pMy*J2!UL
zSZ?gx^KxV7jq}a#HRrC8|IXsSwer_FXXVa1J9p07+}Y>m#?Q-*ugk5Kr=Gh(`|jv)
zifo}8fgB%OvraQOmkiF*T;#vA^*_0B7MYIa*2(R$+&O3GGR9oAcS}^L&DG7}hEO-x
zV%Xmw73dh}7i;aAO0>xw?RR4)zOggkYLt)0PfTqvwDBoOni!sFnN?=YXWfx-Li?T;
z+IdIvwv4vq_+hu>s?2VZ)Jb4>^}=FfK`!)>mfYf#)p^0JH1hnA$v0%$pr4hSM*F)J
ztx;x^oXZ6+MQoNFF?my$N!Ao<l?t5V2{-#@j2Ru)&Kl<7@;-8~%rnyG^=Z;h9&VQ_
z<rzjHHdfM_!k?)cFqYGRHZbWo+J%;xWDL-0UM3Ui`NgPYN4I$^xX!9?N-+}Ghc?7s
zT;D6@_PTLVTzcGK+)Nf7_R9zvbKL-Kr*fr(rn#B+Rjb0KW>!+q+DSEM*5u5ws8MU`
z2mya8k;$rHXgEo>cA$<V>ml1U<N9rLWB9O){NXcpMi|$O<J6O`+luI=p0<5k5)Q1r
zOghk-5vWqYB9bV%+?bX0@}t%$Mhz~MbcL|*V4l%tNdMj~cbIcrLPSC;|EpM@<7ZMQ
zMyoC5xv1Ji^2swIWs)huGM@1>TO!BfspTFFZP!z6C!6{uEy=Atww9wxaoDhY&^+DG
z*&){gQWu3(10$=(){#BHji`y-iskC8%tpo9k?N6rqo^IEY3H^`d!_3p!=}*YnQE)D
zVAh&uAz#Ti%zVDmj+%6^h$hOm;-aZlO_ZlAh!`l79#d<#>g|>(3AOa9YAau^N})=E
zZaa3_$n+T!9Vvs!p<I;=T#;S*?4pOzQ>j6zDGNZ;tX105>5N=v<SMwbwY$qCqbv}P
zkaxXSS*X_L%K3`p470k=v00}Qw)*lIVs;OyDEVncoTEhQ+b{?7)nY}ewiY(PCM{5F
zB*O0O=iz`rUl&fzvuLTX?5t9aIUP7;DN$-yOtIaNj8mhC<e#bLAx7zp=48ZZHzi$i
zm6?RL{^WACP-zz<b9yv4a`5!<LZLCQ4P7oL$+()nKs6>S5jAe5uAk5K&G~s&OS|hM
z2R8*7pP#SH)*9v3!MROIzULR9hfy)SRIV}dMR}4b!-ma;`?i=$nGu;)RK977QM1q}
z&(OBqGHl1T&6B(KO(Io>6nCvvtFNxe{&u9q_ITRj0?ldIOv#vkq}E`eMWe1qRb<s}
zAk}8A)S|t!f^|g|4RZv&U&e3-o;lwK$VD>RT&piM%CiSsCcAmajIS9RGy4i<*ikOg
zz+?<;i_%7`V^e*UZ_oucb2n-r*o^4^x1rbGf$l5%KN8W;SJ3g_3SI(U1fCDB0tK)c
zJPw=&?gzerPJbgf0_MT9!Q+7F`5yp(4&DQ90`CS_gBOA;z_Y+6Ao~Af!KvVz==uKu
zZU!#{M?o1p6HI}P;5@Jbh)v*u;C|q~;0x&fp9Aj%ZLl8<g9n3eQ^z~Om%*377lG9K
z3t;dWaA3fJ0S5*g7;s>~fdK~w9Qdu}z;mHIFm<a6U8|4e>-EaQR4q||$1vPtEfSN7
z*fnG8#_l@H@$AZ!LZj++4+S`#3j0FN%xc<{W}H@zRN*yFp-j~6oH>2=^qg7ibY^aB
zu2pfjU>L<-shNf<s2D;EC?_pUGKE7?QFZJrDtnkHsYB!s)HbcJz^UXVuad{LSAM9K
zN9eTnlt#Nc*_eYR-T<W39OwRgxzXa3uJZ>8hqGd9caK@G%f4r9+hgnOZ2Y^Q;aT7=
z?47*m;>mqe_9|xQtXW}Zuu&Cbs)Dj}YAti-VffQ%^-g8|O(BuH+bt}PC|FTzQcW_6
z+r<7BM@dY*mA0>Oih$^F%m61wuV|{BNpx_@C#Gw6z1vbUv_4bE%vmn1M`ndezS#`Z
z<BhR*t=Yk6?^<?$D;KV{MCPb`X``3eTdh4z={FcU{DlKiSqxF)n>(5<yW857v?2Or
z>byIV5}d6oC}{0vYT+_w&*qD#Cw5#iap}J4iGBMf_fBQ2Xwvwk)|!<oO;i?_%4#|(
zoOBj=@trA6;-cAB6)zc&<dkaBcVowh*hDlly(RKcQ`r`0Nm5&1&W!P7SW|!7V5QSS
zwJb4^_~s#|BgZAR<Ko>D@_P98s-?iKr{suTPS~ptjx@{1FyLu9R+{#I>p2w(kh&Gk
z)ni$R`6w$-O{aW_lA}6WU51G-(b-cun%^o*zm${{Ez7K<bRsgRe9`~q6z~sZqdWS)
z-~awC^!ks1Yd{&C10DcAhhG1u;22l{o56+P6mTcH`;FiRFbg(<2Z7ts%WnnZ)4v4l
z2B(0pqLbeNUInJXCEy~k6KnyG1>ZwAzX`k>Tm{yFyRcWh6+8#zz$)-i@I`#^?*e}W
zwu5b882k+1`)9#BK^4q_ePA4jPhR}-?*h*Q&j3#bKgZ|(Z{QQ)<KPwGG9bSARp7_?
z*#8E+23!U%0jGge!5#S1#ozvBa2XI^`$}*+FyKkx*Z9+a34Q^70A2?+%iqZ17H~88
zOYi~ke(*l<7r=dt;ScG*SRc<n-<)o;yCyFg(h)?>NmVn0JA&)nOnlRr6$!BrKeBo6
z{yvXjW5ZDy`?TF?DJ@9)Qdfryb5)<5!6~=imhK<Y4j4P^lB==Pj?7&j);`%IPdpC&
zGZpvSa&k3OQL}|1lI<5M>$aQY;xU`Kpgv?566p>aGq9A$%7@*!QY#!fQf@{gX5zx#
zd#9A=xFjle;8I_x%xT&At>s3ug~6^WCe(`AWY!EB8S1k0JMx`3yf4rX3-?&5=l3gB
zZlp`?PkP@>q)zEfod3Il1+>wOvPjtufX;WGGo4r5h}`|04s+pSdO;oT@iVOFTuB&D
z;i)*nakSwSFIqpIj_W5T4$qimO0bT>wZ_VAqha=LO+~HU_4@v((b|lwVzsHSvGUhB
zB!0j)0UR{a6#6|vC9J7fY_`_5hWmghD`YATky^bjwc=Kz)eL4TUy5sAZmulg=xS!`
zW2@`9z3St9!7Z(e>(1IU_tcoAbY66|P%l~Cda{guTyhq<Li|>yLKR+W;_W$Ox*VC&
zSWC!r?n1T?6%mWBX{j8R_U3`H%Pc8=CD2WEI{g}#UaBc{rv#upVBW8f+nZr#e)^X-
zYAdA$r*aST^yziQ9Mfy_MTAZY$V=?@(&+nfJ>7h?(DWr*R9LW=b5|l;wp2Qnl}i%{
zEzBug+oW@|!b)1Gl@-;JWZL?q6^f9eqv#>Ues$|;=vujsYkg|hovKIme4=I4&nHc_
z#vT>abbV{sy=X0!$LvZ~hH@M~!k^BStGk?Dy*h?VhiI~polgH&GUT3<m#iVBsB|8_
zQgJ377kcHAE{P0#b^=FD&anL{q%0t(I-M%5Urs)eX61h<!C_O3ds8Y!y6k?-az~#f
zfRV9zfGWwiTV}5w(`C9F3&KY43kx1yvuIkO+a@lF+z!~(I#V9cc*WOG+WtFOs4Z{N
zfEplMs7`F23g=!=iHrp&xO7;o?7s89pChHbo@M7N*eZ$($!e)%nRHU6_N%e(Q96kg
z$H}cZWL3SKRfp_1JCS9e=V?ZHnx4n}%A?61s%kMA#nNk!j>*q0K$lp3Xmw6>{n1g<
za#tYnM0LVz5L<ESjopioUj3YesexafOf5rtk&>GFyA~o#Khe1q>EKf5VkDiR9}t(Z
z9C7*jCweVNRH{9eB%=Sn9NqmkAo@S*vVLwx*M9?eF35vbU>N+urTsMDe<RNfJ_8O6
zI56PAfCB>#3^*|0z<>h-4h%Rj;J|<b0}lKqIbe+>y^fKo8(mz3fkjX)k7ABlAvelR
zdwdJemDqAPr7<6wiPrF38&lt`p4`A+=p5>LcPrkQi}tF+@o;10BI^<5uf6uK8Rur#
zV%XXd&2gY@((fhN`}BQ0+56s(<lgA0Y`yT_MQl7xyjWtl;53tcn)`bCClU)#c(>b6
zYY7q#*uT_z2??VApNX6vjqh3Xe;*(3Hgx?@flq?J2A=@01hV&k0!S=?hXRQMa65Pp
z5IcgL`M(7015X2!U<;T4j{;w3Kfu?(D?tI|!A0Py;Bnw<><jn|`0wBvFb&QIr-8p^
zKfrr|>=$??sDV8o2foI>06FJ>BanRp&jy>olfh}=*X$4YH}F&NJ@EJ7ufSWtF|ZR1
zffe8hK+gGp6?_Q10yMy3kOijzIp_aL@GkIDPy<t78yEqP0H=VzW-q}F;5s1t3yy)K
z;0Tbt2HU_o@L2F|_7!{pTm$|POo0c2_pzry&iAhXKVt8|KZD!BYrq`X3LXIN559pM
z-wv(>mw?9rk?jM)azC~!g<b*c9qcqVsxznaV&cu|uy*xG*u$X{r{d_6+BT)ZYgZAf
zy4;I#@|1Cp-q2y&lU}Xa^>yr|jc?8!n)cEz?!4-#uJ<fBLSRde+WJJxtE^AGbX_&&
zB2`_|i#48dVcG3hrSdq#p^Cumzlyo9qmuZrI<MH@DH)!eBdbqFoh4e=jBHX#Z}^Jo
zQYTdWOX-eVmu}O#hkRO{CRr~#$Eh?0&abst`?eL6@f$m|Z%JL<!Iu)G_U^2{nF~cW
zeM#n)ID45L(~~AURtOg%ZZ3B(m^ax?qcK7HPFeBn$HB&-a9@p<8pH{yMn`p%fD?=N
zp~W&KI)+aYN2idn!C#~<5MhS^sT)TK<$$7;ujlh+&)`wHZ~u@aOo`m3K&mDmVRHvX
zq>tN)EsHA@MI4PNW|?(jWi$`|bLIF{i6n%5{wA(s&121FK1jt94AJw-S$RkUcxWsG
zDRBt~b+r{wK;jPa&58|;$qo<487ZM>7Q(-!S?(KV&a$|J!Tzq}a9270ekUiphcV9;
zzgZ+?N>C{m)P#yklMh54r|D>Yz7+<n@x-tN)Xd=AJmlFY(hWA<34>h2Q$cG$80|%5
zCYw5C*a@P-NEL#;qdnLj6KP8f>nPF+dWCY5$T1SN#d%nd^M|cXQm3RTD22%YBVK6I
z+fPTT<u>R3LP|Clg;j88xrJ>(O&lBIx(G?#IS0?iy*0SDbUSr*%=r`LmWrScbT^qG
z|G3PK_U&?Yw_U&i&rSva-@avV%}C9bYL=2FC^sIz<a+PkBNbV?`koL8(9tyv`kSsA
zUFG-+`Htb6QCC8rkx%GbQnx&PclIr1>2{KC@jbW^u#NJA3?;-s*7hH}b&Sw4W<}7E
zCzBaO8-%fQp_Nsk!pUkIQ9*?t|Cav7$8MpuJGX4`k6gz!&1WdJC$*1$7c<nJ+wT?c
zNrnM`Xg%JbYO)MBg~E}cV0&R4fP`2<JftVnRi|v~ESPMZl;iAq0vC|T6!ciVUXe58
z;b|M)_2@?yAJeJn_Q~6YgOcxe@S~1$;)VEBn`E!Q^2$b^^3LpRNuIWw^=-h89ztc=
zBF74Z(K7n#BXW{}uu~G&p<K1`hI$RR>w>YiqhwYHY<?=)Qe&oDNfO+md0-u7TKUFz
zdxyUwr#MCYe2;4!KXC!px{~)@(+K;ix$r#LGmVYG>}$^nM>tEOn{5ziv$gkNQ!cYD
z+-^}8{eLBj;ZJ05Ir@LaDTyCMx4#<5*?;l>4}pgRiTQU6kobPHK=%Ki3S{s9SHL^K
zwLtXw$AeSB1HcE--(Lk@3?#1Ke()gh3v~BC12=*jz$?H45S{*;=<Tlub+8d+!GpoK
z(9i!D_zZX-h`?#`8@>E`Fb8&lG4KfRUG(s8fUg75*RKO_2G0Y}1ZRSupnu;C-U!|R
zt^`}bQ^DyV1MWr_zY}~Bd<5JAZU)za7lSsq5R8G<;0$m-@Dud*AA&o;2Y|%zdjTkc
zN$_8R#Ok{Ryb#pEIxqwzF5eyCCU6xv2IfH<OoJgHviq6HZ&{yIGr6!*Vz2U!lC+hu
znQ(naDb-L?k8Q2Ak80EEcucN$JSF{cMcT&Zo2y@l$V}60#YL{#Xl5#<ZA7S;CFTg^
zB|mHzx9NRZBrj2{Pah_6jhz$_FcTl8QB_}lG6D1}iDJ8Rjkozp-8IW<Y1N5m!^9FY
zBqO=Gb5dcMP)DcVQ6rv>)YVroTbIK-UK?@GB`gY_k_l_a)}HCZYA40W0~dBa(KQlL
zIE7FHndp)eP+bezY7`8uj?R>z34}Nz;tK9bMWq4txaEZD@5rjTDrc%7#7@EfNR_o#
zRGm_*vvG3EPE8`a5eWfhXKX9X5yB41sVrABw|7)mJF(zT_;F&8br7f-O0VF0T7gc*
z*6d|87WwGW%tVLwQ*6*cgr%0wRBLkmL>7X+DTVuKiDlDj)E0KiX*U~9CtI5HmN#t!
zE=7Dds<@S)EsRwwM1<KJip;hSapGmZDFLg7`0Gfdk!ps`<;BRwKh&*hazL=%lAt-Z
z+uL<82Tb)z#O%&NNIp$Dtmyl>l14wVODl~Rv|mF%WlK?oy@9#Zla6AygxD1wnP@cf
z3lb{GN1t1}DfIw~y5D>6+7(9>?PSir+G@*jV;Auvo2)zNnjA!mOOZxuLnJ2iKhaJ1
zZkyQkv>lVa3K%DS^=&PUR@$=w4sz7D-EaqrQtaZM=#Nojb!3HR^6bga&vGtFq%q|4
z@FH>*?;-5D$S<;GIO-+vs1l|Sf*f6lb!=pGT>G*-th!0XN}p4P<J@08HWD+%<!%MG
zma$Bdc%JHEl~R>;mlO~d=mou*>se^gQE>cdxlc+b7>;pOg52f2v~R%T`RO>y8gYKR
z2P1+`AxrL#{DQmj2{doeor(u)VGCms1pB(qxQMY+HAm4i%fa(hUqYHme+n|ALC1WQ
zF(grtoJAqkI+TUJ34|gINuG<mv812allzjP$W>%E=9_QWdF&luLwoe7ROP^pJelOv
zAxrMVnlUfSj&y+}<KqwZ<olA6?8qsow#cy~x8QCd2d}KgY`2v9DxQE4eNLeagRZX}
z%1AS@{U@HEGJ44k>Gp3r2n4u6Lz)Oj+eLrfr6&t}P~TzDNE%Ux$&x{Qxmm|1>_zgH
zCJMh)6?486Z{_UMdXs_55)wA9ED;j*Un~nDSERTMPApAsJjn}>?=?D-R)D~h9-c^~
z7teZi?vhw^ILxb|5$(=y;>k7iaF#n2=o-uRB&0+V{lcG>O-XMk=B!(j$t>-gI+N`7
ze4#c|&R4fY5{Ir_mEaT^TZ-?}K8dUzbowoAk2BRYHs0#mqiaoRyB|VaX>M`g<^sxN
zp>l~X7;J9LbkiCk5z~D-Q=8U#?WEu^Y0}|h0|2ITiNaka3U{^6cM`2xn-p5VOhK(f
ztR8eOwuFX_QC1dy(BeCuh}>A`CNOJAqnB4|HNuK8H~STi2IQ;DsY@p;{(=_IY+YPz
ztG2XCtSnM>a&=d7;?a@iPW-UO1zBQrw5vPO|DT55C}+<_|M&0LyAxgiE#NvJF#yB{
zumd~=$QgjIfOmq|0*U*7HFzQT10c2mu?w6A9s|CQ{{LQZ6L=G7foZ_&$TJTIcVP$k
z2zUp0D|iEV9e6E}*#F~T4UqVLw__K0EqDPaf=M8;0wl)YBf&pmANVYg*#DmaVn4VT
z>;q>2*$?oq*a|)d{usO(TnVOu#1mKvP6gk`ZXouA+rZnv%fOZ3IH&=!F;qYqYyeLJ
zPXKpdOZYr^6?i3Rf$iWN@DL!e20jd`K;r#B9XtU%9!R_au`9d@%mIlvAh8A>0KSBc
z;632&;17V<4=x3x;E5mu?#5p5Dezu!92@{6U^VzD@_j#e84y`-0RPdC?fCk??b(U$
zfO(8SEAgUEmR-H4*I*EIty=m4upON&WcgTXsykN=lU`-hA~lSSF8U=Ase3N!^kGfq
zAErm1HNv#P3cg2&RKJX@pVbiW6IAR<O6VgviL0BpV~R7;?yO<amINZ{Hff7qN{Ex7
zvZ|^2;W+KK4!E;;xDpWpby_!qFF``1SUW8y+hSMnT$g8XBAH`v{ur5|UMU|?dYaBi
zIf<;bro+M08%?!csnlB<(z{1WKl%Am#PY9s_HyO~E>1iA2HvP{zQ(EjnolTCLRiQm
zDOeW;jCGNOQUbuo9Ha=`P+fHol2Iv*uq{P-yNKgK;I=|aYe&Se1Hn3VGkN%cuHL6(
zWh1<H6JwWVEmjaUwd;)5t~SfF67NR)4s^H|OeEwuV@6<*fetf163k0+gmCM961jA9
zxhqF$%vMYDE2bbx^74-ob|T;83HdK?M39E65|M>W)4xl>_91&oLpv}P%d@f{MMM<b
zk(FdTp$87sROlTo%4*eC%hzGZHr!2tyrhK*_td16Y74O=0zp}afL26Xg1=`2Zo|?s
zD_W(}N2=yIvQ9-{55&67+c}*`d$K)6(3FHaAydqkZV-|>*_WmSF}qzEw6ty$Vqi#$
zSjTj=YRTQZkG#`IUM(Rj9*JEo!ImZTuv>RGX+~Hn)#MP5mK=%2qEs%lXqT;8r=O@Q
z47W{2)?(FRECeb8F^nB8Pt{sS%fnS0Bjq?rl|*kc@CkU3WD;!SXjummP1VD=Vb?k?
zm$`ghIWkpyTBSCVuf!`=a!EuRlqGZ}EV8v<VuIr(ED}JS2dz)Zu1sq-K#1-mgYfRc
zxR}(#Q!q~zouJ*Nv?7#{=Eh@J3=bjvsbnu4MA3qUE}bk0fsGioUAlvzPMdJSMr^og
zHiUL?U*r`iTu$ed8ry}oh%026qiQi9n6E0%x@K#oY!FJxJ18x!=*CqSw_C~&Ssk%;
za$&Z5R6(D$778=Vs(p4m2(sr1;^IcL-EHtm(KIW1=XO2+?-|si;W-PcK8Y}5<;dwq
zRyDG<&^K9#TBKT$y%sL07F$ob=B8pL;yW)TD9~+rolnioL=vbSbD;Qs+l_htoFiM`
zizV_Q<JM+pewBgCM=r7P+VzU3JGVS{Ck-LRt3<4!chmDYkgjV>DM|M9D9<MotjzOT
znqP{|V}*^X^R!1zrNgB4E%%x&O4a#bR_vbf$VulojqYLpwu?8JEqIt#(BU?w_G`58
zq<<&##Js*VM4r`*JD+6@$+OXX;B$_j<?1rs&7S|kTqltYL*|FKXqa1(yW3WZLO;xF
znuz&duh*Kq1Vcg&$yhL3o+n6sBpU#TSZ^H<yp}|gK22}s?Q&USEfHl<roXbhq=W~=
zE=~?ET#N)|@*#1Sv3;Pmh_DectQKM0R7(iEwS=&_FT!5eQ`ma5Q(%m02#mse2%D=i
z)Si4p=@>~*U~j>q!Vabc-T(i)D1yHy`XKuMai<i113mxi;CApj@Eq_ga48rD4+h^s
zx4!|r4v6kQ1<nRpuo654oB{p`9ske30yqL>&;NFi1NR3%L9hQ1cmdc3&IM<J$AVMA
z?*NGtcpbPFTm!BKo4_gHVL)R0e-gX_RKOM>@AW$eJP62p{XP#q0p0^{0`p)iFyP1N
z?mq(m2tEQ{4W0$IfHS}k(A~cTz6kyVTmzm1B!-}Wzu!;M;lBWUEWzjS_a1O2kp2Cq
zf=7XWMW6pNkbMMy2R;wp46XuKfc4;9@W0XNKMQUFF9$^+`vuMfKS!tk0{AF+F}M_5
z49)_wXW&~v_6@uTyb-(<yac=${2|x@RsfOv6G6|<q62|W4E+WIStoXk1CqYyI6&IY
z)nUcHUuD#JHzUsD6nH<Fk4ri_np|Q0T|G9O{FWYP*fOBkM}}=qGDnmuW!4pLA;6F~
zQ9);tg>0Y}LQ1H_=ZwBQi4lfpbEZ+_JsQ@zCw<!8vsc*zDzD{}ghGK`*+K=*l9;w(
zS;lvGoYb7j9s+rY>8W!wXul_dDa)<%wZ_(;PW^k^L8zN@(vDTm!@Q+7$7^qPY<1>B
zJ1cka1mb#~tg^WY=`11qF(+d)G&{B$n-{lrzKd6}exmMaGm)f=SVUAptk$d;bly)U
zNJq(%CJ<?Hy|mkVK+O`(iX9a9l7hu6eAs3v4P|aAJxf`$#ER72jt#})(1$U$@mk?Z
z1KuxubuAzcCo4U&)=;Awy_&Zz<;;F<SH|p*U03mAwk5>xY1^N2tK;PlGRt0`JH#}e
z^1>gu{IUawVwZosj1$K=rjA2c@#PIj<`8ClB{fcQxlM7o*!hba%;i2ouUCzQL<oI3
z$<%TL3Q=!|H<}}s74`O9`Qm2O*xIh@Yogr#ZdPxzs!}RL95+-DT{@>z0BrIwrRCIG
zmys<_vtY=w_mt;uk3;V*4$6ZgRK3S;3pZRW!?A<g$TC2r=t=OJ05w+cj(6_p{lvB7
zJUgziMf^G`zXoobkQLN;oAZEZaSRRK{e#!vddy-;%jn|FUgFV-U#V)Faoi$djgBz=
z94#q&lo&}KmrNnO?a^9)*O6q;0SWa|+ELBIJk$tv5$^2?i|Ti1Y<A3pe$Cf66P;~n
zoN)zvkJQq9vtF8xC2qTu+ux$OE@MNC-46p9UbfOig-7J>D~;f`_*anmt`YMR?ZES9
z@eMTHx|rQc)5GXxDR4V&M7>orx*E!YiK5F{kb0S3O1JXi@sU^eizx|CZl_Uu<jGHb
zy(d!Ts|Z?l9EcR-3zN<;!cJ`&aqXyUN)qkg*#eDxN?8tFv5^YX;wp29w;1G6xb$20
zQ}Ir<tiN~^YlQ1RJVkKfWj-}#pNjq8j*^5FI|{|5B&!TpN_au3UpBF#Y3k=hYB?U)
z(u@wDL`7vasI67}FzQ-gZu@m@-ZGqhr;X?^GT1r2XPdCd-PNm%6C>*;&z8%hqv;!B
z=seDKiOh((E7hM@nxlQ19B*pS19n|gPu!jsYhbL;rgm@HZS*}0+jj3leJ_eu&-xwh
zo;Ul|j)<A4Ve$o5>COO%Ij~tSMyLkTBHUV>oujr=$?s{{r+aldnC=Jg(NPknZ_pfB
zsdrtL+<1*@ykd6+{Kl8;u&5f%+85<vp|F1w2VmD01Y2@;vU-^pBU=*)ujIo&C{dZk
zn89Ewf%bX9p<PBK>nN!ii>S_e->b@i<)(B?n5zw>-%3#Y|Bn&nPjtYJ^Z%bl$A24m
zJrMu@#b6RlfR*4uK;8*(e{di0m+1QM2NkdpYyjf_{~0>IoC~-DOo1`*6mSOk7j*mo
z3FO@W^`HTE0C}(96T!oPoDsMaybH)X{p4K%7XW#u-;dGle*|s;F9DZ>N$|ViG2ol%
z_FmtADSwNve+PIfSOfg~10KoWVgvYR@I&x!a20qS*aQ9x_!_$YSHWAsYk-^|m<16i
zf;<=oa_;{j;8W=Ow}Pv|vw@uTe<JuXy8Tyy`1vmZ<KSP=>HiMg4BiNigIVxYAm;<*
z%>S)G-YIYlYy<Kh0Xg^ob!7KB@F8$LXo77(^!g_Nk?q}m&IhEMN2eyBlV7~#k1D+R
zlSLuDA(w8?I_vb9DbuOqML%85XUbHn-XufTyvf-d;kBIcHP`R7T<Jq|OtsVcO9qYj
zGG=Yw`7Xr8mphk3JJaup*2KERcdH-KUyHn9HYA3DkW%E+LkriPqA@vIf4>f}OS7H)
zbnLW+_fe@kC`(sS$Z~4B8&{MEX*!Q)l*oQmCfrF;xf!MjPJ+fHM>`WFG|C;_>e-7z
zk)}O#QC#fQ0BV{n-l<ft1|^xq?I^0~I?q*Mi{x)NYwM5~FJ|rZyU7&e2W7rrY#r({
zP8N*WA&k6!XSY1<)|$O72ErC4!BN&<A?Vk#f>rCXl>|=<l)7EVG#{4Y#!-P`MyD|$
zf0-Ltluxpc$LF2ip7PsZjFIy!_mVYchPS9M+a42-R_PudUru)_y&Mc;_P5j6lB6Sz
z+L5NT&EOJmhvyP~T^gYqJo*huRwF@^7sZssb`)L4I1xUWl&ueHuk9Gsk|bd{ei^J?
zr9<ONXCNBaT%J7%x&-#MN)KI{i*%NL>tOUOUC3&}e(Ryd=R>O*#WLF|9o?9nfn_K?
zW7@S~Y$OY;TvIu-Q`yDift=l-m##ep$XumUl@_Rlv>N#;uSw^fDvOljis&Vi@Tn<Y
zErmveK~~03Eki06COJ8?oaB1X+vOxzS2Gpnz%<4{ed4Au<=uX5lxvL|lG4Kt?La4h
z-(p)?YTWdF$o27feGu{?efq1fu}fMY)1nN6c1n!RUD_^%Z8n8(pIhPtZs?5DEw&Cx
z9VlC_J<MEjg<GDaEoFP{sk6A?d(T{!9=>`H-nQL!uacEviw)x@iEOYZYKSYO=^Rb|
zOEQ&~D>ez;W^Z@0*wM`Fii%MsD%dl2&Dis(<(|Z>;0<@npA_l`7n;*DBR1tQBZlnM
z?~|Aqf>FC;W=Ln<RTND@^#9GMi06o&i2gs%mwY7V-^anLK?Br54UB*s$bcWB+kXUH
z3l0K_2XGdUIDd}>_W^gK<NrJOIQS@dIk*Zu4?Gtf0~dh%f`3QX|6A}Ga1(enD1ohD
z9e6bO6*~Wy!P|h?0wf0DIpA(|{a=Dx!8?KM189M#gA2i_K;r)03f>8>0e;W_M*e<0
z$bes==id%w-~Xq<r@#xr(}C;-_!WBn?cg?WBX|*z7y_%n{lGWT@4p1T2;L4P)?Xg{
zK3ECF_x~Ai9T1<t#11$HE(IgtXXyI>2>uFO0}g{acs6)E_%-_e--EvZ*MeQ(>0l#}
zm;sLmUqO~PgV%$@pagy&h>Ww~q&~79NPl`QD$)(kz0uX%la1~taGAu8o4`fKkXZ<v
zbbYq>CJLftl`b*0loAtxj?TIO%MR(^y7kizKAWWvdjEuav*KQ<gQR3XzJ-xQlsPwS
zj1&qPxsF+PI3r)N)!YvRa#22M!tC-18%A7&_%th*VLsl@SK!^0XJh7w9|^Bk#d|m_
z-hr7))z=Ri<P@+u<q7R1^d9D5r>#9Bga@w4(cmDsGn4<PaZT!!^BZ1&u~}MQeWztF
zX`Z-8{FQyyU`r1lYM_%bL3^W8+-J=ecdEV%Hpzy*6!?8E;620Zjh<ruzq(xee`Gx-
zgQ4IAmFi$MrVgeFtf-8cA`tsuunrqxXR&PVVF6Y&4UGd)tsT(~r145lNHF?e@F1=!
z@vLa3i?8CeQ58AO#0#sNk&DpGl1l7X=|IiO_GD&~1XI~^?(!iqG~2TXsWCPjKM8CS
z?<9$b>Tm;si+o}=X{E{AyVVLjVF~hYW1e!ejV4UgdLb!J>5}NRmXB0YE#<1#Qm8wy
z@Z_m~D$ltVd3k2L!d@Z5RkvjktOG9w!3o*CE9`W4G3T4?#P9^iNmfzsHJVO~Aw=cs
zv7EY*<#2}baKc`cJK|pUeDZv>&y{l;6GxtL4{>FcKHyGy1oKjgZL7zEog41q)P$r3
z#MNi<T(!p;bQbOzuuYzDDi?pT-A@kZxzKISoTH-G%F;MjRZDT*lzMqBx$IO6Z^TTJ
zn|ec1bUzp0>c=c;a%EDv`>sHk6<>`xX4%05Bu^MD)$`X*hQ)ENHcKDGqi3sjuQX38
z-zA!<L^MLB<(eJfO2J_~vBw7P5UGO`OyZ7@4`S)QQGb4ovSqN2Hwa>Z_Ahx9YvQ62
zbR@KT$`TwOabm-u>~lvgl>MNxovkMEMcAro^~dbk*vQz}xg5ci@AK^U`1p_xU3;Ta
z#5w)~@21`nHIGGy%$8{GQ0+*$a4hG}Rmk9_j;(|B)%jw+-eS9atav&<daGe$)X9Qr
zh3<SRx61iSOfdY)ncC2h_rsSs-NAe6<2vj1)0uX;QcNC+|B{zkyPYSSHFn?bI{W*F
zQjUE~01U<vrdw0WQFE%cC*LX@)Uxx%%h|zOwSlL6uhJ#;guA06wSv@xa={b#Shj1a
zMo5UR3brg%^K<1w5X(ml2wI7S;Z5-a)>`ZM?f6G~g5hR`sjc3`ZjLr*)>K1al*q13
z8fI>cW1?wZ_ma`_7@n|rA!M)~8r9CCy-!arN<gY<_MdHZTiZA&NxA#%sV1|Ejl*s$
zNw1O@XA5STCg-e*b}#;?(Z5Cif1Fbe??%5D{r{uj9pL3a-UWCZ90TWr^+5IkJPO>6
zuKz#4C&8`YML=Q!Tm-}>@L=!*%D4-B4`_KOrHBq58gO91fdK~w92jt5z<~h=1{@f0
zV8DR^2maG?py@swPAwNOU6hNxRCufFTXT<EO_Ja|)`W=lETz@E(=$DS%)|yDYnnO9
zjQn<lM6FJd9;?Q^G}}PEm%s+PtxJga@su%5CZV4sgScwe@@kC~s%s}YJ`Z6~u^h5B
zi(ldpur0n8TLYAAEKiMAOlsQB)a%SoK(r->(lyeB(sLL)PI<=dUVyB_o-VOu3Y~|o
zQ8TdrSSq4cyWW?I$uL_?LWdzj^#3yY`=0>O|8ZgKM|A$TgFgi~f*Zi~K-0N2gRcPx
z1{@f0V8DR^2L>D%aA3fJ0S5*g7;s>~fdK~w9JuE=U<`T54lm-Bx8&7*r<?4q$xDWe
eX!^39t5!8L#m0ht&0jymM`Gw0ieLX{GXDps#)W49


From 0a70462ba156626c11896ca318f5e41f6e99929d Mon Sep 17 00:00:00 2001
From: Samiullah Malik <39503305+Samiullah-Malik@users.noreply.github.com>
Date: Thu, 3 Dec 2020 18:43:43 -0500
Subject: [PATCH 288/555] Delete .nfs000000000002a1270000bfa3

---
 ree/.nfs000000000002a1270000bfa3 | Bin 20480 -> 0 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 ree/.nfs000000000002a1270000bfa3

diff --git a/ree/.nfs000000000002a1270000bfa3 b/ree/.nfs000000000002a1270000bfa3
deleted file mode 100644
index fc89809479486b99aeaa5827cead8ad7ceb811ac..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 20480
zcmeI2ZEPGz8ON8jw1p;ZX+9wM0F$dId*^$xU7*Nz45@R5ywxG+6d`EMdUwvZ&Aw%K
z){gB)r2;{PfT~}LAgUB0nj#_bffp5(wu&k~R4qsmNQe(q5efnX-;0o_%Kw>}-Mu?U
zkb)4ZG^hM>zIlD-|IEzpJiDut_Z>RM?rwj`!sj;2I@Nz}<*^I9tiy|z6*_@`Dx0R9
zfB8w{Fc0#qow;$!-FBMzZSLhet8@b7cXX?rlzRh5w8B0QTOMEML7a4`fesJxujg;^
zMV|*=+jV!gWKxU*Mu9h_KrZ^NkM6PVnx2}ZdUw?JvAwsSd{a%B!bSn3fKk9GU=%P4
z7zK<1MggP1TU!B<TxUIuA?(t_IHSMcHuinJ{(YnVerPQJI{o`D{r$*TzW!;xi~>dh
zqkvJsC}0#Y3K#{90!9I&fKk9GU=;X2DByUO^%DC1IRyal{J%K=zkaJ_T?EgA4KM*d
z2=)RCym5<V{R#XATm(;p3*ZUx2uQ(ckbwKaVQ>TZ`yR{sAvg~{1Mo=FdLOt6Jii-l
zfG5CLz!A^^f4SMRo&t}9Z-B3Z)8GMc7_>kQ+y>rwr)9kkeh;1j-vnO(4+9T;9NZ56
z_72N>8GH>q3Nmmv_z2hsZUL{|WLbX%e*jm&cfohSgP;dm;2#KxTn0Y|-vbxGQ{eO9
z0q|b%*BdYf@GI~GunHV73vLFlzTL8Z30?s&gLB|pAOwq`25tnuMiAya$iY$YNzelC
z0=I%&z#i}-f-x7tli-VB2^;{s!DaZFeEl?d6g&hX5Q0T;5X=Dv=)>57b)GtFd?~T%
zos2%clhI(=R(-}w^3}k1XRK|@4V)~iO0pTI@|ctKXJ!_hbzf{QC6;v-?a8E!i=NAv
z8|Q+}&9PcG@^_8WGWw54UNDPF|0X+Qo&AglJmitc+G}VWrrLE?Gh-9`Rj-ucI)Rhg
zyx%O#f7ZXh$tGptew;Ep6W+{Bh6Yvy^Y>5AGM~YaosF}J3BL~8ycjxgmZ2zqg{^sX
zbHIDhabb)NnmX;E5A!lVj_i8Xx$IU-QdB9sqSj>2F`f+?EM(OvC<S9c<!x&{_C4k(
zCoRYK@l^0LwyY=LpJ^}}>+tXO;y1O@jRbsyLXoJ$-Q=Kbt2M8OyFxOcUru_GJ(L%s
z8cIZ-U5FwFK~;ynt8}J{7_ICDDmU3c7A{pApmpeVkY_$#2%^$<wpCP+MK=aM)U`=T
zSc>A64ZgtC6uh#dUT51dqg3sXED)y{pZ4EK{Z+o^N46)sDeDX=g8pdGek7PTQjsjK
zyh&x~QyA7{H4JBq*R<}vg#$+qoTz#!{6eEev$AJ5<rqm%@I>a?aBL<V;k!)w7aexh
z2&j@OLVxwk<*0m@a*?MI8>~cpLz>=aw*0~7rFpAdIE_JF70^RcX``jF{NQOPNCrEK
z>qIKAXTIuSKN~RG7+Ui*_LaWlia4F69b_yG-6rt{+}xTMPKuGO@HCCnPBG`?uO_pn
z+pI(XHxf3%yn0;<crCurW+i%jaq4kpNMSEB(CtE=rkGSDqcn!*^~lWA#YgNyRp~F(
z*INmVLz*nTQZ%4;l+ua1DbyOC=_4N0lVxmCVhUni(%LFqt7aRm8@W-1n87eR<W82S
z+zw%Yx5hJUYDzG<rIBjoRu=Kuv8}6YEXqTk`tC~LhrY<>l-2VyGof=v?WWXFM3Yu+
za|h#%Xv0ad6EvrqvfpZ!7kSDavbxh?slPTr3-Jbn-F}#dGM5EBS`!22XG{z@gUNWI
z&}p~ZtkJ0!LZ#ub=gM`99r9`tYW0da2-D>@Exo2zRI$mG>9inFj=sojDk>*E(&TqJ
zsfVX8GDR$$Kuu<vxl~W>mAK5|QsC7aERy<EIkb!VLME#HbGVJ2mn~^EEtadyHO8)C
zr$k?RkJM@o_Xla(G~iOwYWwsB##fm=ym4O3iY!|k-(0o4LpuigDIHNQE2PRFD{{jt
zydqaNW%cDnw>NMST=nJOOSI_KoGr|*3ep;*ylr$}?qcL_x}28@($`IyBB#CjQw~MS
zkBbVqLYZ`x%)Uw+UF0LXI_`q<q|CRm&P(}q4<rnw;bZ5SAFVNm1u^bniYq?01vkdU
ze-k@^7{ne6b9cZtxO0l9SWHgDddE-HX&uS+N6RbaSsn;#i&|$=(Pq&O_gEkA^U8Ht
zY<yKQG%_;&p#?-%Vu3qM@Gyx}2Nubeu}j4%b>^u{yzDe%j;Jol(l|XIi$$k@>h)Io
zWO&8t9#xX^1zenAPl@|js6o+z%)%X_TAvJi+$yv|89lyCsiS+O!Il@VvEi00PrC$4
z`d$ua$ceysCj8LBa7W}$(3$Q@ms7gLtHSXk=<W1zm%{iN?&i`f<Rd>!aBJ<0al!k=
zps7rh;O;q#15}G6>cbV-ytV9O%~PX`-D6aIoh$ojQtNADIX99x*o@-P$C%XOf=pC(
zXD|11k9yR-Nf(x@99uC~*evFHn!AGaAgLPjHf}K^2bl@(ju{Vex29%wm57VltaXPU
zxk2u6+(?D(K@G2~7726cq&QEs2lG`%%}v+k20-P@;*>g?fcjlEZO0mR(R1Lh4mCOF
z#~1;3!g)(7)?oMa4l?)!xAB}g8BJ+7^;h9TKaw&XJ+#ohx7URzxMkI8B2HQXz2D9}
z+~U$r(PkU%vE>0ScN=kfiY|L`!Xt>Br;sI!`vMnL+&1g3JeEh;&14UlDj|wXub>rb
zXdzBE)pgZgsI%$GsVUZTeY#!lV>T?rA*KoLBr!!!@Qgbt_J;-J^3>`yEIR*R!g=_o
zI6Kq%zj%Is1!w$A;6?BPcouvWd>O2RE|><p!D~3zQ!L;L_yu?goCBL62Xy|w2Yd|7
zgZF@UgI93oe+E1W&Vg@(Gr$9%0CV8|;CiqN{28%;--7etOW+jff(ft}T*4S$0MCJ+
zfoB1Yi@wdrC}0#Y3K#{90!9I&fKk9G@YYq}?Afz0ekdxKZMEs(s?MbW?)6uAf>XD!
zr<%3yu^D*;TclVq9=XtYKpj~=&}SLK2a%V3E~+)fR7<GGAU>kT`;IT1T*2WXT<P(k
zZ`X&ObciXOvnhKSF>n=Qkw-rqheR50%6<_2YU@+4jIXJ<pn6I{=RsBIPCQ2<n-Le)
zhq9e{V?h#BL5Pzc-wXv`ScK3Y1*@AfYA4&QPMz5n^-%=8CdZ85MqZgKLw}vdem32n
zY;<;pgAn4NDB1Lf8=e0yG9(3)A~b09%Q6t&Jys20xi?PQPLc$h%W?I*LHVeBEQhlr
z#Fj&*NLL1NF+$nCdZ=*JSxcR0O-o)x!{{MJC|xu9n4-z1(AMZ<rOK*ORQU*SIABFF
zwejd4tB(FCUqk`O87kOG!k5X)bof<1=aJ(<_+{+5)==r;*+sq}Kd$?!pm>aWCPu>9
z<U`e)4xld`Ap_{66HatLb+X%AR;e`g*}jyY&M}#fQlt$Lc>7LeRC7u$gQ2>;BZrPx
z6{Y9{CsR7uL&nmd<<Qi$Qcc2Da@7u7zmyBCQUn69Dx?_dao&i+#jvOz)0)|ns^zj@
znioTX$s%r2My8QT=l@UR4F4FQ^FJO}%g<%}eii%<`~>_6d>_bix{UcU3K#{90!9I&
rfKk9GU=%P47zK<1MggOMQQ$vP;D7nLwR|C$Uj9YoLB5KcA;|Jiz1fW!


From b17b9cb1aa2f679ae7e1ea24e70d4d1ca0cb5051 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Thu, 3 Dec 2020 19:53:50 -0500
Subject: [PATCH 289/555] changes CMake em to ree

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index e04f5cf0d..ccd8a0382 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -136,7 +136,7 @@ add_subdirectory(pumi)
 add_subdirectory(ma)
 add_subdirectory(crv)
 add_subdirectory(spr)
-add_subdirectory(em)
+add_subdirectory(ree)
 add_subdirectory(sam)
 add_subdirectory(phasta)
 add_subdirectory(stk)

From d56aff1d6b278349084e8dc69f20bcc6fa15b076 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Thu, 3 Dec 2020 19:58:19 -0500
Subject: [PATCH 290/555] Deletes em subdirectory

---
 em/CMakeLists.txt           |  32 --
 em/cmake/Dependencies.cmake |   3 -
 em/em.h                     |  75 ---
 em/emCorrectedFlux.cc       | 299 ------------
 em/emEstimateError.cc       | 458 ------------------
 em/emFluxCorrection.cc      |  73 ---
 em/emResidualFunctionals.cc | 918 ------------------------------------
 em/emSizeField.cc           | 173 -------
 em/pkg_tribits.cmake        |  19 -
 9 files changed, 2050 deletions(-)
 delete mode 100644 em/CMakeLists.txt
 delete mode 100644 em/cmake/Dependencies.cmake
 delete mode 100644 em/em.h
 delete mode 100644 em/emCorrectedFlux.cc
 delete mode 100644 em/emEstimateError.cc
 delete mode 100644 em/emFluxCorrection.cc
 delete mode 100644 em/emResidualFunctionals.cc
 delete mode 100644 em/emSizeField.cc
 delete mode 100644 em/pkg_tribits.cmake

diff --git a/em/CMakeLists.txt b/em/CMakeLists.txt
deleted file mode 100644
index 4d4adeedb..000000000
--- a/em/CMakeLists.txt
+++ /dev/null
@@ -1,32 +0,0 @@
-if(DEFINED TRIBITS_PACKAGE)
-  include(pkg_tribits.cmake)
-  return()
-endif()
-
-# Package sources
-set(SOURCES
-  emResidualFunctionals.cc
-  emFluxCorrection.cc
-  emCorrectedFlux.cc
-  emEstimateError.cc
-  emSizeField.cc)
-
-# Package headers
-set(HEADERS
-  em.h)
-
-# Add the em library
-add_library(em ${SOURCES})
-
-# Include directories
-target_include_directories(em INTERFACE
-    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-    $<INSTALL_INTERFACE:include>
-    )
-
-# Link this package to these libraries
-target_link_libraries(em PUBLIC apf pcu crv)
-
-scorec_export_library(em)
-
-bob_end_subdir()
diff --git a/em/cmake/Dependencies.cmake b/em/cmake/Dependencies.cmake
deleted file mode 100644
index 9d9456a31..000000000
--- a/em/cmake/Dependencies.cmake
+++ /dev/null
@@ -1,3 +0,0 @@
-TRIBITS_PACKAGE_DEFINE_DEPENDENCIES(
-  LIB_REQUIRED_PACKAGES SCORECapf
-  )
diff --git a/em/em.h b/em/em.h
deleted file mode 100644
index a64c1b578..000000000
--- a/em/em.h
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Copyright (C) 2011 Scientific Computation Research Center
- *
- * This work is open source software, licensed under the terms of the
- * BSD license as described in the LICENSE file in the top-level directory.
- */
-
-#ifndef EM_H
-#define EM_H
-
-
-#include "apf.h"
-#include <apfMesh.h>
-#include <PCU.h>
-#include <lionPrint.h>
-#include "apfShape.h"
-#include "apfField.h"
-#include <mthQR.h>
-#include <mth.h>
-#include <mth_def.h>
-
-namespace em {
-
-/*
- * Computes nodal size field
- */
-apf::Field* getTargetEMSizeField(
-    apf::Field* ef,
-    apf::Field* error_field,
-    int n,
-    double alpha = 0.25,
-    double beta = 2.0);
-/*
- * Takes the solution electric field and computes edge equilibrations.
- */
-apf::Field* equilibrateResiduals(apf::Field* f);
-
-/*
- * Takes the solution electric field and equilibrated field (of face vectors)
- * and computes the 'correction' to the flux vectors on each face.
- */
-apf::Field* computeFluxCorrection(apf::Field* ef, apf::Field* g);
-
-/* Takes the solution electric field and correctiion to the flux vectors on
- * each face and computes the 'corrected' flux vectors on each face
- */
-apf::Field* computeCorrectedFlux(apf::Field* ef, apf::Field* theta);
-
-/* Takes the solution electric field and corrected flux field and solves
- * local element level BVPs to estimate the error.
- * Returns a per-element scalar error field.
- */
-apf::Field* computeErrorField(apf::Field* ef, apf::Field* correctedFlux);
-
-apf::Field* estimateError(apf::Field* f);
-
-
-void assembleCurlCurlElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
-    apf::Field* f, mth::Matrix<double>& elmat);
-
-void assembleVectorMassElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
-    apf::Field* f, mth::Matrix<double>& elmat);
-
-void assembleElementMatrix(apf::Mesh* mesh, apf::MeshEntity*e,
-    apf::Field* f, mth::Matrix<double>& elmat);
-
-void assembleDomainLFElementVector(apf::Mesh* mesh, apf::MeshEntity* e,
-    apf::Field* f, mth::Vector<double>& elvect);
-
-apf::Vector3 computeFaceOutwardNormal(apf::Mesh* m,
-    apf::MeshEntity* t, apf::MeshEntity* f, apf::Vector3 const& p);
-
-
-}
-#endif
diff --git a/em/emCorrectedFlux.cc b/em/emCorrectedFlux.cc
deleted file mode 100644
index d5f0b4082..000000000
--- a/em/emCorrectedFlux.cc
+++ /dev/null
@@ -1,299 +0,0 @@
-/*
- * Copyright (C) 2011 Scientific Computation Research Center
- *
- * This work is open source software, licensed under the terms of the
- * BSD license as described in the LICENSE file in the top-level directory.
- */
-#include <apfCavityOp.h>
-#include "apfElement.h"
-#include "crv.h"
-#include "crvShape.h"
-#include "em.h"
-
-namespace em {
-
-enum {VISITED};
-
-/* overall information useful during computation of corrected flux */
-struct CorrectFlux
-{
-  apf::Mesh* mesh;
-  /* mesh dimension, so far handling 3 only */
-  int dim;
-  /* polynomial order of nedelec space */
-  int order;
-  /* polynomial order of p+1 nedelec space */
-  int orderp1;
-  /* input nedelec field containing scalar dofs for electric field */
-  apf::Field* ef;
-  /* input theta field for flux correction
-   * (3 scalar dofs on each face, 1 per edge) */
-  apf::Field* theta;
-  /* tags each face once it has been visited */
-  apf::MeshTag* tag;
-  /* output per-element field containing correctedFlux vectors,
-   * which are computed using theta and electric fields */
-  apf::Field* correctedFlux;
-};
-
-static void setupCorrectFlux(
-    CorrectFlux* cf,
-    apf::Field* f,
-    apf::Field* theta,
-    apf::Field* correctedFlux)
-{
-  cf->mesh = apf::getMesh(f);
-  cf->dim = cf->mesh->getDimension();
-  cf->order = f->getShape()->getOrder();
-  cf->orderp1 = cf->order+1;
-  cf->ef = f;
-  cf->theta = theta;
-  cf->tag = cf->mesh->createIntTag("isVisited", 1);
-
-  cf->correctedFlux = correctedFlux;
-  int nc = apf::countComponents(cf->correctedFlux);
-  double zeros[nc];
-  for (int i = 0; i < nc; i++)
-    zeros[i] = 0.;
-  apf::MeshEntity* tet;
-  apf::MeshIterator* it = cf->mesh->begin(3);
-  while ((tet = cf->mesh->iterate(it))) {
-    apf::MeshElement* me = apf::createMeshElement(cf->mesh, tet);
-    int orderp1 = cf->order+1;
-    int np = apf::countIntPoints(me, 2*orderp1-1);
-
-    for (int i = 0; i < np; i++) {
-      apf::setComponents(cf->correctedFlux, tet, i, zeros);
-    }
-  }
-  cf->mesh->end(it);
-}
-
-typedef std::vector<apf::MeshEntity*> EntityVector;
-struct FaceCavity
-{
-  apf::Mesh* mesh;
-  apf::MeshEntity* entity;
-  CorrectFlux* correctflux;
-  EntityVector tets;
-};
-
-static void setupFaceCavity(FaceCavity* fc, CorrectFlux* cf)
-{
-  fc->mesh = cf->mesh;
-  fc->correctflux = cf;
-  fc->entity = 0;
-}
-
-static void startFaceCavity(FaceCavity* fc, apf::MeshEntity* f)
-{
-  fc->entity = f;
-  fc->tets.clear();
-}
-
-static void addEntityToCavity(FaceCavity* fc, apf::MeshEntity* e)
-{
-  PCU_ALWAYS_ASSERT(fc->mesh->getType(e) == apf::Mesh::TET);
-  fc->tets.push_back(e);
-}
-
-static void addEntitiesToCavity(
-    FaceCavity* fc, apf::DynamicArray<apf::MeshEntity*>& es)
-{
-  for (std::size_t i=0; i < es.getSize(); ++i)
-    addEntityToCavity(fc, es[i]);
-}
-
-static bool getInitialFaceCavity(FaceCavity* fc, apf::CavityOp* o)
-{
-  if (! o->requestLocality(&fc->entity, 1))
-    return false;
-
-  apf::DynamicArray<apf::MeshEntity*> adjacent;
-  fc->mesh->getAdjacent(fc->entity, 3, adjacent);
-  addEntitiesToCavity(fc, adjacent);
-  return true;
-}
-
-static bool buildFaceCavity(FaceCavity* fc, apf::CavityOp* o)
-{
-  if (!getInitialFaceCavity(fc, o)) return false;
-  return true;
-}
-
-
-static void computeCorrectedFlux(FaceCavity* fc)
-{
-  apf::MeshEntity* face = fc->entity;
-
-  // 1. get upward tets of the face
-  apf::Up up;
-  fc->mesh->getUp(face, up);
-  if (crv::isBoundaryEntity(fc->mesh, face))
-    PCU_ALWAYS_ASSERT(up.n == 1);
-  else
-    PCU_ALWAYS_ASSERT(up.n == 2);
-
-  apf::MeshEntity* firstTet = up.e[0];
-  apf::MeshEntity* secondTet = nullptr;
-  if (up.n == 2)
-    secondTet  = up.e[1];
-
-  // 2. get positions of the face in downward faces of upward tets
-  apf::Downward tet1_faces, tet2_faces;
-  int nf = fc->mesh->getDownward(firstTet, 2, tet1_faces);
-  if (up.n == 2)
-    fc->mesh->getDownward(secondTet, 2, tet2_faces);
-  int tet1_pos = -1;
-  int tet2_pos = -1;
-  tet1_pos = apf::findIn(tet1_faces, nf, face);
-  if (up.n == 2)
-    tet2_pos = apf::findIn(tet2_faces, nf, face);
-
-  // 3. get downward edges of the face
-  apf::Downward edges;
-  int nedges = fc->mesh->getDownward(face, 1, edges);
-
-  // 4. get theta coeffs on the face
-  double components[3];
-  apf::getComponents(fc->correctflux->theta, face, 0, components);
-  mth::Vector<double> theta_coeffs(3);
-  theta_coeffs(0) = components[0];
-  theta_coeffs(1) = components[1];
-  theta_coeffs(2) = components[2];
-
-
-  // 5. Evaluate and save corrected flux vector in an auxiliary field
-  int ftype = fc->mesh->getType(face);
-  PCU_ALWAYS_ASSERT(ftype == apf::Mesh::TRIANGLE);
-  int nfdofs = apf::countElementNodes(fc->correctflux->ef->getShape(), ftype);
-  apf::NewArray<apf::Vector3> vectorshapes(nfdofs);
-
-  apf::MeshElement* fme = apf::createMeshElement(fc->mesh, face);
-  apf::Element* fel = apf::createElement(fc->correctflux->ef, fme);
-  int int_order = 2*fc->correctflux->orderp1-1;
-  int np = apf::countIntPoints(fme, int_order);
-  int nc = apf::countComponents(fc->correctflux->correctedFlux);
-
-  apf::Vector3 p, tet1xi, tet2xi, curl1, curl2, curl,
-      fnormal1, fnormal2, tk, tk1, tk2, vshape;
-  for (int n = 0; n < np; n++) {
-
-    apf::getIntPoint(fme, int_order, n, p);
-
-    // evaluate theta vector using theta coeffs
-    apf::Vector3 theta_vector, theta_vector1, theta_vector2;
-    theta_vector.zero(); theta_vector1.zero(); theta_vector2.zero();
-    apf::NewArray<apf::Vector3> triVectorShapes (nfdofs);
-    apf::getVectorShapeValues(fel, p, triVectorShapes);
-    for (int i = 0; i < nedges; i++) {
-      apf::Vector3 v = triVectorShapes[i];
-      v = v * theta_coeffs[i];
-      theta_vector += v;
-    }
-
-    // orient face outward normals wrt tets and theta vectors
-    fnormal1 = computeFaceOutwardNormal(fc->mesh, firstTet, face, p);
-    fnormal2 = apf::Vector3(0.,0.,0.);
-    theta_vector1 = theta_vector;
-    if (up.n == 2) {
-      fnormal2 = computeFaceOutwardNormal(fc->mesh, secondTet, face, p);
-      theta_vector2 = theta_vector * -1.;
-    }
-
-    curl.zero();
-    // compute curl1
-    tet1xi = apf::boundaryToElementXi(fc->mesh, face, firstTet, p);
-    apf::MeshElement* me1 = apf::createMeshElement(fc->mesh, firstTet);
-    apf::Element* el1 = apf::createElement(fc->correctflux->ef, me1);
-    apf::getCurl(el1, tet1xi, curl1);
-    apf::Vector3 temp1 = apf::cross(fnormal1, curl1);
-    curl += temp1;
-    apf::destroyElement(el1);
-    apf::destroyMeshElement(me1);
-
-    // compute curl2
-    if (up.n == 2) {
-      tet2xi = apf::boundaryToElementXi(fc->mesh, face, secondTet, p);
-      apf::MeshElement* me2 = apf::createMeshElement(fc->mesh, secondTet);
-      apf::Element* el2 = apf::createElement(fc->correctflux->ef, me2);
-      apf::getCurl(el2, tet2xi, curl2);
-      apf::Vector3 temp2 = apf::cross(fnormal2, curl2);
-      curl += (temp2 * -1.);
-      curl = curl * 1./2.;
-      apf::destroyElement(el2);
-      apf::destroyMeshElement(me2);
-    }
-
-    tk = curl;
-    tk1 = tk;
-    apf::Vector3 theta_plus_tk1 = theta_vector1 + tk1;
-
-    // get and set components in the auxiliary field
-    double comp1[nc];
-    apf::getComponents(fc->correctflux->correctedFlux, firstTet, n, comp1);
-    int id1 = tet1_pos * 3;
-    comp1[id1] = theta_plus_tk1[0];
-    comp1[id1+1] = theta_plus_tk1[1];
-    comp1[id1+2] = theta_plus_tk1[2];
-    apf::setComponents(fc->correctflux->correctedFlux, firstTet, n, comp1);
-
-    if (up.n == 2) {
-      tk2 = tk * -1.;
-      apf::Vector3 theta_plus_tk2 = theta_vector2 + tk2;
-      double comp2[nc];
-      apf::getComponents(fc->correctflux->correctedFlux, secondTet, n, comp2);
-      int id2 = tet2_pos * 3;
-      comp2[id2] = theta_plus_tk2[0];
-      comp2[id2+1] = theta_plus_tk2[1];
-      comp2[id2+2] = theta_plus_tk2[2];
-      apf::setComponents(fc->correctflux->correctedFlux, secondTet, n, comp2);
-    }
-  }
-}
-
-class FaceCavityOp : public apf::CavityOp
-{
-public:
-  FaceCavityOp(CorrectFlux* cf):
-    apf::CavityOp(cf->mesh)
-  {
-    setupFaceCavity(&face_cavity, cf);
-  }
-  virtual Outcome setEntity(apf::MeshEntity* e)
-  {
-    if (face_cavity.mesh->hasTag(e, face_cavity.correctflux->tag))
-      return SKIP;
-    startFaceCavity(&face_cavity, e);
-    if ( ! buildFaceCavity(&face_cavity, this)) {
-      return REQUEST;
-    }
-    return OK;
-  }
-  virtual void apply()
-  {
-    computeCorrectedFlux(&face_cavity);
-    int n = VISITED;
-    face_cavity.mesh->setIntTag(
-        face_cavity.entity, face_cavity.correctflux->tag, &n);
-  }
-  FaceCavity face_cavity;
-};
-
-apf::Field* computeCorrectedFlux(apf::Field* ef, apf::Field* theta)
-{
-  int dim = apf::getMesh(ef)->getDimension();
-  int order = ef->getShape()->getOrder() + 1; // local BVPs require p+1
-  int int_order = 2*order-1;
-  int nc = 4*3; // 1 flux vector per face
-  apf::Field* correctedFlux = createPackedField(
-      apf::getMesh(ef), "correctedFlux", nc, apf::getIPShape(dim, int_order));
-
-  CorrectFlux correctflux;
-  setupCorrectFlux(&correctflux, ef, theta, correctedFlux);
-  FaceCavityOp op (&correctflux);
-  op.applyToDimension(2);
-  return correctedFlux;
-
-}
-}
diff --git a/em/emEstimateError.cc b/em/emEstimateError.cc
deleted file mode 100644
index 412c97522..000000000
--- a/em/emEstimateError.cc
+++ /dev/null
@@ -1,458 +0,0 @@
-/*
- * Copyright (C) 2011 Scientific Computation Research Center
- *
- * This work is open source software, licensed under the terms of the
- * BSD license as described in the LICENSE file in the top-level directory.
- */
-
-#include <apfCavityOp.h>
-#include "apfElement.h"
-#include "crv.h"
-#include "crvShape.h"
-#include "em.h"
-
-namespace em {
-
-static void computeResidualBLF(apf::Mesh* mesh, apf::MeshEntity* e,
-  apf::Field* f, apf::Field* fp1, mth::Vector<double>& blf)
-{
-  apf::FieldShape* fp1s = fp1->getShape();
-  int type = mesh->getType(e);
-  PCU_ALWAYS_ASSERT(type == apf::Mesh::TET);
-  int nd = apf::countElementNodes(fp1s, type);
-  int dim = apf::getDimension(mesh, e);
-  double w;
-
-  apf::NewArray<apf::Vector3> curlshape(nd);
-  apf::NewArray<apf::Vector3> vectorshape(nd);
-  mth::Matrix<double> phys_curlshape(nd, dim);
-  mth::Matrix<double> vectorShape(nd, dim);
-  mth::Vector<double> curlcurl_vec (nd);
-  mth::Vector<double> mass_vec (nd);
-  blf.resize(nd);
-
-  apf::MeshElement* me = apf::createMeshElement(mesh, e);
-  apf::Element* fp1el = apf::createElement(fp1, me);
-  apf::Element* fel = apf::createElement(f, me);
-  int int_order = 2 * fp1s->getOrder();
-  int np = apf::countIntPoints(me, int_order);
-
-  // 1. Compute Curl Curl Integration
-  curlcurl_vec.zero();
-  apf::Vector3 p, curl;
-  for (int i = 0; i < np; i++) {
-    apf::getIntPoint(me, int_order, i, p);
-    double weight = apf::getIntWeight(me, int_order, i);
-    apf::Matrix3x3 J;
-    apf::getJacobian(me, p, J);
-    w = weight;
-
-    apf::getCurl(fel, p, curl);
-
-    // get curlshape values
-    fp1el->getShape()->getLocalVectorCurls(mesh, e, p, curlshape);
-    phys_curlshape.zero();
-    for (int i = 0; i < nd; i++)
-      for (int j = 0; j < dim; j++)
-        for (int k = 0; k < dim; k++)
-          phys_curlshape(i,j) += curlshape[i][k] * J[k][j];
-
-    // multiply
-    mth::Vector<double> V (nd);
-    V.zero();
-    mth::Vector<double> c (dim);
-    c(0) = curl[0]; c(1) = curl[1]; c(2) = curl[2];
-    mth::multiply(phys_curlshape, c, V);
-    V *= w;
-
-    curlcurl_vec += V;
-  }
-
-  // 2. Compute Vector Mass Integration
-  int_order = 2 * fp1s->getOrder();
-  np = apf::countIntPoints(me, int_order);
-
-  mass_vec.zero();
-  apf::Vector3 vvalue;
-  for (int i = 0; i < np; i++) {
-    apf::getIntPoint(me, int_order, i, p);
-    double weight = apf::getIntWeight(me, int_order, i);
-    apf::Matrix3x3 J;
-    apf::getJacobian(me, p, J);
-    double jdet = apf::getJacobianDeterminant(J, dim);
-
-    w = weight * jdet;
-
-    apf::getVector(fel, p, vvalue);
-
-    apf::getVectorShapeValues(fp1el, p, vectorshape);
-    vectorShape.zero();
-    for (int i = 0; i < nd; i++)
-      for (int j = 0; j < dim; j++)
-        vectorShape(i,j) = vectorshape[i][j];
-
-
-    mth::Vector<double> V (nd);
-    V.zero();
-    mth::Vector<double> v (dim);
-    v(0) = vvalue[0]; v(1) = vvalue[1]; v(2) = vvalue[2];
-    mth::multiply(vectorShape, v, V);
-    V *= w;
-
-    mass_vec += V;
-  }
-
-  // 3. get result
-  blf.zero();
-  blf += curlcurl_vec;
-  blf += mass_vec;
-
-  apf::destroyMeshElement(me);
-  apf::destroyElement(fp1el);
-  apf::destroyElement(fel);
-}
-
-static void computeLambdaVector(
-    apf::Mesh* mesh,
-    apf::MeshEntity* e,
-    apf::Field* f,
-    apf::Field* fp1,
-    apf::Field* flux_field,
-    mth::Vector<double>& lambda)
-{
-  apf::FieldShape* fp1s = fp1->getShape();
-  int order = fp1s->getOrder();
-  int etype = mesh->getType(e);
-  PCU_ALWAYS_ASSERT(etype == apf::Mesh::TET);
-  int nedofs = apf::countElementNodes(fp1s, etype);
-  lambda.resize(nedofs);
-
-  int nc = apf::countComponents(flux_field);
-
-  // get the downward faces of the element
-  apf::Downward faces;
-  int nf = mesh->getDownward(e, 2, faces);
-
-  lambda.zero();
-  // assemble lambda vector LOOP OVER DOWNWARD FACES
-  for (int ii = 0; ii < nf; ii++) {
-    apf::MeshEntity* face = faces[ii];
-
-    int ftype = mesh->getType(face);
-    PCU_ALWAYS_ASSERT(ftype == apf::Mesh::TRIANGLE);
-    int nfdofs = apf::countElementNodes(f->getShape(), ftype);
-    apf::NewArray<apf::Vector3> vectorshape(nfdofs);
-
-    apf::MeshElement* fme = apf::createMeshElement(mesh, face);
-    int np = apf::countIntPoints(fme, 2*order-1);
-
-    // 4. Compute integral on the face
-    apf::Vector3 p;
-    for (int n = 0; n < np; n++) {
-
-      apf::getIntPoint(fme, 2*order-1, n, p);
-      double weight = apf::getIntWeight(fme, 2*order-1, n);
-      apf::Matrix3x3 fJ;
-      apf::getJacobian(fme, p, fJ);
-      double jdet = apf::getJacobianDeterminant(
-          fJ, apf::getDimension(mesh, face));
-
-      // obtain corrected flux vector
-      double comp[nc];
-      apf::getComponents(flux_field, e, n, comp);
-      apf::Vector3 theta_plus_tk;
-      int index = ii*3;
-      theta_plus_tk[0] = comp[index];
-      theta_plus_tk[1] = comp[index+1];
-      theta_plus_tk[2] = comp[index+2];
-
-      // compute p+1 order 3D vector shapes
-      apf::NewArray<apf::Vector3> tetVectorShapes (nedofs);
-      apf::MeshElement* me = apf::createMeshElement(mesh, e);
-      apf::Element* el = apf::createElement(fp1, me);
-      apf::Vector3 tetxi = apf::boundaryToElementXi(mesh, face, e, p);
-      apf::getVectorShapeValues(el, tetxi, tetVectorShapes);
-
-      apf::destroyElement(el);
-      apf::destroyMeshElement(me);
-
-      // compute integral
-      double w = weight * jdet;
-      theta_plus_tk = theta_plus_tk * w;
-
-      // matrix vector multiplication
-      for (int i = 0; i < nedofs; i++)
-        lambda(i) += theta_plus_tk * tetVectorShapes[i];
-
-    } // end integral loop
-    apf::destroyMeshElement(fme);
-  } // end face loop
-}
-
-static void getEssentialElementNDDofs(apf::Mesh* mesh, apf::MeshEntity* e,
-    apf::Field* f, apf::NewArray<int>& ess_dofs, apf::NewArray<int>& uness_dofs)
-{
-  apf::FieldShape* fs = f->getShape();
-  int etype = mesh->getType(e);
-  PCU_ALWAYS_ASSERT(etype == apf::Mesh::TET);
-  int nedofs = apf::countElementNodes(fs, etype);
-
-  apf::NewArray<int> marker_list (nedofs);
-  for (int i = 0; i < nedofs; i++)
-    marker_list[i] = 0;
-
-  // populate marker list by iterating over downward edges and faces
-  apf::Downward edges;
-  int nedges = mesh->getDownward(e, 1, edges);
-  PCU_ALWAYS_ASSERT(nedges == 6);
-  for (int i = 0; i < nedges; i++) {
-    int nodesOnEdge = fs->countNodesOn(mesh->getType(edges[i]));
-    if ( crv::isBoundaryEntity(mesh, edges[i]) ) {
-      for (int n = 0; n < nodesOnEdge; n++) {
-        marker_list[(i*nodesOnEdge)+n] = -1;
-      }
-    }
-  }
-  int nodesOnEdges = fs->countNodesOn(apf::Mesh::EDGE) * nedges;
-
-  apf::Downward faces;
-  int nfaces = mesh->getDownward(e, 2, faces);
-  PCU_ALWAYS_ASSERT(nfaces == 4);
-  for (int i = 0; i < nfaces; i++) {
-    int nodesOnFace = fs->countNodesOn(mesh->getType(faces[i]));
-    if ( crv::isBoundaryEntity(mesh, faces[i]) ) {
-      for (int n = 0; n < nodesOnFace; n++) {
-        marker_list[(nodesOnEdges + i*nodesOnFace)+n] = -1;
-      }
-    }
-  }
-
-  int num_marked = 0;
-  for (int i = 0; i < nedofs; i++) {
-    if (marker_list[i])
-      num_marked++;
-  }
-
-  // use marker list to get ess_dofs list
-  ess_dofs.allocated() ? ess_dofs.resize(num_marked)
-    : ess_dofs.allocate(num_marked);
-  uness_dofs.allocated() ? uness_dofs.resize(nedofs-num_marked)
-    : uness_dofs.allocate(nedofs-num_marked);
-  int ess_dof_counter = 0;
-  int uness_dof_counter = 0;
-  for (int i = 0; i < nedofs; i++) {
-    if(marker_list[i])
-      ess_dofs[ess_dof_counter++] = i;
-    else
-      uness_dofs[uness_dof_counter++] = i;
-  }
-}
-
-/**
- * Inputs: Matrix A, Vector X, Vector B, essential dofs, other dofs.
- * Output: reduced matrix A, reduced rhs B.
- */
-static void eliminateDBCs(
-    mth::Matrix<double> const &A,
-    mth::Vector<double> const &X,
-    mth::Vector<double> const &B,
-    apf::NewArray<int>  const &ess_dofs,
-    apf::NewArray<int>  const &uness_dofs,
-    mth::Matrix<double> &Anew,
-    mth::Vector<double> &Bnew)
-{
-  int num_ess_dofs = ess_dofs.size();
-  int num_uness_dofs = uness_dofs.size();
-
-  // 1. Remove rows of A corresponding to
-	// ess_dofs by copying into Anew
-  Anew.resize(num_uness_dofs, num_uness_dofs);
-	for(int rr = 0; rr < num_uness_dofs; rr++) {
-    int i = uness_dofs[rr];
-    for(int cc = 0; cc < num_uness_dofs; cc++) {
-      int j = uness_dofs[cc];
-      Anew(rr,cc) = A(i,j);
-    }
-  }
-
-	// 2. Assemble new B
-	Bnew.resize(num_uness_dofs);
-  for(int i = 0; i < num_uness_dofs; i++) {
-    Bnew(i) = B(uness_dofs[i]);
-  }
-
-  if (num_ess_dofs > 0) {
-	  // 3. Subtract from Bnew: (Bnew -= Ae*Xe)
-    mth::Matrix<double> Ae(num_uness_dofs, num_ess_dofs);
-    for(int rr = 0; rr < num_uness_dofs; rr++) {
-      int i = uness_dofs[rr];
-      for(int cc = 0; cc < num_ess_dofs; cc++) {
-        int j = ess_dofs[cc];
-        Ae(rr,cc) = A(i,j);
-      }
-    }
-
-    mth::Vector<double> Xe(num_ess_dofs);
-    for(int i = 0; i < num_ess_dofs; i++) {
-      Xe(i) = X(ess_dofs[i]);
-    }
-
-    mth::Vector<double> temp;
-    mth::multiply(Ae, Xe, temp);
-    Bnew -= temp;
-  }
-}
-
-static double computeL2Error(apf::Mesh* mesh, apf::MeshEntity* e,
-	apf::Field* f, mth::Vector<double> const error_dofs)
-{
-	double error = 0.0;
-
-  apf::FieldShape* fs = f->getShape();
-  int type = mesh->getType(e);
-	PCU_ALWAYS_ASSERT(type == apf::Mesh::TET);
-  int nd = apf::countElementNodes(fs, type);
-  int dim = apf::getDimension(mesh, e);
-  double w;
-
-  apf::MeshElement* me = apf::createMeshElement(mesh, e);
-  apf::Element* el = apf::createElement(f, me);
-  int order = 2 * fs->getOrder();
-  int np = apf::countIntPoints(me, order);
-
-  apf::NewArray<apf::Vector3> vectorshape(nd);
-
-  apf::Vector3 p;
-  for (int i = 0; i < np; i++) {
-    apf::getIntPoint(me, order, i, p);
-    double weight = apf::getIntWeight(me, order, i);
-    apf::Matrix3x3 J;
-    apf::getJacobian(me, p, J);
-    double jdet = apf::getJacobianDeterminant(J, dim);
-    w = weight * jdet;
-
-    apf::getVectorShapeValues(el, p, vectorshape);
-    mth::Matrix<double> vectorShape (nd, dim);
-    for (int j = 0; j < nd; j++)
-      for (int k = 0; k < dim; k++)
-        vectorShape(j,k) = vectorshape[j][k];
-
-    mth::Matrix<double> vectorShapeT (dim, nd);
-    mth::transpose(vectorShape, vectorShapeT);
-
-		mth::Vector<double> err_func;
-		mth::multiply(vectorShapeT, error_dofs, err_func);
-
-		error += w * (err_func * err_func);
-	}
-	if (error < 0.0)
-		error = -error;
-
-	return sqrt(error);
-}
-
-apf::Field* computeErrorField(apf::Field* ef, apf::Field* correctedFlux)
-{
-
-	// 1. Create per-element SCALAR error field
-  apf::Field* error_field = apf::createIPField(
-		apf::getMesh(ef), "residual_error_field", apf::SCALAR, 1);
-
-  // 2. Create p+1 order tet ND field
-  int order = ef->getShape()->getOrder();
-  int orderp1 = order+1;
-  apf::Field* efp1 = apf::createField(apf::getMesh(ef),
-	 "orderp1_nedelec_field", apf::SCALAR, apf::getNedelec(orderp1));
-  apf::zeroField(efp1);
-
-  // 2. iterate over all elements of the mesh
-  apf::MeshEntity* el;
-  apf::MeshIterator* it = apf::getMesh(ef)->begin(3);
-  while ((el = apf::getMesh(ef)->iterate(it))) {
-
-    // 2(a). Assemble LHS element matrix
-    mth::Matrix<double> A;
-    assembleElementMatrix( apf::getMesh(ef), el, efp1, A);
-
-    // 2(b). Compute Bilinear Form Vector
-    mth::Vector<double> blf;
-    computeResidualBLF(apf::getMesh(efp1), el, ef, efp1, blf);
-
-    // 2(c). Compute Linear Form Vector
-    mth::Vector<double> lf;
-    assembleDomainLFElementVector(apf::getMesh(efp1), el, efp1, lf);
-
-    // 2(d). Compute Lambda Vector
-    mth::Vector<double> lambda;
-    computeLambdaVector(
-        apf::getMesh(ef), el, ef, efp1, correctedFlux, lambda);
-
-    // 2(e). Assemble RHS element vector = blf - lf - lambda
-    mth::Vector<double> B(blf.size());
-    B.zero();
-    B += blf; B -= lf; B -= lambda;
-
-    // 2(f). Get List of Essential Dofs
-    apf::NewArray<int> ess_dofs, uness_dofs;
-    getEssentialElementNDDofs(
-        apf::getMesh(efp1), el, efp1, ess_dofs, uness_dofs);
-
-    // 2(g). eliminate Dirichlet (Essential) Boundary Conditions
-		mth::Vector<double> X, Bnew;
-    mth::Matrix<double> Anew;
-		X.resize(B.size());
-		X.zero(); // initialize X with exact DBC (e = 0.0)
-		eliminateDBCs(A, X, B, ess_dofs, uness_dofs, Anew, Bnew);
-
-		// 2(h). Solve the reduced system
-	  mth::Matrix<double> Q, R;
-		mth::decomposeQR(Anew, Q, R);
-		mth::Vector<double> Xnew;
-		mth::solveFromQR(Q, R, Bnew, Xnew);
-
-		// 2(i). Recover the solution
-		mth::Vector<double> error_dofs(B.size());
-    for(unsigned int i = 0; i < ess_dofs.size(); i++) {
-    	int index = ess_dofs[i];
-      error_dofs(index) = X(index);
-    }
-    for(unsigned int i = 0; i < uness_dofs.size(); i++) {
-      int index = uness_dofs[i];
-      error_dofs(index) = Xnew(i);
-    }
-
-		// 2(j). Compute L2 Norm Error
-		double l2_error = computeL2Error(apf::getMesh(ef), el, efp1, error_dofs);
-		apf::setScalar(error_field, el, 0, l2_error);
-  }
-  apf::getMesh(ef)->end(it);
-	apf::destroyField(efp1);
-
-	return error_field;
-}
-
-apf::Field* estimateError(apf::Field* f)
-{
-  double t0 = PCU_Time();
-  apf::Field* g = em::equilibrateResiduals(f);
-  lion_eprint(1,"1/4: residuals equilibrated \n");
-  apf::Field* theta = em::computeFluxCorrection(f, g);
-  lion_eprint(1,"2/4: flux corrections computed \n");
-  apf::destroyField(g);
-  PCU_Barrier();
-
-  apf::Field* correctedFlux = em::computeCorrectedFlux(f, theta);
-  lion_eprint(1,"3/4: corrected flux field computed\n");
-  apf::destroyField(theta);
-
-  apf::Field* error_field = em::computeErrorField(f, correctedFlux);
-  lion_eprint(1,"4/4: error computed \n");
-  apf::destroyField(correctedFlux);
-
-  double t1 = PCU_Time();
-  if (!PCU_Comm_Self())
-    lion_eprint(1,"EM: Error estimated in %f seconds\n",t1-t0);
-
-  return error_field;
-}
-}
diff --git a/em/emFluxCorrection.cc b/em/emFluxCorrection.cc
deleted file mode 100644
index 1dac4b02a..000000000
--- a/em/emFluxCorrection.cc
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Copyright (C) 2011 Scientific Computation Research Center
- *
- * This work is open source software, licensed under the terms of the
- * BSD license as described in the LICENSE file in the top-level directory.
- */
-#include "crv.h"
-#include "crvShape.h"
-#include "apfElement.h"
-#include "em.h"
-
-namespace em {
-
-struct QRDecomp {
-  mth::Matrix<double> Q;
-  mth::Matrix<double> R;
-};
-
-/*
- * This function takes the NedelecField and g parameters field
- * and returns the THETA field. It computes the 3 scalar parameters
- * per face for the flux correction vectors.
- */
-apf::Field* computeFluxCorrection(apf::Field* ef, apf::Field* g)
-{
-  apf::Mesh* mesh = apf::getMesh(ef);
-  apf::Field* THETA_Field =  createPackedField(
-      mesh, "theta_field", 3, apf::getConstant(2));
-
-  apf::MeshEntity* face;
-  apf::MeshIterator* it = mesh->begin(2);
-  while ((face = mesh->iterate(it))) {
-
-    // 1. assemble RHS vector
-    double components[3];
-    apf::getComponents(g, face, 0, components);
-
-    mth::Vector<double> rhs(3);
-    apf::Downward edges;
-    int ne = mesh->getDownward(face, 1, edges);
-    for (int i = 0; i < ne; i++) {
-      int which, rotate; bool flip;
-      apf::getAlignment(mesh, face, edges[i], which, flip, rotate);
-      if (flip)
-        rhs(i) = -1. * components[i];
-      else
-        rhs(i) = components[i];
-    }
-
-    // 2. assemble LHS face mass matrix
-    mth::Matrix<double> M;
-    assembleVectorMassElementMatrix(mesh, face, ef, M);
-
-    // 3. solve the system
-    QRDecomp qr;
-    mth::decomposeQR(M, qr.Q, qr.R);
-    mth::Vector<double> theta;
-    mth::solveFromQR(qr.Q, qr.R, rhs, theta);
-
-    // set solution vector on face field
-    components[0] = theta(0);
-    components[1] = theta(1);
-    components[2] = theta(2);
-    apf::setComponents(THETA_Field, face, 0, components);
-  }
-  mesh->end(it);
-
-  return THETA_Field;
-}
-
-}
-
-
diff --git a/em/emResidualFunctionals.cc b/em/emResidualFunctionals.cc
deleted file mode 100644
index 015dc5a00..000000000
--- a/em/emResidualFunctionals.cc
+++ /dev/null
@@ -1,918 +0,0 @@
-/*
- * Copyright (C) 2011 Scientific Computation Research Center
- *
- * This work is open source software, licensed under the terms of the
- * BSD license as described in the LICENSE file in the top-level directory.
- */
-#include <algorithm>
-#include <apfCavityOp.h>
-#include "apfElement.h"
-#include "crv.h"
-#include "crvShape.h"
-#include "em.h"
-
-namespace em {
-
-enum {VISITED};
-
-/* overall information useful during equilibration */
-struct Equilibration {
-  apf::Mesh* mesh;
-  /* mesh dimension, so far handling 3 only */
-  int dim;
-  /* polynomial order of Nedelec space */
-  int order;
-  /* input scalar field containing Nedelec dofs for solution electric field */
-  apf::Field* ef;
-  /* tags each edge once it has been visited during the equilibration process */
-  apf::MeshTag* tag;
-  /* output field containing correction values.
-   * currently 3 scalar values are stored on each face
-   * in order corresponding to downward edges of the face */
-  apf::Field* g;
-};
-
-static void setupEquilibration(Equilibration* eq, apf::Field* f, apf::Field* g)
-{
-  eq->mesh = apf::getMesh(f);
-  eq->tag = eq->mesh->createIntTag("isVisited", 1);
-  eq->dim = eq->mesh->getDimension();
-  eq->ef = f;
-  eq->order = f->getShape()->getOrder();
-  eq->g = g;
-
-  double zeros[3] = {0., 0., 0.};
-  apf::MeshEntity* face;
-  apf::MeshIterator* it = eq->mesh->begin(2);
-  while ((face = eq->mesh->iterate(it))) {
-    apf::setComponents(eq->g, face, 0, zeros);
-  }
-  eq->mesh->end(it);
-}
-
-struct QRDecomp {
-  mth::Matrix<double> Q;
-  mth::Matrix<double> R;
-};
-
-typedef std::vector<apf::MeshEntity*> EntityVector;
-
-struct EdgePatch {
-  apf::Mesh* mesh;
-  Equilibration* equilibration;
-  /* the edge entity around which the patch
-     is centered. a patch collects entities
-     (faces & tets) around this edge entity */
-  apf::MeshEntity* entity;
-  bool isOnBdry;
-  EntityVector tets;
-  EntityVector faces;
-  mth::Matrix<double> A;
-  mth::Matrix<double> At;
-  mth::Matrix<double> T; // T = A*At + 1
-  mth::Vector<double> b;
-  mth::Vector<double> x;
-  QRDecomp qr;
-};
-
-static void setupEdgePatch(EdgePatch* ep, Equilibration* eq)
-{
-  ep->mesh = eq->mesh;
-  ep->equilibration = eq;
-  ep->entity = 0;
-}
-
-static void startEdgePatch(EdgePatch* ep, apf::MeshEntity* e)
-{
-  ep->tets.clear();
-  ep->faces.clear();
-  ep->entity = e;
-  ep->isOnBdry = crv::isBoundaryEntity(ep->mesh, ep->entity);
-}
-
-static void addEntityToPatch(EdgePatch* ep, apf::MeshEntity* e)
-{
-  if(ep->mesh->getType(e) == apf::Mesh::TRIANGLE)
-    ep->faces.push_back(e);
-  if(ep->mesh->getType(e) == apf::Mesh::TET)
-    ep->tets.push_back(e);
-}
-
-static void addEntitiesToPatch(
-    EdgePatch* ep, apf::DynamicArray<apf::MeshEntity*>& es)
-{
-  for (std::size_t i=0; i < es.getSize(); ++i)
-    addEntityToPatch(ep, es[i]);
-}
-
-static bool getInitialEdgePatch(EdgePatch* ep, apf::CavityOp* o)
-{
-  if ( ! o->requestLocality(&ep->entity,1))
-    return false;
-  apf::DynamicArray<apf::MeshEntity*> adjacent;
-  ep->mesh->getAdjacent(ep->entity, 3, adjacent);
-  addEntitiesToPatch(ep, adjacent);
-
-  ep->mesh->getAdjacent(ep->entity, 2, adjacent);
-  addEntitiesToPatch(ep, adjacent);
-
-  return true;
-}
-
-static bool buildEdgePatch(EdgePatch* ep, apf::CavityOp* o)
-{
-  if (!getInitialEdgePatch(ep, o)) return false;
-  return true;
-}
-
-/*
- * Reference: Leszek Demkowicz, Computing with hp-adaptive
- * finite elements, vol2, equation (11.118, 11.119, 11.122).
- */
-static void assembleEdgePatchLHS(EdgePatch* ep)
-{
-  int ne = ep->tets.size();
-  int nf = ep->faces.size();
-  if( crv::isBoundaryEntity(ep->mesh, ep->entity) ) {
-    ep->T.resize(ne+nf, ne+nf);
-    ep->T.zero();
-    for (int i = 0; i < nf; i++)
-      ep->T(i,i) = 2.;
-    for (int i = 0; i < ne-1; i++) {
-      ep->T(i+nf,i) = 1.; ep->T(i+nf,i+1) = -1.;
-      ep->T(i,i+nf) = 1.; ep->T(i+1,i+nf) = -1.;
-    }
-    ep->T(ne+nf-1, ne-1) = 1.; ep->T(ne+nf-1, ne) = 1.;
-    ep->T(ne-1, ne+nf-1) = 1.; ep->T(ne, ne+nf-1) = 1.;
-  }
-  else if( ! crv::isBoundaryEntity(ep->mesh, ep->entity) ) {
-    ep->A.resize(ne, nf);
-    ep->A.zero();
-    for (int i = 0; i < ne-1; i++) {
-      ep->A(i,i) = 1.;  ep->A(i,i+1) = -1.;
-    }
-    ep->A(ne-1,0) = -1.; ep->A(ne-1,ne-1) = 1.;
-    // A is singular so do (A*At) + 1.0
-    // to pick a particular solution
-    ep->At.resize(nf,ne);
-    mth::transpose(ep->A, ep->At);
-    mth::multiply(ep->A, ep->At, ep->T);
-
-    for (int i = 0; i < ne; i++)
-      for (int j = 0; j < ne; j++)
-        ep->T(i,j) += 1.;
-  }
-  mth::decomposeQR(ep->T, ep->qr.Q, ep->qr.R);
-}
-
-/*
- * Performs Curl Curl integration using curl vector Nedelec shapes
- */
-void assembleCurlCurlElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
-    apf::Field* f, mth::Matrix<double>& elmat)
-{
-  apf::FieldShape* fs = f->getShape();
-  int type = mesh->getType(e);
-  PCU_ALWAYS_ASSERT(type == apf::Mesh::TET);
-  int nd = apf::countElementNodes(fs, type);
-  int dim = apf::getDimension(mesh, e);
-  int dimc = (dim == 3) ? 3 : 1;
-  double w;
-
-  apf::NewArray<apf::Vector3> curlshape(nd);
-  mth::Matrix<double> phys_curlshape(nd, dimc);
-  elmat.resize(nd,nd);
-
-  apf::MeshElement* me = apf::createMeshElement(mesh, e);
-  apf::Element* el = apf::createElement(f, me);
-  int int_order = 2 * fs->getOrder() - 2;
-  int np = apf::countIntPoints(me, int_order);
-
-  elmat.zero();
-  apf::Vector3 p;
-  for (int i = 0; i < np; i++) {
-    apf::getIntPoint(me, int_order, i, p);
-    double weight = apf::getIntWeight(me, int_order, i);
-    apf::Matrix3x3 J;
-    apf::getJacobian(me, p, J);
-    double jdet = apf::getJacobianDeterminant(J, dim);
-    w = weight / jdet;
-
-    if (dim == 3) {
-      el->getShape()->getLocalVectorCurls(mesh, e, p, curlshape);
-      phys_curlshape.zero();
-      for (int j = 0; j < nd; j++)
-        for (int k = 0; k < dim; k++)
-          for (int l = 0; l < dim; l++)
-            phys_curlshape(j,k) += curlshape[j][l] * J[l][k];
-    }
-    mth::Matrix<double> phys_curlshapeT;
-    mth::transpose(phys_curlshape, phys_curlshapeT);
-
-    mth::Matrix<double> M (nd, nd);
-    M.zero();
-    mth::multiply(phys_curlshape, phys_curlshapeT, M);
-    M *= w;
-    elmat += M;
-  }
-  apf::destroyElement(el);
-  apf::destroyMeshElement(me);
-}
-
-/*
- * Performs Vector Vector Mass integration using vector Nedelec shapes
- */
-void assembleVectorMassElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
-    apf::Field* f, mth::Matrix<double>& elmat)
-{
-  apf::FieldShape* fs = f->getShape();
-  int type = mesh->getType(e);
-  PCU_ALWAYS_ASSERT(type == apf::Mesh::TET || type == apf::Mesh::TRIANGLE);
-  int nd = apf::countElementNodes(fs, type);
-  int dim = apf::getDimension(mesh, e);
-  int sdim = mesh->getDimension();
-  double w;
-
-  apf::NewArray<apf::Vector3> vectorshapes(nd);
-  elmat.resize(nd,nd);
-
-  apf::MeshElement* me = apf::createMeshElement(mesh, e);
-  apf::Element* el = apf::createElement(f, me);
-  int int_order = 2 * fs->getOrder();
-  int np = apf::countIntPoints(me, int_order);
-
-  elmat.zero();
-  apf::Vector3 p;
-  for (int i = 0; i < np; i++) {
-    apf::getIntPoint(me, int_order, i, p);
-    double weight = apf::getIntWeight(me, int_order, i);
-    apf::Matrix3x3 J;
-    apf::getJacobian(me, p, J);
-    double jdet = apf::getJacobianDeterminant(J, dim);
-    w = weight * jdet;
-
-    apf::getVectorShapeValues(el, p, vectorshapes);
-    mth::Matrix<double> vectorShapes (nd, sdim);
-    for (int j = 0; j < nd; j++)
-      for (int k = 0; k < sdim; k++)
-        vectorShapes(j,k) = vectorshapes[j][k];
-
-    mth::Matrix<double> vectorShapesT (sdim, nd);
-    mth::transpose(vectorShapes, vectorShapesT);
-
-    mth::Matrix<double> M (nd,nd);
-    M.zero();
-    mth::multiply(vectorShapes, vectorShapesT, M);
-    M *= w;
-    elmat += M;
-  }
-
-  apf::destroyElement(el);
-  apf::destroyMeshElement(me);
-}
-
-void assembleElementMatrix(apf::Mesh* mesh, apf::MeshEntity*e,
-    apf::Field* f, mth::Matrix<double>& elmat)
-{
-  mth::Matrix<double> curl_elmat, mass_elmat;
-  assembleCurlCurlElementMatrix(mesh, e, f, curl_elmat);
-  assembleVectorMassElementMatrix(mesh, e, f, mass_elmat);
-
-  elmat.resize(curl_elmat.rows(), curl_elmat.cols());
-  elmat.zero();
-  elmat += curl_elmat;
-  elmat += mass_elmat;
-}
-
-/*
- * computes local bilinear form integral restricted to
- * an edge of a tet element
- */
-static double getLocalEdgeBLF(EdgePatch* ep, apf::MeshEntity* tet)
-{
-  PCU_ALWAYS_ASSERT(ep->mesh->getType(tet) == apf::Mesh::TET);
-  // findIn edge in downward edges of tet
-  apf::Downward e;
-  int ne = ep->mesh->getDownward(tet, 1, e);
-  int ei = apf::findIn(e, ne, ep->entity);
-  // get Element Dofs
-  apf::MeshElement* me = apf::createMeshElement(ep->mesh, tet);
-  apf::Element* el = apf::createElement(ep->equilibration->ef, me);
-  int type = ep->mesh->getType(tet);
-  int nd = apf::countElementNodes(el->getFieldShape(), type);
-  apf::NewArray<double> d (nd);
-  el->getElementDofs(d);
-  mth::Vector<double> dofs (nd);
-  for (int i = 0; i < nd; i++)
-    dofs(i) = d[i];
-  // assemble curl curl element matrix
-  mth::Matrix<double> curl_elmat;
-  assembleCurlCurlElementMatrix(ep->mesh, tet,
-      ep->equilibration->ef, curl_elmat);
-  // assemble vector mass element matrix
-  mth::Matrix<double> mass_elmat;
-  assembleVectorMassElementMatrix(ep->mesh, tet,
-      ep->equilibration->ef, mass_elmat);
-  // add element matrices
-  mth::Matrix<double> elmat(nd, nd);
-  elmat.zero();
-  elmat += curl_elmat;
-  elmat += mass_elmat;
-  // multiply element matrix with element dofs
-  mth::Vector<double> blf_integrals (nd);
-  mth::multiply(elmat, dofs, blf_integrals);
-
-  apf::destroyElement(el);
-  apf::destroyMeshElement(me);
-
-  // pick edge index from the resulting vector
-  // negation of negative ND dofs
-  int which, rotate; bool flip;
-  apf::getAlignment(ep->mesh, tet, ep->entity, which, flip, rotate);
-  if (flip)
-    blf_integrals(ei) = -1*blf_integrals(ei);
-  return blf_integrals(ei);
-}
-
-/*
- * Capability can be added in the future to allow the user
- * to define a custom  function */
-void pumiUserFunction(apf::Mesh* mesh, apf::MeshEntity* e,
-    const apf::Vector3& x, mth::Vector<double>& f)
-{
-  double freq = 1.;
-  double kappa = freq * M_PI;
-  int dim = apf::getDimension(mesh, e);
-  if (dim == 3) {
-      f(0) = (1. + kappa * kappa) * sin(kappa * x[1]);
-      f(1) = (1. + kappa * kappa) * sin(kappa * x[2]);
-      f(2) = (1. + kappa * kappa) * sin(kappa * x[0]);
-      /*f(0) = 0.;
-      f(1) = 0.;
-      f(2) = 0.;*/
-  }
-  else {
-     f(0) = (1. + kappa * kappa) * sin(kappa * x[1]);
-     f(1) = (1. + kappa * kappa) * sin(kappa * x[0]);
-     f(2) = 0.0;
-  }
-}
-void assembleDomainLFElementVector(apf::Mesh* mesh, apf::MeshEntity* e,
-    apf::Field* f, mth::Vector<double>& elvect)
-{
-  apf::FieldShape* fs = f->getShape();
-  int type = mesh->getType(e);
-  PCU_ALWAYS_ASSERT(type == apf::Mesh::TET);
-  int nd = apf::countElementNodes(fs, type);
-  int dim = apf::getDimension(mesh, e);
-  double w;
-
-  apf::NewArray<apf::Vector3> vectorshapes(nd);
-  elvect.resize(nd);
-  mth::Vector<double> val (dim);
-  val.zero();
-  apf::Vector3 p;
-
-  apf::MeshElement* me = apf::createMeshElement(mesh, e);
-  apf::Element* el = apf::createElement(f, me);
-  int int_order = 2 * fs->getOrder();
-  int np = apf::countIntPoints(me, int_order);
-
-  elvect.zero();
-  for (int i = 0; i < np; i++) {
-    apf::getIntPoint(me, int_order, i, p);
-    double weight = apf::getIntWeight(me, int_order, i);
-    apf::Matrix3x3 J;
-    apf::getJacobian(me, p, J);
-    double jdet = apf::getJacobianDeterminant(J, dim);
-    w = weight * jdet;
-
-    apf::getVectorShapeValues(el, p, vectorshapes);
-    apf::Vector3 global;
-    apf::mapLocalToGlobal(me, p, global);
-    pumiUserFunction(mesh, e, global, val);
-    val *= w;
-
-    mth::Matrix<double> vectorShapes (nd, dim);
-    for (int j = 0; j < nd; j++)
-      for (int k = 0; k < dim; k++)
-        vectorShapes(j,k) = vectorshapes[j][k];
-    mth::Vector<double> V (nd);
-    V.zero();
-    mth::multiply(vectorShapes, val, V);
-    elvect += V;
-  }
-
-  apf::destroyElement(el);
-  apf::destroyMeshElement(me);
-}
-
-/*
- * computes local linear form integral restricted to
- * an edge of a tet element
- */
-static double getLocalEdgeLF(EdgePatch* ep, apf::MeshEntity* tet)
-{
-  PCU_ALWAYS_ASSERT(ep->mesh->getType(tet) == apf::Mesh::TET);
-  // findIn edge in downward edges of tet
-  apf::Downward e;
-  int ne = ep->mesh->getDownward(tet, 1, e);
-  int ei = apf::findIn(e, ne, ep->entity);
-  // assemble Domain LF Vector
-  mth::Vector<double> elvect;
-  assembleDomainLFElementVector(ep->mesh, tet,
-      ep->equilibration->ef, elvect);
-  int which, rotate; bool flip;
-  apf::getAlignment(ep->mesh, tet, ep->entity, which, flip, rotate);
-  if (flip)
-    elvect(ei) = -1*elvect(ei);
-  return elvect(ei);
-}
-
-/*
- * Given a tet and one of its faces, the vertex of the tet
- * opposite to the given face is returned.
- */
-static apf::MeshEntity* getTetOppVert(
-    apf::Mesh* m, apf::MeshEntity* t, apf::MeshEntity* f)
-{
-  apf::Downward fvs;
-  int fnv = m->getDownward(f, 0, fvs);
-  apf::Downward tvs;
-  int tnv = m->getDownward(t, 0, tvs);
-  PCU_ALWAYS_ASSERT(tnv == 4 && fnv == 3);
-  for (int i = 0; i < tnv; i++) {
-    if (apf::findIn(fvs, fnv, tvs[i]) == -1)
-      return tvs[i];
-  }
-  return 0;
-}
-
-/*
- * Given a face and one of its edges, the vertex of the face
- * opposite to the given edge is returned.
- */
-static apf::MeshEntity* getFaceOppVert(
-    apf::Mesh* m, apf::MeshEntity* f, apf::MeshEntity* e)
-{
-  apf::Downward evs;
-  int env = m->getDownward(e, 0, evs);
-  apf::Downward fvs;
-  int fnv = m->getDownward(f, 0, fvs);
-  PCU_ALWAYS_ASSERT(env == 2 && fnv == 3);
-  for (int i = 0; i < fnv; i++) {
-    if (apf::findIn(evs, env, fvs[i]) == -1)
-      return fvs[i];
-  }
-  return 0;
-}
-
-static apf::Vector3 computeFaceNormal(
-    apf::Mesh* m, apf::MeshEntity* f, apf::Vector3 const& p)
-{
-  // Compute face normal using face Jacobian,
-  // so it can also be used for curved mesh elements
-  apf::MeshElement* me = apf::createMeshElement(m, f);
-  apf::Matrix3x3 J;
-  apf::getJacobian(me, p, J);
-  apf::destroyMeshElement(me);
-
-  apf::Vector3 g1 = J[0];
-  apf::Vector3 g2 = J[1];
-  apf::Vector3 n = apf::cross( g1, g2 );
-  return n.normalize();
-}
-
-apf::Vector3 computeFaceOutwardNormal(apf::Mesh* m,
-    apf::MeshEntity* t, apf::MeshEntity* f, apf::Vector3 const& p)
-{
-  apf::Vector3 n = computeFaceNormal(m, f, p);
-
-  // orient the normal outwards from the tet
-  apf::MeshEntity* oppVert = getTetOppVert(m, t, f);
-  apf::Vector3 vxi = apf::Vector3(0.,0.,0.);
-
-  apf::Vector3 txi;
-  m->getPoint(oppVert, 0, txi);
-
-  apf::MeshElement* fme = apf::createMeshElement(m, f);
-  apf::Vector3 pxi;
-  apf::mapLocalToGlobal(fme, p, pxi);
-  apf::destroyMeshElement(fme);
-
-  apf::Vector3 pxiTotxi = txi - pxi;
-  if (pxiTotxi*n > 0) {
-    n = n*-1.;
-  }
-  return n;
-}
-
-/*
- * computes local flux integral restricted to
- * an edge of a tet element
- */
-static double getLocalFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
-{
-  PCU_ALWAYS_ASSERT(ep->mesh->getType(tet) == apf::Mesh::TET);
-
-  double fluxIntegral = 0.0;
-  // 1. get faces of the tet in the patch
-  apf::Downward f;
-  int nf = ep->mesh->getDownward(tet, 2, f);
-  std::vector<apf::MeshEntity*> patchFaces;
-  for (int i = 0; i < nf; i++) {
-    if(std::find(ep->faces.begin(), ep->faces.end(), f[i]) != ep->faces.end())
-      patchFaces.push_back(f[i]);
-  }
-  PCU_ALWAYS_ASSERT(patchFaces.size() == 2);
-
-  //  2. loop over the patch faces
-  for (unsigned int i = 0; i < patchFaces.size(); i++) {
-    double fluxFaceIntegral = 0.0;
-    // 3. get upward tets of the current face
-    apf::Up up;
-    apf::MeshEntity* currentFace = patchFaces[i];
-    ep->mesh->getUp(currentFace, up);
-    if (crv::isBoundaryEntity(ep->mesh, currentFace))
-      PCU_ALWAYS_ASSERT( up.n == 1);
-    else
-      PCU_ALWAYS_ASSERT( up.n == 2);
-
-    apf::MeshEntity* firstTet = up.e[0];
-    apf::MeshEntity* secondTet = nullptr;
-    if (up.n == 2)
-      secondTet  = up.e[1];
-
-    // 4. findIn edge in downward edges of current face
-    apf::Downward e;
-    int ne = ep->mesh->getDownward(currentFace, 1, e);
-    PCU_ALWAYS_ASSERT(ne == 3);
-    int ei = apf::findIn(e, ne, ep->entity);
-
-    // 5. count integration points for flux face integral
-    apf::FieldShape* fs = ep->equilibration->ef->getShape();
-    int int_order = 2 * fs->getOrder();
-    apf::MeshElement* fme = apf::createMeshElement(ep->mesh, currentFace);
-    apf::Element* fel = apf::createElement(ep->equilibration->ef, fme);
-    int np = apf::countIntPoints(fme, int_order);
-
-    // loop over integration points
-    apf::Vector3 p, tet1xi, tet2xi, curl1, curl2, curl,
-      fn1, fn2, tk, vshape;
-    for (int n = 0; n < np; n++) {
-      apf::getIntPoint(fme, int_order, n, p);
-      double weight = apf::getIntWeight(fme, int_order, n);
-      apf::Matrix3x3 fJ;
-      apf::getJacobian(fme, p, fJ);
-      double jdet = apf::getJacobianDeterminant(
-          fJ, apf::getDimension(ep->mesh, currentFace));
-
-      // compute face outward normals wrt tets
-      if (tet == firstTet) {
-        fn1 = computeFaceOutwardNormal(ep->mesh, firstTet, currentFace, p);
-        fn2 = apf::Vector3(0.,0.,0.);
-      }
-      else {
-        fn1 = computeFaceOutwardNormal(ep->mesh, secondTet, currentFace, p);
-        fn2 = apf::Vector3(0.,0.,0.);
-      }
-      if (up.n == 2) {
-        if (tet == firstTet)
-          fn2 = computeFaceOutwardNormal(ep->mesh, secondTet, currentFace, p);
-        else
-          fn2 = computeFaceOutwardNormal(ep->mesh, firstTet, currentFace, p);
-      }
-
-      curl.zero();
-      // compute curl1
-      tet1xi = apf::boundaryToElementXi(ep->mesh, currentFace, firstTet, p);
-      apf::MeshElement* me1 = apf::createMeshElement(ep->mesh, firstTet);
-      apf::Element* el1 = apf::createElement(ep->equilibration->ef, me1);
-      apf::getCurl(el1, tet1xi, curl1);
-      apf::Vector3 temp1 = apf::cross(fn1, curl1);
-      curl += temp1;
-      apf::destroyElement(el1);
-      apf::destroyMeshElement(me1);
-
-      // compute curl2
-      if (up.n == 2) {
-        tet2xi = apf::boundaryToElementXi(ep->mesh, currentFace, secondTet, p);
-        apf::MeshElement* me2 = apf::createMeshElement(ep->mesh, secondTet);
-        apf::Element* el2 = apf::createElement(ep->equilibration->ef, me2);
-        apf::getCurl(el2, tet2xi, curl2);
-        apf::Vector3 temp2 = apf::cross(fn2, curl2);
-        curl += (temp2 * -1.);
-        curl = curl * 1./2.;
-        apf::destroyElement(el2);
-        apf::destroyMeshElement(me2);
-      }
-
-      // compute tk (inter-element averaged flux)
-      tk = curl;
-
-      // compute vector shape
-      int type = apf::Mesh::TRIANGLE;
-      int nd = apf::countElementNodes(fs, type);
-      apf::NewArray<apf::Vector3> vectorshapes (nd);
-      apf::getVectorShapeValues(fel, p, vectorshapes);
-      vshape = vectorshapes[ei];
-
-      // compute integral
-      fluxFaceIntegral += (tk * vshape) * weight * jdet;
-    }
-    apf::destroyElement(fel);
-    apf::destroyMeshElement(fme);
-
-    fluxIntegral += fluxFaceIntegral;
-  }
-  return fluxIntegral;
-}
-
-static void assembleEdgePatchRHS(EdgePatch* p)
-{
-  if (p->isOnBdry) {
-    p->b.resize(p->tets.size() + p->faces.size());
-    p->b.zero();
-  }
-  else {
-    p->b.resize(p->tets.size());
-    p->b.zero();
-  }
-  int ne = p->tets.size();
-  int nf = p->faces.size();
-  for (int i = 0; i < ne; i++) {
-    apf::MeshEntity* tet = p->tets[i];
-    double blfIntegral = getLocalEdgeBLF(p, tet);
-    double lfIntegral = getLocalEdgeLF(p, tet);
-    double fluxIntegral = getLocalFluxIntegral(p, tet);
-    if(p->isOnBdry)
-      p->b(nf+i) = blfIntegral - lfIntegral - fluxIntegral;
-    else
-      p->b(i) = blfIntegral - lfIntegral - fluxIntegral;
-  }
-}
-
-static apf::MeshEntity* getTetOppFaceSharingEdge(
-    apf::Mesh* m, apf::MeshEntity* t, apf::MeshEntity* f, apf::MeshEntity* e)
-{
-  apf::MeshEntity* fs[4];
-  m->getDownward(t, 2, fs);
-  for (int i = 0; i < 4; i++) {
-    if (fs[i] == f) continue;
-    apf::MeshEntity* es[3];
-    m->getDownward(fs[i], 1, es);
-    if (apf::findIn(es, 3, e) > -1)
-      return fs[i];
-  }
-  return 0;
-}
-
-/*
- * Orders tets and faces in an edge cavity in a
- * clockwise (or ccw) direction around the edge.
- */
-static void getOrderedTetsandFaces(apf::Mesh* mesh, apf::MeshEntity* edge,
-     EntityVector& tets, EntityVector& faces)
-{
-  tets.clear();
-  faces.clear();
-  if( ! crv::isBoundaryEntity(mesh, edge) ) {
-    apf::MeshEntity* currentFace = mesh->getUpward(edge, 0);
-    apf::Up up;
-    mesh->getUp(currentFace, up);
-    PCU_ALWAYS_ASSERT(up.n == 2);
-    apf::MeshEntity* firstTet = up.e[0];
-    apf::MeshEntity* nextTet  = up.e[1];
-    tets.push_back(firstTet);
-    apf::MeshEntity* firstFace = getTetOppFaceSharingEdge(mesh, firstTet,
-        currentFace, edge);
-    faces.push_back(firstFace);
-
-    while (nextTet != firstTet) {
-      tets.push_back(nextTet);
-      faces.push_back(currentFace);
-      currentFace = getTetOppFaceSharingEdge(mesh, nextTet, currentFace, edge);
-      PCU_ALWAYS_ASSERT(currentFace);
-      apf::Up up;
-      mesh->getUp(currentFace, up);
-      PCU_ALWAYS_ASSERT(up.n == 2);
-      if (nextTet != up.e[0])
-        nextTet = up.e[0];
-      else
-        nextTet = up.e[1];
-    }
-  }
-  else {
-    apf::Up up;
-    mesh->getUp(edge, up);
-    apf::MeshEntity* firstFace;
-    for (int i = 0; i < up.n; i++) {
-      if ( crv::isBoundaryEntity(mesh, up.e[i]) ) {
-        firstFace = up.e[i]; break;
-      }
-    }
-    faces.push_back(firstFace);
-    mesh->getUp(firstFace, up);
-    PCU_ALWAYS_ASSERT(up.n == 1);
-    apf::MeshEntity* firstTet = up.e[0];
-    tets.push_back(firstTet);
-
-    apf::MeshEntity* nextFace = getTetOppFaceSharingEdge(mesh, firstTet,
-        firstFace, edge);
-    apf::MeshEntity* nextTet = firstTet;
-    mesh->getUp(nextFace, up);
-    while( up.n == 2) {
-      faces.push_back(nextFace);
-      if (nextTet != up.e[0])
-        nextTet = up.e[0];
-      else
-        nextTet = up.e[1];
-      tets.push_back(nextTet);
-
-      nextFace = getTetOppFaceSharingEdge(mesh, nextTet, nextFace, edge);
-      mesh->getUp(nextFace, up);
-    }
-    faces.push_back(nextFace);
-  }
-}
-
-void getClockwiseTetsandFaces(EdgePatch* p)
-{
-  if (p->isOnBdry) {
-    if (p->tets.size() > 1) {
-      apf::MeshEntity* secondFace = p->faces[1];
-      apf::MeshEntity* firstTet = p->tets[0];
-      apf::MeshEntity* secondTet = p->tets[1];
-
-      apf::Downward firstTetFaces, secondTetFaces;
-      int n_firstTetFaces = p->mesh->getDownward(firstTet, 2, firstTetFaces);
-      int n_secondTetFaces = p->mesh->getDownward(secondTet, 2, secondTetFaces);
-      int fi1 = apf::findIn(firstTetFaces, n_firstTetFaces, secondFace);
-      int fi2 = apf::findIn(secondTetFaces, n_secondTetFaces, secondFace);
-      PCU_ALWAYS_ASSERT(fi1 != -1 && fi2 != -1);
-
-      // first tet opp vertex crd
-      apf::MeshEntity* firstTetOppVert = getTetOppVert(
-        p->mesh, firstTet, secondFace);
-      apf::Vector3 firstTetOppVertCrd;
-      p->mesh->getPoint(firstTetOppVert, 0, firstTetOppVertCrd);
-
-      // second tet opp vertex crd
-      apf::MeshEntity* secondTetOppVert = getTetOppVert(
-        p->mesh, secondTet, secondFace);
-      apf::Vector3 secondTetOppVertCrd;
-      p->mesh->getPoint(secondTetOppVert, 0, secondTetOppVertCrd);
-
-      // normal to the face
-      apf::Downward edge_vertices;
-      p->mesh->getDownward(p->entity, 0, edge_vertices);
-      apf::Vector3 p0, p1, p2;
-      p->mesh->getPoint(edge_vertices[0], 0, p0);
-      p->mesh->getPoint(edge_vertices[1], 0, p1);
-      apf::MeshEntity* faceOppVert = getFaceOppVert(
-        p->mesh, secondFace, p->entity);
-      p->mesh->getPoint(faceOppVert, 0, p2);
-
-      apf::Vector3 normal = apf::cross(p1-p0, p2-p0);
-
-      // direction vectors from p0 to opp tet verts
-      apf::Vector3 vFirst = firstTetOppVertCrd - p0;
-      apf::Vector3 vLast = secondTetOppVertCrd - p0;
-
-      if ((vFirst * normal > 0)  && (vLast * normal < 0)) {
-        // reverse list of tets and faces
-        std::reverse(p->tets.begin(), p->tets.end());
-        std::reverse(p->faces.begin(), p->faces.begin());
-      }
-    }
-  }
-  else {
-    apf::MeshEntity* firstFace = p->faces[0];
-    apf::MeshEntity* firstTet = p->tets[0];
-    apf::MeshEntity* lastTet = p->tets[p->tets.size()-1];
-
-    apf::Downward firstTetFaces, lastTetFaces;
-    int n_firstTetFaces = p->mesh->getDownward(firstTet, 2, firstTetFaces);
-    int n_lastTetFaces = p->mesh->getDownward(lastTet, 2, lastTetFaces);
-    int fi1 = apf::findIn(firstTetFaces, n_firstTetFaces, firstFace);
-    int filast = apf::findIn(lastTetFaces, n_lastTetFaces, firstFace);
-    PCU_ALWAYS_ASSERT(fi1 != -1 && filast != -1);
-
-    // first tet opp vertex crd
-    apf::MeshEntity* firstTetOppVert = getTetOppVert(
-        p->mesh, firstTet, firstFace);
-    apf::Vector3 firstTetOppVertCrd;
-    p->mesh->getPoint(firstTetOppVert, 0, firstTetOppVertCrd);
-
-    // last tet opp vertex crd
-    apf::MeshEntity* lastTetOppVert = getTetOppVert(
-        p->mesh, lastTet, firstFace);
-    apf::Vector3 lastTetOppVertCrd;
-    p->mesh->getPoint(lastTetOppVert, 0, lastTetOppVertCrd);
-
-    // normal to the face
-    apf::Downward edge_vertices;
-    p->mesh->getDownward(p->entity, 0, edge_vertices);
-    apf::Vector3 p0, p1, p2;
-    p->mesh->getPoint(edge_vertices[0], 0, p0);
-    p->mesh->getPoint(edge_vertices[1], 0, p1);
-    apf::MeshEntity* faceOppVert = getFaceOppVert(
-        p->mesh, firstFace, p->entity);
-    p->mesh->getPoint(faceOppVert, 0, p2);
-
-    apf::Vector3 normal = apf::cross(p1-p0, p2-p0);
-
-    // direction vectors from p0 to opp tet verts
-    apf::Vector3 vFirst = firstTetOppVertCrd - p0;
-    apf::Vector3 vLast = lastTetOppVertCrd - p0;
-
-    if ((vFirst * normal > 0)  && (vLast * normal < 0)) {
-      // reverse list of tets and faces
-      std::reverse(p->tets.begin(), p->tets.end());
-      std::reverse(p->faces.begin(), p->faces.begin());
-    }
-  }
-}
-
-static void runErm(EdgePatch* ep)
-{
-  getOrderedTetsandFaces(ep->mesh, ep->entity, ep->tets, ep->faces);
-  //getClockwiseTetsandFaces(ep);
-  assembleEdgePatchLHS(ep);
-  assembleEdgePatchRHS(ep);
-  mth::solveFromQR(ep->qr.Q, ep->qr.R, ep->b, ep->x);
-
-  if (!ep->isOnBdry) { // solve At*mu = g for g
-    mth::Vector<double> temp(ep->tets.size());
-    mth::multiply(ep->At, ep->x, temp);
-    for (size_t i = 0; i < ep->tets.size(); i++) {
-      ep->x(i) = temp(i);
-    }
-  }
-
-  int nf = ep->faces.size();
-  for(int i = 0; i < nf; i++) {
-    apf::MeshEntity* face = ep->faces[i];
-
-    apf::Downward e;
-    int ned = ep->mesh->getDownward(face, 1, e);
-    int ei = apf::findIn(e, ned, ep->entity);
-    PCU_ALWAYS_ASSERT(ned == 3 && ei != -1);
-
-    double components[3];
-    apf::getComponents(ep->equilibration->g, face, 0, components);
-    components[ei] = ep->x(i);
-    apf::setComponents(ep->equilibration->g, face, 0, components);
-  }
-}
-
-
-class EdgePatchOp : public apf::CavityOp
-{
-public:
-  EdgePatchOp(Equilibration* eq):
-    apf::CavityOp(eq->mesh)
-  {
-    setupEdgePatch(&edgePatch, eq);
-  }
-  virtual Outcome setEntity(apf::MeshEntity* e)
-  {
-    if (edgePatch.mesh->hasTag(e, edgePatch.equilibration->tag))
-      return SKIP;
-    startEdgePatch(&edgePatch, e);
-    if ( ! buildEdgePatch(&edgePatch, this))
-      return REQUEST;
-    return OK;
-  }
-  virtual void apply()
-  {
-    runErm(&edgePatch);
-    int n = VISITED;
-    edgePatch.mesh->setIntTag(
-        edgePatch.entity, edgePatch.equilibration->tag, &n);
-  }
-  EdgePatch edgePatch;
-};
-
-apf::Field* equilibrateResiduals(apf::Field* f)
-{
-  apf::Field* g = createPackedField(
-      apf::getMesh(f), "g", 3, apf::getConstant(2) );
-  Equilibration equilibration;
-  setupEquilibration(&equilibration, f, g);
-  EdgePatchOp op(&equilibration);
-  op.applyToDimension(1); // edges
-
-  apf::MeshEntity* ent;
-  apf::MeshIterator* it = apf::getMesh(f)->begin(1);
-  while ((ent = apf::getMesh(f)->iterate(it))) {
-    apf::getMesh(f)->removeTag(ent, equilibration.tag);
-  }
-  apf::getMesh(f)->end(it);
-  apf::getMesh(f)->destroyTag(equilibration.tag);
-
-  return g;
-}
-
-
-}
diff --git a/em/emSizeField.cc b/em/emSizeField.cc
deleted file mode 100644
index 15132809e..000000000
--- a/em/emSizeField.cc
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Copyright (C) 2011 Scientific Computation Research Center
- *
- * This work is open source software, licensed under the terms of the
- * BSD license as described in the LICENSE file in the top-level directory.
- */
-#include "apfElement.h"
-#include "crv.h"
-#include "crvShape.h"
-#include <apfCavityOp.h>
-#include "em.h"
-
-namespace em {
-
-
-struct Sizefield {
-  apf::Mesh* mesh;
-  /* the polynomial order of the solution electric Nedelec field */
-  int order;
-  /* the input solution electric field obtained after the FEM solve */
-  apf::Field* ef;
-  /* the per element error field obtained after executing the implicit
-     residual error estimator */
-  apf::Field* errorField;
-  double size_factor;
-  /* target error = sum of all element erros / total number of elements */
-  double target_error;
-  double alpha;
-  double beta;
-  /* a temporary field storing desired sizes at elements */
-  apf::Field* element_size;
-  /* the resulting size field, recovered from the element_size field
-     (using a local average recovery method much weaker than SPR) */
-  apf::Field* size;
-};
-
-static void setupSizefield(
-    Sizefield* sz,
-    apf::Field* ef,
-    apf::Field* error_field,
-    int n,
-    double alpha,
-    double beta)
-{
-  sz->mesh = ef->getMesh();
-  sz->order = ef->getShape()->getOrder();
-  sz->errorField = error_field;
-  sz->size_factor = 0;
-
-  apf::MeshEntity* entity;
-  int d = sz->mesh->getDimension();
-  apf::MeshIterator* it = sz->mesh->begin(d);
-  double total_error = 0.;
-  while ((entity = sz->mesh->iterate(it))) {
-    total_error += apf::getScalar(sz->errorField, entity, 0);
-  }
-  sz->mesh->end(it);
-  sz->target_error = total_error / (sz->mesh->count(d)* n);
-
-  sz->alpha = alpha;
-  sz->beta = beta;
-  sz->element_size = 0;
-  sz->size = 0;
-}
-
-static double getCurrentSize(apf::Mesh* m, apf::MeshEntity* e)
-{
-  /* right now minimum edge length is the formula... */
-  apf::Downward edges;
-  int ne = m->getDownward(e,1,edges);
-  double h = std::numeric_limits<double>::max();
-  for (int i=0; i < ne; ++i)
-    h = std::min(h, measure(m, edges[i]));
-  return h;
-}
-
-static double getDesiredSize(Sizefield* sz, apf::MeshEntity* entity)
-{
-  double element_error = apf::getScalar(sz->errorField, entity, 0);
-  double h = getCurrentSize(sz->mesh, entity);
-
-  double p = sz->order;
-  int d = sz->mesh->getDimension();
-  sz->size_factor = pow(sz->target_error/element_error, (2. / (2.*p + d)));
-
-  double h_new = h * sz->size_factor;
-  if (h_new < sz->alpha*h) h_new = sz->alpha*h;
-  if (h_new > sz->beta*h) h_new = sz->beta*h;
-  return h_new;
-}
-
-static void getElementSizeField(Sizefield* sz)
-{
-  apf::Field* eSize = apf::createStepField(
-      sz->mesh, "esize", apf::SCALAR);
-  int d = sz->mesh->getDimension();
-  apf::MeshEntity* entity;
-  apf::MeshIterator* elements = sz->mesh->begin(d);
-  while ((entity = sz->mesh->iterate(elements))) {
-    double h = getDesiredSize(sz, entity);
-    apf::setScalar(eSize, entity, 0, h);
-  }
-  sz->mesh->end(elements);
-  sz->element_size = eSize;
-}
-
-
-void averageToVertex(apf::Field* ef, apf::Field* vf, apf::MeshEntity* ent)
-{
-  apf::Mesh* m = apf::getMesh(ef);
-  apf::Adjacent elements;
-  m->getAdjacent(ent, m->getDimension(), elements);
-  double s=0;
-  for (std::size_t i=0; i < elements.getSize(); ++i)
-    s += apf::getScalar(ef, elements[i], 0);
-  s /= elements.getSize();
-  apf::setScalar(vf, ent, 0, s);
-}
-
-class AverageOp : public apf::CavityOp
-{
-public:
-  AverageOp(Sizefield* sz):
-    apf::CavityOp(sz->mesh),
-    sizefield(sz),
-    entity(0)
-  {
-  }
-  virtual Outcome setEntity(apf::MeshEntity* e)
-  {
-    entity = e;
-    if (apf::hasEntity(sizefield->size, entity))
-      return SKIP;
-    if ( !requestLocality(&entity,1))
-      return REQUEST;
-    return OK;
-  }
-  virtual void apply()
-  {
-    averageToVertex(sizefield->element_size,
-        sizefield->size, entity);
-  }
-  Sizefield* sizefield;
-  apf::MeshEntity* entity;
-};
-
-void averageSizeField(Sizefield* sz)
-{
-  sz->size = apf::createLagrangeField(sz->mesh, "size", apf::SCALAR, 1);
-  AverageOp op(sz);
-  op.applyToDimension(0);
-}
-
-apf::Field* getTargetEMSizeField(
-    apf::Field* ef,
-    apf::Field* error_field,
-    int n,
-    double alpha /*= 0.25*/,
-    double beta /*= 2.0*/)
-{
-  double t0 = PCU_Time();
-  Sizefield sz;
-  setupSizefield(&sz, ef, error_field, n, alpha, beta);
-  getElementSizeField(&sz);
-  averageSizeField(&sz);
-  apf::destroyField(sz.element_size);
-  double t1 = PCU_Time();
-  if (!PCU_Comm_Self())
-    lion_eprint(1,"EM: SizeField computed in %f seconds\n",t1-t0);
-  return sz.size;
-}
-
-}
diff --git a/em/pkg_tribits.cmake b/em/pkg_tribits.cmake
deleted file mode 100644
index fd64fa9e4..000000000
--- a/em/pkg_tribits.cmake
+++ /dev/null
@@ -1,19 +0,0 @@
-tribits_package(SCORECem)
-
-# THIS IS WHERE TRIBITS GETS HEADERS
-include_directories(${CMAKE_CURRENT_SOURCE_DIR})
-
-#Sources & Headers
-set(SOURCES
-  emResidualFunctionals.cc)
-
-set(HEADERS
-  em.h)
-
-#Library
-tribits_add_library(
-  em
-  HEADERS ${HEADERS}
-  SOURCES ${SOURCES})
-
-tribits_package_postprocess()

From 595d2da01cf3a56d3a3a6d492713b77f9e3ad9ba Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Thu, 3 Dec 2020 20:26:17 -0500
Subject: [PATCH 291/555] fixes and add checks in residualErrorEstimation test

---
 test/CMakeLists.txt                  | 2 +-
 test/residualErrorEstimation_test.cc | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index c282995cb..aab38cf59 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -185,7 +185,6 @@ test_exe_func(create_mis create_mis.cc)
 test_exe_func(fieldReduce fieldReduce.cc)
 test_exe_func(test_integrator test_integrator.cc)
 test_exe_func(test_matrix_gradient test_matrix_grad.cc)
-test_exe_func(residualErrorEstimation_test residualErrorEstimation_test.cc)
 
 if(ENABLE_DSP)
   test_exe_func(graphdist graphdist.cc)
@@ -198,6 +197,7 @@ if(ENABLE_SIMMETRIX)
   test_exe_func(degenerate_test degenerateSurfs.cc)
   test_exe_func(simZBalance simZBalance.cc)
   test_exe_func(crack_test crack_test.cc)
+  test_exe_func(residualErrorEstimation_test residualErrorEstimation_test.cc)
 endif()
 
 # send all the newly added utility executable targets
diff --git a/test/residualErrorEstimation_test.cc b/test/residualErrorEstimation_test.cc
index 4ce4bdb41..557beb863 100644
--- a/test/residualErrorEstimation_test.cc
+++ b/test/residualErrorEstimation_test.cc
@@ -35,6 +35,7 @@ int main(int argc, char** argv)
   m->verify();
 
   apf::Field* electric_field = m->getField(0);
+  PCU_ALWAYS_ASSERT(electric_field);
   apf::Field* implicit_error_field = ree::estimateError(electric_field);
   apf::destroyField(implicit_error_field);
 

From 1738518be80f33814a9ce696db13ae927ca731b3 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Thu, 3 Dec 2020 22:31:24 -0500
Subject: [PATCH 292/555] updates mesh and model path for ree test

---
 test/testing.cmake | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/testing.cmake b/test/testing.cmake
index d8c97ab8b..ca21df2c5 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -296,8 +296,8 @@ mpi_test(tet_serial 1
   "tet.smb")
 mpi_test(residualErrorEstimation_test 1
   ./residualErrorEstimation_test
-  "${MESHES}/fichera/fichera.x_t"
-  "${MESHES}/fichera/fichera.smb")
+  "${MESHES}/electromagnetic/fichera.x_t"
+  "${MESHES}/electromagnetic/fichera.smb")
 if(PCU_COMPRESS)
   set(MESHFILE "bz2:pipe_2_.smb")
 else()

From ecbdeda7607c5a0441f359fdf84d989417ae1e7e Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Sat, 5 Dec 2020 11:55:42 -0500
Subject: [PATCH 293/555] changes getElementDofs to getElementNodeData

---
 apf/apfElement.cc | 2 +-
 apf/apfElement.h  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/apf/apfElement.cc b/apf/apfElement.cc
index 5b666fa57..161de8df1 100644
--- a/apf/apfElement.cc
+++ b/apf/apfElement.cc
@@ -119,7 +119,7 @@ void Element::getNodeData()
   field->getData()->getElementData(entity,nodeData);
 }
 
-void Element::getElementDofs(NewArray<double>& d)
+void Element::getElementNodeData(NewArray<double>& d)
 {
   d.allocated() ? d.resize(nen) : d.allocate(nen);
   for (int i = 0; i < nen; i++)
diff --git a/apf/apfElement.h b/apf/apfElement.h
index 96b3677ce..b54cd5901 100644
--- a/apf/apfElement.h
+++ b/apf/apfElement.h
@@ -35,7 +35,7 @@ class Element
     EntityShape* getShape() {return shape;}
     FieldShape* getFieldShape() {return field->getShape();}
     void getComponents(Vector3 const& xi, double* c);
-    void getElementDofs(NewArray<double>& d);
+    void getElementNodeData(NewArray<double>& d);
   protected:
     void init(Field* f, MeshEntity* e, VectorElement* p);
     void getNodeData();

From 0eab738474ac3e3de231d0e48982187069e24d01 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Sat, 5 Dec 2020 11:56:27 -0500
Subject: [PATCH 294/555] clean up of 2d piola transformation

---
 apf/apf.cc | 30 ++++--------------------------
 1 file changed, 4 insertions(+), 26 deletions(-)

diff --git a/apf/apf.cc b/apf/apf.cc
index 7a9538507..d33c484ae 100644
--- a/apf/apf.cc
+++ b/apf/apf.cc
@@ -4,7 +4,6 @@
  * This work is open source software, licensed under the terms of the
  * BSD license as described in the LICENSE file in the top-level directory.
  */
-#include <iostream>
 #include "apf.h"
 #include "apfScalarField.h"
 #include "apfScalarElement.h"
@@ -482,35 +481,14 @@ void getVectorShapeValues(Element* e, Vector3 const& local,
   }
   else
   {
-    // TODO clean up
-    apf::Matrix3x3 J;
-    apf::getJacobian(e->getParent(), local, J);
-
-    apf::Matrix3x3 JT = apf::transpose(J);
-    apf::Matrix3x3 JJT = J * JT;
-    // take inverse of JTJ and transpose it
-    apf::Matrix<2,2> jjt;
-    for (int i = 0; i < 2; i++)
-      for (int j = 0; j < 2; j++)
-        jjt[i][j] = JJT[i][j];
-    apf::Matrix<2,2> JJTinv = apf::invert(jjt);
-    apf::Matrix<2,2> JJTinvT = apf::transpose(JJTinv);
-
-    apf::Matrix<2,3> JJTinvTJ; // JJTinvT * J
-    for( int i = 0; i < 2; i++ ) {
-      for ( int j = 0; j < 3; j++ ) {
-        JJTinvTJ[i][j] = 0.;
-        for ( int k = 0; k < 2; k++ )
-          JJTinvTJ[i][j] += JJTinvT[i][k] * J[k][j];
-      }
-    }
-
-    // u(x_hat) * J(x_hat)^{-1}
+    apf::Matrix3x3 Jinv;
+    apf::getJacobianInv( e->getParent(), local, Jinv );
+    apf::Matrix3x3 JinvT = apf::transpose(Jinv);
     for( size_t i = 0; i < values.size(); i++ ) {
       for ( int j = 0; j < 3; j++ ) {
         values[i][j] = 0.;
         for ( int k = 0; k < 2; k++ )
-          values[i][j] += vvals[i][k] * JJTinvTJ[k][j];
+          values[i][j] += vvals[i][k] * JinvT[k][j];
       }
     }
   }

From aae68411ffdd2ac0644e9add4bb800d0d3fbea90 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Sat, 5 Dec 2020 11:57:05 -0500
Subject: [PATCH 295/555] cleans ups in ree subdirectory

---
 ree/ree.h                     | 59 +++++++++++++++++++++++++----------
 ree/reeCorrectedFlux.cc       |  3 ++
 ree/reeEstimateError.cc       | 15 +++++----
 ree/reeFluxCorrection.cc      |  2 --
 ree/reeResidualFunctionals.cc |  2 +-
 ree/reeSizeField.cc           |  2 --
 6 files changed, 55 insertions(+), 28 deletions(-)

diff --git a/ree/ree.h b/ree/ree.h
index 8278c5424..3a98c77e7 100644
--- a/ree/ree.h
+++ b/ree/ree.h
@@ -9,6 +9,10 @@
 #define REE_H
 
 
+/** \file ree.h
+ *  \brief The Residual Error Estimator (REE) interface
+ */
+
 #include "apf.h"
 #include <apfMesh.h>
 #include <PCU.h>
@@ -19,39 +23,60 @@
 #include <mth.h>
 #include <mth_def.h>
 
+/** \namespace ree
+  * \brief All Residual Error Estimator functions
+  */
 namespace ree {
 
-/*
- * Computes nodal size field
- */
+/** @brief Computes a nodal size field from the element error field
+  *        obtained after running the residual error estimator.
+  * @param ef (In) nedelec electric field
+  * @param error_field (In) per-element residual error field
+  * @param n a parameter to prescribe allowable error
+  * @param alpha floor on the size field; alpha < h_new/h_old < beta
+  * @param beta ceiling on the size field; alpha < h_new/h_old < beta
+  * @returns a scalar mesh size field at mesh vertices
+  */
 apf::Field* getTargetEMSizeField(
     apf::Field* ef,
     apf::Field* error_field,
     int n,
     double alpha = 0.25,
     double beta = 2.0);
-/*
- * Takes the solution electric field and computes edge equilibrations.
- */
+
+/** @brief Computes equilibrated residuals using the fem nedelec electric field.
+  * @param f (In) nedelec electric field
+  */
 apf::Field* equilibrateResiduals(apf::Field* f);
 
-/*
- * Takes the solution electric field and equilibrated field (of face vectors)
- * and computes the 'correction' to the flux vectors on each face.
- */
+/** @brief Uses the fem nedelec electric field and the equilibrated residuals to
+  *        compute the 'correction' to the flux vectors on each face.
+  * @param ef (In) nedelec electric field
+  * @param g (In) equilibrated residuals field
+  */
 apf::Field* computeFluxCorrection(apf::Field* ef, apf::Field* g);
 
-/* Takes the solution electric field and correctiion to the flux vectors on
- * each face and computes the 'corrected' flux vectors on each face
- */
+/** @brief Uses the fem nedelec electric field and the 'correction' to the flux
+  *        vectors to compute the 'corrected' flux vectors on each face.
+  * @param ef (In) nedelec electric field
+  * @param theta (In) correction to the flux vectors
+  */
 apf::Field* computeCorrectedFlux(apf::Field* ef, apf::Field* theta);
 
-/* Takes the solution electric field and corrected flux field and solves
- * local element level BVPs to estimate the error.
- * Returns a per-element scalar error field.
- */
+/** @brief Solves additional local BVPs on a one order higher nedelec field on
+  *        each element to estimate the dsicretization error.
+  * @param ef (In) nedelec electric field
+  * @param correctedFlux (In) flux field which provides Neumann BCs for each
+  *        local BVP.
+  * @returns a per-element scalar residual error field.
+  */
 apf::Field* computeErrorField(apf::Field* ef, apf::Field* correctedFlux);
 
+/** @brief run the residual error estimator.
+  * @param f the fem nedelec electric field
+  *                    scales the output size field.
+  * @returns a per-element scalar residual error field.
+  */
 apf::Field* estimateError(apf::Field* f);
 
 
diff --git a/ree/reeCorrectedFlux.cc b/ree/reeCorrectedFlux.cc
index d700857c8..af03c28c4 100644
--- a/ree/reeCorrectedFlux.cc
+++ b/ree/reeCorrectedFlux.cc
@@ -65,6 +65,7 @@ static void setupCorrectFlux(
     for (int i = 0; i < np; i++) {
       apf::setComponents(cf->correctedFlux, tet, i, zeros);
     }
+    apf::destroyMeshElement(me);
   }
   cf->mesh->end(it);
 }
@@ -250,6 +251,8 @@ static void computeCorrectedFlux(FaceCavity* fc)
       apf::setComponents(fc->correctflux->correctedFlux, secondTet, n, comp2);
     }
   }
+  apf::destroyElement(fel);
+  apf::destroyMeshElement(fme);
 }
 
 class FaceCavityOp : public apf::CavityOp
diff --git a/ree/reeEstimateError.cc b/ree/reeEstimateError.cc
index f137e4f1a..308941f07 100644
--- a/ree/reeEstimateError.cc
+++ b/ree/reeEstimateError.cc
@@ -107,9 +107,9 @@ static void computeResidualBLF(apf::Mesh* mesh, apf::MeshEntity* e,
   blf += curlcurl_vec;
   blf += mass_vec;
 
-  apf::destroyMeshElement(me);
-  apf::destroyElement(fp1el);
   apf::destroyElement(fel);
+  apf::destroyElement(fp1el);
+  apf::destroyMeshElement(me);
 }
 
 static void computeLambdaVector(
@@ -340,11 +340,14 @@ static double computeL2Error(apf::Mesh* mesh, apf::MeshEntity* e,
     mth::Matrix<double> vectorShapeT (dim, nd);
     mth::transpose(vectorShape, vectorShapeT);
 
-		mth::Vector<double> err_func;
-		mth::multiply(vectorShapeT, error_dofs, err_func);
+    mth::Vector<double> err_func;
+    mth::multiply(vectorShapeT, error_dofs, err_func);
+
+    error += w * (err_func * err_func);
+  }
+  apf::destroyElement(el);
+  apf::destroyMeshElement(me);
 
-		error += w * (err_func * err_func);
-	}
 	if (error < 0.0)
 		error = -error;
 
diff --git a/ree/reeFluxCorrection.cc b/ree/reeFluxCorrection.cc
index a369cb5ba..7efcf5044 100644
--- a/ree/reeFluxCorrection.cc
+++ b/ree/reeFluxCorrection.cc
@@ -4,8 +4,6 @@
  * This work is open source software, licensed under the terms of the
  * BSD license as described in the LICENSE file in the top-level directory.
  */
-#include "crv.h"
-#include "crvShape.h"
 #include "apfElement.h"
 #include "ree.h"
 
diff --git a/ree/reeResidualFunctionals.cc b/ree/reeResidualFunctionals.cc
index 08685b384..bf8d22b3a 100644
--- a/ree/reeResidualFunctionals.cc
+++ b/ree/reeResidualFunctionals.cc
@@ -301,7 +301,7 @@ static double getLocalEdgeBLF(EdgePatch* ep, apf::MeshEntity* tet)
   int type = ep->mesh->getType(tet);
   int nd = apf::countElementNodes(el->getFieldShape(), type);
   apf::NewArray<double> d (nd);
-  el->getElementDofs(d);
+  el->getElementNodeData(d);
   mth::Vector<double> dofs (nd);
   for (int i = 0; i < nd; i++)
     dofs(i) = d[i];
diff --git a/ree/reeSizeField.cc b/ree/reeSizeField.cc
index fdd382925..00765a01a 100644
--- a/ree/reeSizeField.cc
+++ b/ree/reeSizeField.cc
@@ -5,8 +5,6 @@
  * BSD license as described in the LICENSE file in the top-level directory.
  */
 #include "apfElement.h"
-#include "crv.h"
-#include "crvShape.h"
 #include <apfCavityOp.h>
 #include "ree.h"
 

From d9f3beace6fb933437b9afe4b424d09a698672ed Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Sun, 6 Dec 2020 00:43:04 -0500
Subject: [PATCH 296/555] updates residual test to use 1k edge coarse mesh

---
 test/residualErrorEstimation_test.cc | 113 ++++++++++++++++++++++++++-
 test/testing.cmake                   |   2 +-
 2 files changed, 112 insertions(+), 3 deletions(-)

diff --git a/test/residualErrorEstimation_test.cc b/test/residualErrorEstimation_test.cc
index 557beb863..cf8e5bfcf 100644
--- a/test/residualErrorEstimation_test.cc
+++ b/test/residualErrorEstimation_test.cc
@@ -14,6 +14,15 @@
 #endif
 #include <pcu_util.h>
 
+#include <iostream>
+using namespace std;
+
+void E_exact(const apf::Vector3 &x, apf::Vector3& E);
+double computeElementExactError(apf::Mesh* mesh, apf::MeshEntity* e,
+  apf::Field* f);
+
+double freq = 1.0, kappa;
+int dim = 3;
 
 int main(int argc, char** argv)
 {
@@ -34,10 +43,53 @@ int main(int argc, char** argv)
   ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile);
   m->verify();
 
+  kappa = freq * M_PI;
+
+  // get electric field
   apf::Field* electric_field = m->getField(0);
   PCU_ALWAYS_ASSERT(electric_field);
-  apf::Field* implicit_error_field = ree::estimateError(electric_field);
-  apf::destroyField(implicit_error_field);
+
+  // compute residual error
+  apf::Field* residual_error_field = ree::estimateError(electric_field);
+
+  // compute exact error
+  apf::Field* exact_error_field = apf::createIPField(
+      m, "exact_error_field", apf::SCALAR, 1);
+  apf::MeshEntity* ent;
+  apf::MeshIterator* itr = m->begin(3);
+  while ((ent = m->iterate(itr)))
+  {
+    double exact_element_error = computeElementExactError(
+        m, ent, electric_field);
+    apf::setScalar(exact_error_field, ent, 0, exact_element_error);
+  }
+  m->end(itr);
+
+  // get max and avg computed and exact errors
+  double max_exact_error = 0., max_computed_error = 0.;
+  double avg_exact_error = 0., avg_computed_error = 0.;
+  itr = m->begin(3);
+  while ((ent = m->iterate(itr)))
+  {
+    double exact_error = apf::getScalar(exact_error_field, ent, 0);
+    if (exact_error > max_exact_error) max_exact_error = exact_error;
+    avg_exact_error += exact_error;
+
+    double computed_error = apf::getScalar(residual_error_field, ent, 0);
+    if (computed_error > max_computed_error) max_computed_error = computed_error;
+    avg_computed_error += computed_error;
+  }
+  m->end(itr);
+  avg_exact_error /= m->count(3);
+  avg_computed_error /= m->count(3);
+
+  cout << "Max Exact Error: " << max_exact_error << endl;
+  cout << "Average Exact Error: " << avg_exact_error << endl;
+  cout << "Max Computed Error: " << max_computed_error << endl;
+  cout << "Average Computed Error: " << avg_computed_error << endl;
+
+  apf::destroyField(residual_error_field);
+  apf::destroyField(exact_error_field);
 
   m->destroyNative();
   apf::destroyMesh(m);
@@ -51,4 +103,61 @@ int main(int argc, char** argv)
   MPI_Finalize();
 }
 
+void E_exact(const apf::Vector3 &x, apf::Vector3& E)
+{
+   if (dim == 3)
+   {
+      E[0] = sin(kappa * x[1]);
+      E[1] = sin(kappa * x[2]);
+      E[2] = sin(kappa * x[0]);
+   }
+   else
+   {
+      E[0] = sin(kappa * x[1]);
+      E[1] = sin(kappa * x[0]);
+      E[2] = 0.0;
+   }
+}
+
+double computeElementExactError(apf::Mesh* mesh, apf::MeshEntity* e,
+  apf::Field* f)
+{
+  double error = 0.0;
+
+  apf::FieldShape* fs = f->getShape();
+  int type = mesh->getType(e);
+  PCU_ALWAYS_ASSERT(type == apf::Mesh::TET);
+  int dim = apf::getDimension(mesh, e);
+  double w;
+
+  apf::MeshElement* me = apf::createMeshElement(mesh, e);
+  apf::Element* el = apf::createElement(f, me);
+  int order = 2*fs->getOrder() + 1;
+  int np = apf::countIntPoints(me, order);
+
+  apf::Vector3 femsol, exsol;
+
+  apf::Vector3 p;
+  for (int i = 0; i < np; i++) {
+    apf::getIntPoint(me, order, i, p);
+    double weight = apf::getIntWeight(me, order, i);
+    apf::Matrix3x3 J;
+    apf::getJacobian(me, p, J);
+    double jdet = apf::getJacobianDeterminant(J, dim);
+    w = weight * jdet;
+
+    apf::getVector(el, p, femsol);
+
+    apf::Vector3 global;
+    apf::mapLocalToGlobal(me, p, global);
+    E_exact(global, exsol);
+    apf::Vector3 diff = exsol - femsol;
+
+    error += w * (diff * diff);
+  }
+  if (error < 0.0)
+    error = -error;
+
+  return sqrt(error);
+}
 
diff --git a/test/testing.cmake b/test/testing.cmake
index ca21df2c5..fc1904918 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -297,7 +297,7 @@ mpi_test(tet_serial 1
 mpi_test(residualErrorEstimation_test 1
   ./residualErrorEstimation_test
   "${MESHES}/electromagnetic/fichera.x_t"
-  "${MESHES}/electromagnetic/fichera.smb")
+  "${MESHES}/electromagnetic/fichera_1k.smb")
 if(PCU_COMPRESS)
   set(MESHFILE "bz2:pipe_2_.smb")
 else()

From 091f578f5d3d1dc31e75ae6562b21bac51bbd6f0 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Sun, 6 Dec 2020 13:35:05 -0500
Subject: [PATCH 297/555] Small style changes to ree.h

---
 ree/ree.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ree/ree.h b/ree/ree.h
index 3a98c77e7..663a019be 100644
--- a/ree/ree.h
+++ b/ree/ree.h
@@ -74,12 +74,12 @@ apf::Field* computeErrorField(apf::Field* ef, apf::Field* correctedFlux);
 
 /** @brief run the residual error estimator.
   * @param f the fem nedelec electric field
-  *                    scales the output size field.
+  *        scales the output size field.
   * @returns a per-element scalar residual error field.
   */
 apf::Field* estimateError(apf::Field* f);
 
-
+// helper functions
 void assembleCurlCurlElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e,
     apf::Field* f, mth::Matrix<double>& elmat);
 

From 9279eeaa97f73666237db4295fbfb54e7938c62e Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Sun, 6 Dec 2020 13:36:10 -0500
Subject: [PATCH 298/555] Removed unneeded header

---
 ree/reeFluxCorrection.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ree/reeFluxCorrection.cc b/ree/reeFluxCorrection.cc
index 7efcf5044..f7e759cdc 100644
--- a/ree/reeFluxCorrection.cc
+++ b/ree/reeFluxCorrection.cc
@@ -4,7 +4,6 @@
  * This work is open source software, licensed under the terms of the
  * BSD license as described in the LICENSE file in the top-level directory.
  */
-#include "apfElement.h"
 #include "ree.h"
 
 namespace ree {

From ddd720bea30aa42850027507f92aadc781df4c47 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Sun, 6 Dec 2020 13:36:50 -0500
Subject: [PATCH 299/555] Removed the dependence on crv

---
 ree/ree.h                     | 4 ++++
 ree/reeCorrectedFlux.cc       | 5 +----
 ree/reeEstimateError.cc       | 9 +++------
 ree/reeResidualFunctionals.cc | 7 +++++++
 4 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/ree/ree.h b/ree/ree.h
index 663a019be..5e337484c 100644
--- a/ree/ree.h
+++ b/ree/ree.h
@@ -94,5 +94,9 @@ void assembleDomainLFElementVector(apf::Mesh* mesh, apf::MeshEntity* e,
 
 apf::Vector3 computeFaceOutwardNormal(apf::Mesh* m,
     apf::MeshEntity* t, apf::MeshEntity* f, apf::Vector3 const& p);
+
+bool isOnDomainBoundary(apf::Mesh* m, apf::MeshEntity* e);
+
 }
+
 #endif
diff --git a/ree/reeCorrectedFlux.cc b/ree/reeCorrectedFlux.cc
index af03c28c4..c8c11f1e9 100644
--- a/ree/reeCorrectedFlux.cc
+++ b/ree/reeCorrectedFlux.cc
@@ -5,9 +5,6 @@
  * BSD license as described in the LICENSE file in the top-level directory.
  */
 #include <apfCavityOp.h>
-#include "apfElement.h"
-#include "crv.h"
-#include "crvShape.h"
 #include "ree.h"
 
 namespace ree {
@@ -130,7 +127,7 @@ static void computeCorrectedFlux(FaceCavity* fc)
   // 1. get upward tets of the face
   apf::Up up;
   fc->mesh->getUp(face, up);
-  if (crv::isBoundaryEntity(fc->mesh, face))
+  if (isOnDomainBoundary(fc->mesh, face))
     PCU_ALWAYS_ASSERT(up.n == 1);
   else
     PCU_ALWAYS_ASSERT(up.n == 2);
diff --git a/ree/reeEstimateError.cc b/ree/reeEstimateError.cc
index 308941f07..0592f09ee 100644
--- a/ree/reeEstimateError.cc
+++ b/ree/reeEstimateError.cc
@@ -5,10 +5,7 @@
  * BSD license as described in the LICENSE file in the top-level directory.
  */
 
-#include <apfCavityOp.h>
-#include "apfElement.h"
-#include "crv.h"
-#include "crvShape.h"
+#include <apfElement.h>
 #include "ree.h"
 
 namespace ree {
@@ -207,7 +204,7 @@ static void getEssentialElementNDDofs(apf::Mesh* mesh, apf::MeshEntity* e,
   PCU_ALWAYS_ASSERT(nedges == 6);
   for (int i = 0; i < nedges; i++) {
     int nodesOnEdge = fs->countNodesOn(mesh->getType(edges[i]));
-    if ( crv::isBoundaryEntity(mesh, edges[i]) ) {
+    if ( isOnDomainBoundary(mesh, edges[i]) ) {
       for (int n = 0; n < nodesOnEdge; n++) {
         marker_list[(i*nodesOnEdge)+n] = -1;
       }
@@ -220,7 +217,7 @@ static void getEssentialElementNDDofs(apf::Mesh* mesh, apf::MeshEntity* e,
   PCU_ALWAYS_ASSERT(nfaces == 4);
   for (int i = 0; i < nfaces; i++) {
     int nodesOnFace = fs->countNodesOn(mesh->getType(faces[i]));
-    if ( crv::isBoundaryEntity(mesh, faces[i]) ) {
+    if ( isOnDomainBoundary(mesh, faces[i]) ) {
       for (int n = 0; n < nodesOnFace; n++) {
         marker_list[(nodesOnEdges + i*nodesOnFace)+n] = -1;
       }
diff --git a/ree/reeResidualFunctionals.cc b/ree/reeResidualFunctionals.cc
index bf8d22b3a..741291eb5 100644
--- a/ree/reeResidualFunctionals.cc
+++ b/ree/reeResidualFunctionals.cc
@@ -507,6 +507,13 @@ apf::Vector3 computeFaceOutwardNormal(apf::Mesh* m,
   return n;
 }
 
+// TODO this needs generalisation for cases with internal model boundaries
+// (i.e., interfaces)
+bool isOnDomainBoundary(apf::Mesh* m, apf::MeshEntity* e)
+{
+  return m->getModelType(m->toModel(e)) < m->getDimension();
+}
+
 /*
  * computes local flux integral restricted to
  * an edge of a tet element

From 4020c21ad0b368fba559b5ebe23a294b9e279ee9 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Sun, 6 Dec 2020 13:42:40 -0500
Subject: [PATCH 300/555] Renames the residual test for consistency

---
 test/testing.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/testing.cmake b/test/testing.cmake
index fc1904918..21cdfc28b 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -294,7 +294,7 @@ mpi_test(tet_serial 1
   "${MDIR}/pipe.${GXT}"
   "pipe.smb"
   "tet.smb")
-mpi_test(residualErrorEstimation_test 1
+mpi_test(test_residual_error_estimate 1
   ./residualErrorEstimation_test
   "${MESHES}/electromagnetic/fichera.x_t"
   "${MESHES}/electromagnetic/fichera_1k.smb")

From 0c48da05c9e2ea0326088d2ba4f74bbac8d2cea3 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Sun, 6 Dec 2020 13:43:41 -0500
Subject: [PATCH 301/555] Replaces couts with lion_oprint

---
 test/residualErrorEstimation_test.cc | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/test/residualErrorEstimation_test.cc b/test/residualErrorEstimation_test.cc
index cf8e5bfcf..9cbd63491 100644
--- a/test/residualErrorEstimation_test.cc
+++ b/test/residualErrorEstimation_test.cc
@@ -14,9 +14,6 @@
 #endif
 #include <pcu_util.h>
 
-#include <iostream>
-using namespace std;
-
 void E_exact(const apf::Vector3 &x, apf::Vector3& E);
 double computeElementExactError(apf::Mesh* mesh, apf::MeshEntity* e,
   apf::Field* f);
@@ -83,10 +80,10 @@ int main(int argc, char** argv)
   avg_exact_error /= m->count(3);
   avg_computed_error /= m->count(3);
 
-  cout << "Max Exact Error: " << max_exact_error << endl;
-  cout << "Average Exact Error: " << avg_exact_error << endl;
-  cout << "Max Computed Error: " << max_computed_error << endl;
-  cout << "Average Computed Error: " << avg_computed_error << endl;
+  lion_oprint(1, "Max     Exact Error: %e\n", max_exact_error);
+  lion_oprint(1, "Average Exact Error: %e\n", avg_exact_error);
+  lion_oprint(1, "Max     Computed Error: %e\n", max_computed_error);
+  lion_oprint(1, "Average Computed Error: %e\n", avg_computed_error);
 
   apf::destroyField(residual_error_field);
   apf::destroyField(exact_error_field);

From 547d012bad55274be4ac7d12a6155e970ddc9947 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Sun, 6 Dec 2020 14:20:27 -0500
Subject: [PATCH 302/555] Removes the remaining dependence on crv

---
 ree/CMakeLists.txt            |  2 +-
 ree/reeResidualFunctionals.cc | 14 ++++++--------
 2 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/ree/CMakeLists.txt b/ree/CMakeLists.txt
index c574820e8..e7b935540 100644
--- a/ree/CMakeLists.txt
+++ b/ree/CMakeLists.txt
@@ -25,7 +25,7 @@ target_include_directories(ree INTERFACE
     )
 
 # Link this package to these libraries
-target_link_libraries(ree PUBLIC apf pcu crv)
+target_link_libraries(ree PUBLIC apf pcu)
 
 scorec_export_library(ree)
 
diff --git a/ree/reeResidualFunctionals.cc b/ree/reeResidualFunctionals.cc
index 741291eb5..9f5d33f97 100644
--- a/ree/reeResidualFunctionals.cc
+++ b/ree/reeResidualFunctionals.cc
@@ -7,8 +7,6 @@
 #include <algorithm>
 #include <apfCavityOp.h>
 #include "apfElement.h"
-#include "crv.h"
-#include "crvShape.h"
 #include "ree.h"
 
 namespace ree {
@@ -87,7 +85,7 @@ static void startEdgePatch(EdgePatch* ep, apf::MeshEntity* e)
   ep->tets.clear();
   ep->faces.clear();
   ep->entity = e;
-  ep->isOnBdry = crv::isBoundaryEntity(ep->mesh, ep->entity);
+  ep->isOnBdry = isOnDomainBoundary(ep->mesh, ep->entity);
 }
 
 static void addEntityToPatch(EdgePatch* ep, apf::MeshEntity* e)
@@ -133,7 +131,7 @@ static void assembleEdgePatchLHS(EdgePatch* ep)
 {
   int ne = ep->tets.size();
   int nf = ep->faces.size();
-  if( crv::isBoundaryEntity(ep->mesh, ep->entity) ) {
+  if( isOnDomainBoundary(ep->mesh, ep->entity) ) {
     ep->T.resize(ne+nf, ne+nf);
     ep->T.zero();
     for (int i = 0; i < nf; i++)
@@ -145,7 +143,7 @@ static void assembleEdgePatchLHS(EdgePatch* ep)
     ep->T(ne+nf-1, ne-1) = 1.; ep->T(ne+nf-1, ne) = 1.;
     ep->T(ne-1, ne+nf-1) = 1.; ep->T(ne, ne+nf-1) = 1.;
   }
-  else if( ! crv::isBoundaryEntity(ep->mesh, ep->entity) ) {
+  else if( ! isOnDomainBoundary(ep->mesh, ep->entity) ) {
     ep->A.resize(ne, nf);
     ep->A.zero();
     for (int i = 0; i < ne-1; i++) {
@@ -540,7 +538,7 @@ static double getLocalFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet)
     apf::Up up;
     apf::MeshEntity* currentFace = patchFaces[i];
     ep->mesh->getUp(currentFace, up);
-    if (crv::isBoundaryEntity(ep->mesh, currentFace))
+    if (isOnDomainBoundary(ep->mesh, currentFace))
       PCU_ALWAYS_ASSERT( up.n == 1);
     else
       PCU_ALWAYS_ASSERT( up.n == 2);
@@ -683,7 +681,7 @@ static void getOrderedTetsandFaces(apf::Mesh* mesh, apf::MeshEntity* edge,
 {
   tets.clear();
   faces.clear();
-  if( ! crv::isBoundaryEntity(mesh, edge) ) {
+  if( ! isOnDomainBoundary(mesh, edge) ) {
     apf::MeshEntity* currentFace = mesh->getUpward(edge, 0);
     apf::Up up;
     mesh->getUp(currentFace, up);
@@ -714,7 +712,7 @@ static void getOrderedTetsandFaces(apf::Mesh* mesh, apf::MeshEntity* edge,
     mesh->getUp(edge, up);
     apf::MeshEntity* firstFace;
     for (int i = 0; i < up.n; i++) {
-      if ( crv::isBoundaryEntity(mesh, up.e[i]) ) {
+      if ( isOnDomainBoundary(mesh, up.e[i]) ) {
         firstFace = up.e[i]; break;
       }
     }

From a0f05d1f9fe57279ea66a58b27f82a70a00c4054 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Sun, 6 Dec 2020 15:08:36 -0500
Subject: [PATCH 303/555] Reverts "Adds residual based error estimation test"
 partially

adds back the deleted sub-module pumi-meshes

This reverts commit 7bc29a3d2aba20bb04dd006e52cf25e52310491f.

Conflicts:
	test/CMakeLists.txt
	test/residualErrorEstimation_test.cc
---
 pumi-meshes | 1 +
 1 file changed, 1 insertion(+)
 create mode 160000 pumi-meshes

diff --git a/pumi-meshes b/pumi-meshes
new file mode 160000
index 000000000..94870b0ce
--- /dev/null
+++ b/pumi-meshes
@@ -0,0 +1 @@
+Subproject commit 94870b0ce0f9d953e985ed95c47c5b65246ae361

From c826c755922885138954a5b127d5542b5c34018e Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Sun, 6 Dec 2020 19:14:15 -0500
Subject: [PATCH 304/555] Updates the pumi-meshes submodule

---
 pumi-meshes | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pumi-meshes b/pumi-meshes
index 94870b0ce..4d293fe6d 160000
--- a/pumi-meshes
+++ b/pumi-meshes
@@ -1 +1 @@
-Subproject commit 94870b0ce0f9d953e985ed95c47c5b65246ae361
+Subproject commit 4d293fe6d329ef255ea90b08ed97884219a4547f

From b6d2d75255cd669eaa86dc1c8d3cac8f9fc2d7d8 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Mon, 7 Dec 2020 09:55:25 -0500
Subject: [PATCH 305/555] clean ups

---
 apf/apf.cc              | 39 +++++++++--------------------
 ree/reeCorrectedFlux.cc |  1 +
 ree/reeEstimateError.cc | 55 ++++++++++++++++++++---------------------
 3 files changed, 40 insertions(+), 55 deletions(-)

diff --git a/apf/apf.cc b/apf/apf.cc
index d33c484ae..0d83ddfdd 100644
--- a/apf/apf.cc
+++ b/apf/apf.cc
@@ -463,33 +463,18 @@ void getVectorShapeValues(Element* e, Vector3 const& local,
   NewArray<Vector3> vvals(values.size());
   e->getShape()->getVectorValues(e->getMesh(), e->getEntity(), local, vvals);
 
-  // Perform Piola transformation
-  if( e->getDimension() == e->getMesh()->getDimension() ) // i.e. J is square
-  {
-    apf::Matrix3x3 Jinv;
-    apf::getJacobianInv( e->getParent(), local, Jinv );
-    apf::Matrix3x3 JinvT = apf::transpose(Jinv);
-
-    // u(x_hat) * J(x_hat)^{-1}
-    for( size_t i = 0; i < values.size(); i++ ) {
-      for ( int j = 0; j < 3; j++ ) {
-        values[i][j] = 0.;
-        for ( int k = 0; k < 3; k++ )
-          values[i][j] += vvals[i][k] * JinvT[k][j];
-      }
-    }
-  }
-  else
-  {
-    apf::Matrix3x3 Jinv;
-    apf::getJacobianInv( e->getParent(), local, Jinv );
-    apf::Matrix3x3 JinvT = apf::transpose(Jinv);
-    for( size_t i = 0; i < values.size(); i++ ) {
-      for ( int j = 0; j < 3; j++ ) {
-        values[i][j] = 0.;
-        for ( int k = 0; k < 2; k++ )
-          values[i][j] += vvals[i][k] * JinvT[k][j];
-      }
+  apf::Matrix3x3 Jinv;
+  apf::getJacobianInv( e->getParent(), local, Jinv );
+  apf::Matrix3x3 JinvT = apf::transpose(Jinv);
+
+  // Perform Piola transformation - u(x_hat) * J(x_hat)^{-1}
+  int d = 0;
+  (e->getDimension() == e->getMesh()->getDimension()) ? d = 3 : d = 2;
+  for( size_t i = 0; i < values.size(); i++ ) {
+    for ( int j = 0; j < 3; j++ ) {
+      values[i][j] = 0.;
+      for ( int k = 0; k < d; k++ )
+        values[i][j] += vvals[i][k] * JinvT[k][j];
     }
   }
 }
diff --git a/ree/reeCorrectedFlux.cc b/ree/reeCorrectedFlux.cc
index c8c11f1e9..492b693c8 100644
--- a/ree/reeCorrectedFlux.cc
+++ b/ree/reeCorrectedFlux.cc
@@ -41,6 +41,7 @@ static void setupCorrectFlux(
 {
   cf->mesh = apf::getMesh(f);
   cf->dim = cf->mesh->getDimension();
+  PCU_ALWAYS_ASSERT(cf->dim == 3);
   cf->order = f->getShape()->getOrder();
   cf->orderp1 = cf->order+1;
   cf->ef = f;
diff --git a/ree/reeEstimateError.cc b/ree/reeEstimateError.cc
index 0592f09ee..849824b0e 100644
--- a/ree/reeEstimateError.cc
+++ b/ree/reeEstimateError.cc
@@ -262,9 +262,9 @@ static void eliminateDBCs(
   int num_uness_dofs = uness_dofs.size();
 
   // 1. Remove rows of A corresponding to
-	// ess_dofs by copying into Anew
+  // ess_dofs by copying into Anew
   Anew.resize(num_uness_dofs, num_uness_dofs);
-	for(int rr = 0; rr < num_uness_dofs; rr++) {
+  for(int rr = 0; rr < num_uness_dofs; rr++) {
     int i = uness_dofs[rr];
     for(int cc = 0; cc < num_uness_dofs; cc++) {
       int j = uness_dofs[cc];
@@ -272,14 +272,14 @@ static void eliminateDBCs(
     }
   }
 
-	// 2. Assemble new B
-	Bnew.resize(num_uness_dofs);
+  // 2. Assemble new B
+  Bnew.resize(num_uness_dofs);
   for(int i = 0; i < num_uness_dofs; i++) {
     Bnew(i) = B(uness_dofs[i]);
   }
 
   if (num_ess_dofs > 0) {
-	  // 3. Subtract from Bnew: (Bnew -= Ae*Xe)
+    // 3. Subtract from Bnew: (Bnew -= Ae*Xe)
     mth::Matrix<double> Ae(num_uness_dofs, num_ess_dofs);
     for(int rr = 0; rr < num_uness_dofs; rr++) {
       int i = uness_dofs[rr];
@@ -353,16 +353,15 @@ static double computeL2Error(apf::Mesh* mesh, apf::MeshEntity* e,
 
 apf::Field* computeErrorField(apf::Field* ef, apf::Field* correctedFlux)
 {
-
-	// 1. Create per-element SCALAR error field
+  // 1. Create per-element SCALAR error field
   apf::Field* error_field = apf::createIPField(
-		apf::getMesh(ef), "residual_error_field", apf::SCALAR, 1);
+      apf::getMesh(ef), "residual_error_field", apf::SCALAR, 1);
 
   // 2. Create p+1 order tet ND field
   int order = ef->getShape()->getOrder();
   int orderp1 = order+1;
   apf::Field* efp1 = apf::createField(apf::getMesh(ef),
-	 "orderp1_nedelec_field", apf::SCALAR, apf::getNedelec(orderp1));
+      "orderp1_nedelec_field", apf::SCALAR, apf::getNedelec(orderp1));
   apf::zeroField(efp1);
 
   // 2. iterate over all elements of the mesh
@@ -398,22 +397,22 @@ apf::Field* computeErrorField(apf::Field* ef, apf::Field* correctedFlux)
         apf::getMesh(efp1), el, efp1, ess_dofs, uness_dofs);
 
     // 2(g). eliminate Dirichlet (Essential) Boundary Conditions
-		mth::Vector<double> X, Bnew;
+    mth::Vector<double> X, Bnew;
     mth::Matrix<double> Anew;
-		X.resize(B.size());
-		X.zero(); // initialize X with exact DBC (e = 0.0)
-		eliminateDBCs(A, X, B, ess_dofs, uness_dofs, Anew, Bnew);
-
-		// 2(h). Solve the reduced system
-	  mth::Matrix<double> Q, R;
-		mth::decomposeQR(Anew, Q, R);
-		mth::Vector<double> Xnew;
-		mth::solveFromQR(Q, R, Bnew, Xnew);
-
-		// 2(i). Recover the solution
-		mth::Vector<double> error_dofs(B.size());
+    X.resize(B.size());
+    X.zero(); // initialize X with exact DBC (e = 0.0)
+    eliminateDBCs(A, X, B, ess_dofs, uness_dofs, Anew, Bnew);
+
+    // 2(h). Solve the reduced system
+    mth::Matrix<double> Q, R;
+    mth::decomposeQR(Anew, Q, R);
+    mth::Vector<double> Xnew;
+    mth::solveFromQR(Q, R, Bnew, Xnew);
+
+    // 2(i). Recover the solution
+    mth::Vector<double> error_dofs(B.size());
     for(unsigned int i = 0; i < ess_dofs.size(); i++) {
-    	int index = ess_dofs[i];
+      int index = ess_dofs[i];
       error_dofs(index) = X(index);
     }
     for(unsigned int i = 0; i < uness_dofs.size(); i++) {
@@ -421,14 +420,14 @@ apf::Field* computeErrorField(apf::Field* ef, apf::Field* correctedFlux)
       error_dofs(index) = Xnew(i);
     }
 
-		// 2(j). Compute L2 Norm Error
-		double l2_error = computeL2Error(apf::getMesh(ef), el, efp1, error_dofs);
-		apf::setScalar(error_field, el, 0, l2_error);
+    // 2(j). Compute L2 Norm Error
+    double l2_error = computeL2Error(apf::getMesh(ef), el, efp1, error_dofs);
+    apf::setScalar(error_field, el, 0, l2_error);
   }
   apf::getMesh(ef)->end(it);
-	apf::destroyField(efp1);
+  apf::destroyField(efp1);
 
-	return error_field;
+  return error_field;
 }
 
 apf::Field* estimateError(apf::Field* f)

From b4426c6252c54e26fbdc058a0a7d01f53b08f3f0 Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Mon, 7 Dec 2020 10:11:09 -0500
Subject: [PATCH 306/555] further clean ups

---
 README.md          |  1 +
 mth/mth_def.h      |  4 ++--
 test/testing.cmake | 10 ++++++----
 3 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 7b21e5acb..e9a406082 100644
--- a/README.md
+++ b/README.md
@@ -26,6 +26,7 @@ For more information, start at our
 * MTH: Math containers and routines
 * CRV: Support for curved meshes with Bezier Shapes
 * PYCORE: Python Wrappers (see python_wrappers/README.md for build instructions)
+* REE: Residual based implicit error estimator
 
 ### How do I get set up? ###
 
diff --git a/mth/mth_def.h b/mth/mth_def.h
index 393c2f442..11624f3cc 100644
--- a/mth/mth_def.h
+++ b/mth/mth_def.h
@@ -61,8 +61,8 @@ void transpose(Matrix<T,M,N> const& a,
   unsigned n = a.rows();
   b.resize(m, n);
   for (unsigned i=0; i < m; ++i)
-  for (unsigned j=0; j < n; ++j)
-    b(i,j) = a(j,i);
+    for (unsigned j=0; j < n; ++j)
+      b(i,j) = a(j,i);
 }
 
 template <class T>
diff --git a/test/testing.cmake b/test/testing.cmake
index df93c07e0..4e5845ec4 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -298,10 +298,12 @@ mpi_test(tet_serial 1
   "${MDIR}/pipe.${GXT}"
   "pipe.smb"
   "tet.smb")
-mpi_test(test_residual_error_estimate 1
-  ./residualErrorEstimation_test
-  "${MESHES}/electromagnetic/fichera.x_t"
-  "${MESHES}/electromagnetic/fichera_1k.smb")
+if(ENABLE_SIMMETRIX)
+  mpi_test(test_residual_error_estimate 1
+    ./residualErrorEstimation_test
+    "${MESHES}/electromagnetic/fichera.x_t"
+    "${MESHES}/electromagnetic/fichera_1k.smb")
+endif()
 if(PCU_COMPRESS)
   set(MESHFILE "bz2:pipe_2_.smb")
 else()

From 0c2cc97bf3760ba1a87853b43d1cdff4511ce3ba Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Mon, 7 Dec 2020 11:03:16 -0700
Subject: [PATCH 307/555] wedges added

---
 phasta/phPartition.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/phasta/phPartition.cc b/phasta/phPartition.cc
index 6c8ab423f..6c05048c0 100644
--- a/phasta/phPartition.cc
+++ b/phasta/phPartition.cc
@@ -31,6 +31,7 @@ void setWeight(apf::Mesh* m, apf::MeshTag* tag, int dim) {
     if(dimEnt==3) {
       nverts = m->getDownward(e, 0, verts);
       if(nverts==8) w=6.0;
+      if(nverts==6) w=3.0; 
     }
     m->setDoubleTag(e, tag, &w);
   }

From cc648141e2ab4720e7036e898b42160eefb46e4e Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Tue, 8 Dec 2020 13:33:22 -0500
Subject: [PATCH 308/555] removes ternary statement and replaces tabs with
 spaces

---
 ree/reeEstimateError.cc | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/ree/reeEstimateError.cc b/ree/reeEstimateError.cc
index 849824b0e..3c7ee200f 100644
--- a/ree/reeEstimateError.cc
+++ b/ree/reeEstimateError.cc
@@ -231,10 +231,8 @@ static void getEssentialElementNDDofs(apf::Mesh* mesh, apf::MeshEntity* e,
   }
 
   // use marker list to get ess_dofs list
-  ess_dofs.allocated() ? ess_dofs.resize(num_marked)
-    : ess_dofs.allocate(num_marked);
-  uness_dofs.allocated() ? uness_dofs.resize(nedofs-num_marked)
-    : uness_dofs.allocate(nedofs-num_marked);
+  ess_dofs.resize(num_marked);
+  uness_dofs.resize(nedofs-num_marked);
   int ess_dof_counter = 0;
   int uness_dof_counter = 0;
   for (int i = 0; i < nedofs; i++) {
@@ -301,13 +299,13 @@ static void eliminateDBCs(
 }
 
 static double computeL2Error(apf::Mesh* mesh, apf::MeshEntity* e,
-	apf::Field* f, mth::Vector<double> const error_dofs)
+  apf::Field* f, mth::Vector<double> const error_dofs)
 {
-	double error = 0.0;
+  double error = 0.0;
 
   apf::FieldShape* fs = f->getShape();
   int type = mesh->getType(e);
-	PCU_ALWAYS_ASSERT(type == apf::Mesh::TET);
+  PCU_ALWAYS_ASSERT(type == apf::Mesh::TET);
   int nd = apf::countElementNodes(fs, type);
   int dim = apf::getDimension(mesh, e);
   double w;
@@ -345,10 +343,10 @@ static double computeL2Error(apf::Mesh* mesh, apf::MeshEntity* e,
   apf::destroyElement(el);
   apf::destroyMeshElement(me);
 
-	if (error < 0.0)
-		error = -error;
+  if (error < 0.0)
+    error = -error;
 
-	return sqrt(error);
+  return sqrt(error);
 }
 
 apf::Field* computeErrorField(apf::Field* ef, apf::Field* correctedFlux)

From 7b22d3f06cc3ea6eb44bfe417000257274182b1b Mon Sep 17 00:00:00 2001
From: Samiullah Malik <maliks2@romulus.scorec.rpi.edu>
Date: Tue, 8 Dec 2020 18:35:18 -0500
Subject: [PATCH 309/555] adds assert for mesh dimension

---
 ree/reeCorrectedFlux.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ree/reeCorrectedFlux.cc b/ree/reeCorrectedFlux.cc
index 492b693c8..634ed5b79 100644
--- a/ree/reeCorrectedFlux.cc
+++ b/ree/reeCorrectedFlux.cc
@@ -284,6 +284,7 @@ class FaceCavityOp : public apf::CavityOp
 apf::Field* computeCorrectedFlux(apf::Field* ef, apf::Field* theta)
 {
   int dim = apf::getMesh(ef)->getDimension();
+  PCU_ALWAYS_ASSERT(dim==3);
   int order = ef->getShape()->getOrder() + 1; // local BVPs require p+1
   int int_order = 2*order-1;
   int nc = 4*3; // 1 flux vector per face

From a7b5a01bea197ddb5c9ce67ccfd2935b92685744 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Mon, 14 Dec 2020 15:15:47 -0500
Subject: [PATCH 310/555] Adds visualizeSizeField to maDBG

Also updates the utility "visualizeSizeField.cc" to use that.
---
 ma/maDBG.cc                 | 129 ++++++++++++++++++++++++++++++++++++
 ma/maDBG.h                  |   8 +++
 test/visualizeAnisoSizes.cc | 121 +--------------------------------
 3 files changed, 140 insertions(+), 118 deletions(-)

diff --git a/ma/maDBG.cc b/ma/maDBG.cc
index 233bb4a10..808bedf3d 100644
--- a/ma/maDBG.cc
+++ b/ma/maDBG.cc
@@ -26,6 +26,7 @@
 #include <cassert>
 #include <stdarg.h>
 
+static double PI = 3.14159265359;
 
 namespace ma_dbg {
 
@@ -256,5 +257,133 @@ void createCavityMesh(ma::Adapt* a,
   createCavityMesh(a, tetsArray, prefix);
 }
 
+static apf::Vector3 getPointOnEllipsoid(
+    apf::Vector3 center,
+    apf::Vector3 abc,
+    apf::Matrix3x3 rotation,
+    double scaleFactor,
+    double u,
+    double v)
+{
+  apf::Vector3 result;
+  result[0] = abc[0] * cos(u) * cos(v);
+  result[1] = abc[1] * cos(u) * sin(v);
+  result[2] = abc[2] * sin(u);
+
+  result = result * scaleFactor;
+
+  result = rotation * result + center;
+  return result;
+}
+
+static void makeEllipsoid(
+    apf::Mesh2* msf,
+    apf::Mesh2* mesh,
+    apf::Field* sizes,
+    apf::Field* frames,
+    apf::MeshEntity* ent,
+    int node,
+    double scaleFactor,
+    int sampleSize[2])
+{
+  // first get the coordinate at node location
+  apf::Vector3 xi;
+  apf::Vector3 center;
+  apf::FieldShape* fs = apf::getShape(sizes);
+  fs->getNodeXi(mesh->getType(ent), node, xi);
+  apf::MeshElement* me = apf::createMeshElement(mesh, ent);
+  apf::mapLocalToGlobal(me, xi, center);
+  apf::destroyMeshElement(me);
+
+
+  // second get the sizes and frames at node
+  apf::Vector3 abc;
+  apf::getVector(sizes, ent, node, abc);
+
+  apf::Matrix3x3 rotation;
+  apf::getMatrix(frames, ent, node, rotation);
+
+
+  double U0 = 0.0;
+  double U1 = 2 * PI;
+  double V0 = -PI/2.;
+  double V1 =  PI/2.;
+  int n = sampleSize[0];
+  int m = sampleSize[1];
+  double dU = (U1 - U0) / (n-1);
+  double dV = (V1 - V0) / (m-1);
+
+  // make the array of vertex coordinates in the physical space
+  std::vector<ma::Vector> ps;
+  for (int j = 0; j < m; j++) {
+    for (int i = 0; i < n; i++) {
+      double u = U0 + i * dU;
+      double v = V0 + j * dV;
+      apf::Vector3 pt = getPointOnEllipsoid(center, abc, rotation, scaleFactor, u, v);
+      ps.push_back(pt);
+    }
+  }
+  // make the vertexes and set the coordinates using the array
+  std::vector<apf::MeshEntity*> vs;
+  for (size_t i = 0; i < ps.size(); i++) {
+    apf::MeshEntity* newVert = msf->createVert(0);
+    msf->setPoint(newVert, 0, ps[i]);
+    vs.push_back(newVert);
+  }
+
+  PCU_ALWAYS_ASSERT(vs.size() == ps.size());
+
+  apf::MeshEntity* v[3];
+  // make the lower/upper t elems
+  for (int i = 0; i < n-1; i++) {
+    for (int j = 0; j < m-1; j++) {
+      // upper triangle
+      v[0] = vs[(i + 0) + n * (j + 0)];
+      v[1] = vs[(i + 0) + n * (j + 1)];
+      v[2] = vs[(i + 1) + n * (j + 0)];
+      apf::buildElement(msf, 0, apf::Mesh::TRIANGLE, v);
+      // upper triangle
+      v[0] = vs[(i + 0) + n * (j + 1)];
+      v[1] = vs[(i + 1) + n * (j + 1)];
+      v[2] = vs[(i + 1) + n * (j + 0)];
+      apf::buildElement(msf, 0, apf::Mesh::TRIANGLE, v);
+    }
+  }
+}
+
+void visualizeSizeField(
+    apf::Mesh2* m,
+    apf::Field* sizes,
+    apf::Field* frames,
+    int sampleSize[2],
+    double userScale,
+    const char* outputPrefix)
+{
+  // create the size-field visualization mesh
+  apf::Mesh2* msf = apf::makeEmptyMdsMesh(gmi_load(".null"), 2, false);
+
+  apf::FieldShape* fs = apf::getShape(sizes);
+  int dim = m->getDimension();
+
+  apf::MeshEntity* ent;
+  apf::MeshIterator* it;
+
+  for (int d = 0; d <= dim; d++) {
+    if (!fs->hasNodesIn(d)) continue;
+    it = m->begin(d);
+    while ( (ent = m->iterate(it)) ) {
+      int type = m->getType(ent);
+      int non = fs->countNodesOn(type);
+      for (int n = 0; n < non; n++)
+	makeEllipsoid(msf, m, sizes, frames, ent, n, userScale , sampleSize);
+    }
+    m->end(it);
+  }
+
+  apf::writeVtkFiles(outputPrefix, msf);
+
+  msf->destroyNative();
+  apf::destroyMesh(msf);
+}
 
 }
diff --git a/ma/maDBG.h b/ma/maDBG.h
index 8e1c7bc64..751450aa3 100644
--- a/ma/maDBG.h
+++ b/ma/maDBG.h
@@ -60,5 +60,13 @@ void createCavityMesh(ma::Adapt* a,
     ma::EntitySet& tets,
     const char* prefix);
 
+void visualizeSizeField(
+    apf::Mesh2* m,
+    apf::Field* sizes,
+    apf::Field* frames,
+    int smapleSize[2],
+    double userScale,
+    const char* OutputPrefix);
+
 }
 #endif
diff --git a/test/visualizeAnisoSizes.cc b/test/visualizeAnisoSizes.cc
index 4f9c75e02..00a1c2178 100644
--- a/test/visualizeAnisoSizes.cc
+++ b/test/visualizeAnisoSizes.cc
@@ -3,6 +3,7 @@
 #include <maStats.h>
 #include <gmi_mesh.h>
 #include <gmi_null.h>
+#include <maDBG.h>
 #include <PCU.h>
 #include <lionPrint.h>
 
@@ -25,27 +26,10 @@
 #include <errno.h> /* for checking the error from mkdir */
 // ===============================
 
-static double PI = 3.14159265359;
-
 void safe_mkdir(const char* path);
 double getLargetsSize(
     apf::Mesh2* m,
     apf::Field* sizes);
-apf::Vector3 getPointOnEllipsoid(
-    apf::Vector3 center,
-    apf::Vector3 abc,
-    apf::Matrix3x3 rotation,
-    double scaleFactor,
-    double u,
-    double v);
-void makeEllipsoid(
-    apf::Mesh2* msf,
-    apf::Mesh2* m,
-    apf::Field* sizes,
-    apf::Field* frames,
-    apf::MeshEntity* vert,
-    double scaleFactor,
-    int sampleSize[2]);
 void visualizeSizeField(
     const char* modelFile,
     const char* meshFile,
@@ -140,94 +124,6 @@ double getLargetsSize(
   return maxSize;
 }
 
-apf::Vector3 getPointOnEllipsoid(
-    apf::Vector3 center,
-    apf::Vector3 abc,
-    apf::Matrix3x3 rotation,
-    double scaleFactor,
-    double u,
-    double v)
-{
-  apf::Vector3 result;
-  result[0] = abc[0] * cos(u) * cos(v);
-  result[1] = abc[1] * cos(u) * sin(v);
-  result[2] = abc[2] * sin(u);
-
-  result = result * scaleFactor;
-
-  result = rotation * result + center;
-  return result;
-}
-
-
-void makeEllipsoid(
-    apf::Mesh2* msf,
-    apf::Mesh2* mesh,
-    apf::Field* sizes,
-    apf::Field* frames,
-    apf::MeshEntity* vert,
-    double scaleFactor,
-    int sampleSize[2])
-{
-
-  apf::Vector3 center;
-  mesh->getPoint(vert, 0, center);
-
-  apf::Vector3 abc;
-  apf::getVector(sizes, vert, 0, abc);
-
-  apf::Matrix3x3 rotation;
-  apf::getMatrix(frames, vert, 0, rotation);
-
-
-  double U0 = 0.0;
-  double U1 = 2 * PI;
-  double V0 = -PI/2.;
-  double V1 =  PI/2.;
-  int n = sampleSize[0];
-  int m = sampleSize[1];
-  double dU = (U1 - U0) / (n-1);
-  double dV = (V1 - V0) / (m-1);
-
-  // make the array of vertex coordinates in the physical space
-  std::vector<ma::Vector> ps;
-  for (int j = 0; j < m; j++) {
-    for (int i = 0; i < n; i++) {
-      double u = U0 + i * dU;
-      double v = V0 + j * dV;
-      apf::Vector3 pt = getPointOnEllipsoid(center, abc, rotation, scaleFactor, u, v);
-      ps.push_back(pt);
-    }
-  }
-  // make the vertexes and set the coordinates using the array
-  std::vector<apf::MeshEntity*> vs;
-  for (size_t i = 0; i < ps.size(); i++) {
-    apf::MeshEntity* newVert = msf->createVert(0);
-    msf->setPoint(newVert, 0, ps[i]);
-    vs.push_back(newVert);
-  }
-
-  PCU_ALWAYS_ASSERT(vs.size() == ps.size());
-
-  apf::MeshEntity* v[3];
-  // make the lower/upper t elems
-  for (int i = 0; i < n-1; i++) {
-    for (int j = 0; j < m-1; j++) {
-      // upper triangle
-      v[0] = vs[(i + 0) + n * (j + 0)];
-      v[1] = vs[(i + 0) + n * (j + 1)];
-      v[2] = vs[(i + 1) + n * (j + 0)];
-      apf::buildElement(msf, 0, apf::Mesh::TRIANGLE, v);
-      // upper triangle
-      v[0] = vs[(i + 0) + n * (j + 1)];
-      v[1] = vs[(i + 1) + n * (j + 1)];
-      v[2] = vs[(i + 1) + n * (j + 0)];
-      apf::buildElement(msf, 0, apf::Mesh::TRIANGLE, v);
-    }
-  }
-}
-
-
 void visualizeSizeField(
     const char* modelFile,
     const char* meshFile,
@@ -290,23 +186,12 @@ void visualizeSizeField(
 
   m->verify();
 
-  // create the size-field visualization mesh
-  apf::Mesh2* msf = apf::makeEmptyMdsMesh(gmi_load(".null"), 2, false);
-
-  apf::MeshEntity* vert;
-  apf::MeshIterator* it = m->begin(0);
-  while ( (vert = m->iterate(it)) )
-    if (m->isOwned(vert))
-      makeEllipsoid(msf, m, sizes, frames, vert, userScale , sampleSize);
-  m->end(it);
-
   std::stringstream ss;
   ss << outputPrefix << "/size_field_vis";
-  apf::writeVtkFiles(ss.str().c_str(), msf);
+  ma_dbg::visualizeSizeField(
+      m, sizes, frames, sampleSize, userScale, ss.str().c_str());
   ss.str("");
 
-  msf->destroyNative();
-  apf::destroyMesh(msf);
   m->destroyNative();
   apf::destroyMesh(m);
 }

From a08cbe0812077e47a4c1cc42bce2497997b17161 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Tue, 15 Dec 2020 08:39:08 -0500
Subject: [PATCH 311/555] remove coverity link - not maintained

---
 README.md | 2 --
 1 file changed, 2 deletions(-)

diff --git a/README.md b/README.md
index e9a406082..0c0f0c714 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,5 @@
 # SCOREC Core #
 
-[![Coverity Scan Build Status](https://scan.coverity.com/projects/6698/badge.svg)](https://scan.coverity.com/projects/scorec-core)
-
 The SCOREC Core is a set of C/C++ libraries for unstructured mesh
 simulations on supercomputers.
 

From 391702be378d84fe3a88c491c37be8e5189fa039 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Sat, 2 Jan 2021 08:50:50 -0700
Subject: [PATCH 312/555] this was a path KEJ was proceeding along to get
 triface into wedge boundary element string but it also pulled into interior
 element string. Revert and attempt to fix in convert instead

---
 phasta/phBlock.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/phasta/phBlock.cc b/phasta/phBlock.cc
index 441847e7a..df7eedc21 100644
--- a/phasta/phBlock.cc
+++ b/phasta/phBlock.cc
@@ -246,9 +246,11 @@ std::string getElementType
   ,"tetrahedron "
   ,"hexahedron "
   ,"wedge "
+  ,"wedge triface"
   ,"wedge quadface "
   ,"pyramid "
-  ,"pyramid triface "};
+  ,"pyramid triface "
+  ,"pyramid quadface"};
   return typeTable[elementType];
 }
 

From a3ee8c9d169ea00f6f36562279e99b04d472a95e Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Thu, 7 Jan 2021 22:32:05 -0700
Subject: [PATCH 313/555] not pretty but renamed wedge to wedge triface and it
 may just work

---
 phasta/phBlock.cc | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/phasta/phBlock.cc b/phasta/phBlock.cc
index df7eedc21..573532e6d 100644
--- a/phasta/phBlock.cc
+++ b/phasta/phBlock.cc
@@ -245,12 +245,10 @@ std::string getElementType
   {NULL
   ,"tetrahedron "
   ,"hexahedron "
-  ,"wedge "
-  ,"wedge triface"
+  ,"wedge triface" // this will push this into the interior element as well but have to propagate
   ,"wedge quadface "
-  ,"pyramid "
-  ,"pyramid triface "
-  ,"pyramid quadface"};
+  ,"pyramid quadface"
+  ,"pyramid triface "};
   return typeTable[elementType];
 }
 

From 14325ad4f5c486f935b7cff85259cfe8edde7b28 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Wed, 27 Jan 2021 10:23:46 -0800
Subject: [PATCH 314/555] fixed weights such that better than 1.1 nodal
 imbalance obtained on > 7k parts with wedge-tet mesh.  Also has a flag to 
 textCoord to request (1) or not (0) coords.<part> files.

---
 apf/apfConvertTags.h  | 2 +-
 phasta/phCook.cc      | 1 +
 phasta/phGeomBC.cc    | 2 ++
 phasta/phInput.cc     | 2 ++
 phasta/phInput.h      | 1 +
 phasta/phOutput.cc    | 1 +
 phasta/phOutput.h     | 1 +
 phasta/phPartition.cc | 2 ++
 8 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/apf/apfConvertTags.h b/apf/apfConvertTags.h
index 3d5a0e1ea..eff252129 100644
--- a/apf/apfConvertTags.h
+++ b/apf/apfConvertTags.h
@@ -18,7 +18,7 @@ namespace {
   template <class T> inline
   apf::MeshTag* createTag(apf::Mesh*,
       const char*, const int) {
-    exit(EXIT_FAILURE);
+//    exit(EXIT_FAILURE);
     return 0;
   }
   
diff --git a/phasta/phCook.cc b/phasta/phCook.cc
index a085e4272..76f9ddf2e 100644
--- a/phasta/phCook.cc
+++ b/phasta/phCook.cc
@@ -136,6 +136,7 @@ namespace chef {
 namespace ph {
   void checkBalance(apf::Mesh2* m, ph::Input& in) {
     /* check if balancing was requested */
+      Parma_PrintPtnStats(m, "postSplit", false);
       if (in.prePhastaBalanceMethod != "none" && PCU_Comm_Peers() > 1)
         ph::balance(in,m);
   }
diff --git a/phasta/phGeomBC.cc b/phasta/phGeomBC.cc
index fcb3ea55b..84be1e226 100644
--- a/phasta/phGeomBC.cc
+++ b/phasta/phGeomBC.cc
@@ -368,6 +368,7 @@ void writeGeomBC(Output& o, std::string path, int timestep)
   int npts=params[0];
   char coordfilename[64];
   //bzero((void*)coordfilename,64);
+ if (o.txtCoord == 1) {
   int rank = PCU_Comm_Self() + 1;
   sprintf(coordfilename, "coords.%d",rank);
   FILE* fc = fopen(coordfilename, "w");
@@ -381,6 +382,7 @@ void writeGeomBC(Output& o, std::string path, int timestep)
     fprintf ( fc, "%.15E,%.15E,%.15E,\n", x,y,z);
   }
   fclose(fc);
+  }
 
   writeInt(f, "number of processors", PCU_Comm_Peers());
   writeInt(f, "size of ilwork array", o.nlwork);
diff --git a/phasta/phInput.cc b/phasta/phInput.cc
index c7d23524e..167de61a4 100644
--- a/phasta/phInput.cc
+++ b/phasta/phInput.cc
@@ -83,6 +83,7 @@ static void setDefaults(Input& in)
   in.nRigidBody = 0;
   in.nRBParam = 12;
   in.gradingFactor = 1.25;
+  in.txtCoord = 0; // write coords.<part> in run directory if 1
   in.spanAvg = 0; // prepare and write spanwise average arrays
   in.nfathers = 0; // number of father nodes (# of pts in xy (2D) plane for z averaging)
   in.nsons = 0; // number of sons for each father (constant nz for each father ONLY for ijk grids)
@@ -167,6 +168,7 @@ static void formMaps(Input& in, StringMap& stringMap, IntMap& intMap, DblMap& db
   intMap["nRigidBody"] = &in.nRigidBody;
   intMap["nRBParam"] = &in.nRBParam;
   dblMap["gradingFactor"] = &in.gradingFactor;
+  intMap["textCoord"] = &in.txtCoord;
   intMap["spanAverage"] = &in.spanAvg;
   intMap["nfathers"] = &in.nfathers;
   intMap["nsons"] = &in.nsons;
diff --git a/phasta/phInput.h b/phasta/phInput.h
index 108133085..b4c1de98f 100644
--- a/phasta/phInput.h
+++ b/phasta/phInput.h
@@ -191,6 +191,7 @@ class Input
     double gradingFactor;
     /* Stuff for spanwise averaging */
     int spanAvg;
+    int txtCoord;
     int nfathers;
     int nsons;
 };
diff --git a/phasta/phOutput.cc b/phasta/phOutput.cc
index f3b238183..f5dd4985d 100644
--- a/phasta/phOutput.cc
+++ b/phasta/phOutput.cc
@@ -948,6 +948,7 @@ static void getSpanwiseAverageArrays(Input& in, Output& o) {
     int nnodes = m->count(0); // number of nodes of whole mesh or part??
     /* this will come from the adapt.inp file and is constant for all geombc
      it is the total number of father nodes, nx*ny, and each geombc loads this */
+    o.txtCoord=in.txtCoord;  // controls whether we write coords.part
     int nfather = in.nfathers;
     o.arrays.nfather = nfather;
     /* this will come from the adapt.inp file and is constant for all geombc
diff --git a/phasta/phOutput.h b/phasta/phOutput.h
index ef8c82458..55f4c4e5b 100644
--- a/phasta/phOutput.h
+++ b/phasta/phOutput.h
@@ -144,6 +144,7 @@ struct Output
   ~Output();
   Input* in;
   apf::Mesh* mesh;
+  int txtCoord;
   int nOverlapNodes;
   int nOwnedNodes;
   int nBoundaryElements;
diff --git a/phasta/phPartition.cc b/phasta/phPartition.cc
index 6c05048c0..a5b84732b 100644
--- a/phasta/phPartition.cc
+++ b/phasta/phPartition.cc
@@ -29,6 +29,7 @@ void setWeight(apf::Mesh* m, apf::MeshTag* tag, int dim) {
   while ((e = m->iterate(it))){
     int dimEnt=getDimension(m,e);
     if(dimEnt==3) {
+      w=1.0;
       nverts = m->getDownward(e, 0, verts);
       if(nverts==8) w=6.0;
       if(nverts==6) w=3.0; 
@@ -67,6 +68,7 @@ apf::Migration* getSplitPlan(Input& in, apf::Mesh2* m)
     }
 //    apf::MeshTag* weights = Parma_WeighByMemory(m);
     apf::MeshTag* weights = setWeights(m);
+//    Parma_PrintPtnStats(m, "preSplit",false);
     plan = splitter->split(weights, 1.01, in.splitFactor);
     apf::removeTagFromDimension(m, weights, m->getDimension());
     m->destroyTag(weights);

From 22b697a43c874807fe6466382d93334eeba32b89 Mon Sep 17 00:00:00 2001
From: Avinash <moharana.avin@gmail.com>
Date: Wed, 3 Mar 2021 14:07:46 -0500
Subject: [PATCH 315/555] updates convert Interpolating to Control points.

---
 crv/crv.h           |   3 +
 crv/crvCurveMesh.cc | 134 ++++++++++++++++++++++++++++++--------------
 2 files changed, 94 insertions(+), 43 deletions(-)

diff --git a/crv/crv.h b/crv/crv.h
index 60e8c2c1a..e31e8cf43 100644
--- a/crv/crv.h
+++ b/crv/crv.h
@@ -37,6 +37,9 @@ int getBlendingOrder(const int type);
 /** \brief count invalid elements of the mesh */
 int countNumberInvalidElements(apf::Mesh2* m);
 
+/** \brief converts Interpolating nodes to Control points for a Bezier mesh*/
+void interpolatingToBezier(apf::Mesh2* m);
+
 /** \brief Base Mesh curving object
   \details P is the order, S is the space dimension,
   different from the mesh dimension, used to distinguish between planar 2D
diff --git a/crv/crvCurveMesh.cc b/crv/crvCurveMesh.cc
index 5cc39bbfb..2f4bcc148 100644
--- a/crv/crvCurveMesh.cc
+++ b/crv/crvCurveMesh.cc
@@ -44,6 +44,53 @@ void convertInterpolationPoints(apf::Mesh2* m, apf::MeshEntity* e,
   apf::destroyElement(elem);
 }
 
+void interpolatingToBezier(apf::Mesh2* m)
+{
+  apf::FieldShape * fs = m->getShape();
+  int order = fs->getOrder();
+
+  int md = m->getDimension();
+  int blendingOrder = getBlendingOrder(apf::Mesh::simplexTypes[md]);
+  // go downward, and convert interpolating to control points
+  int startDim = md - (blendingOrder > 0);
+
+  for(int d = startDim; d >= 1; --d){
+    if(!fs->hasNodesIn(d)) continue;
+    int n = fs->getEntityShape(apf::Mesh::simplexTypes[d])->countNodes();
+    int ne = fs->countNodesOn(apf::Mesh::simplexTypes[d]);
+    apf::NewArray<double> c;
+    getBezierTransformationCoefficients(order,
+        apf::Mesh::simplexTypes[d],c);
+    apf::MeshEntity* e;
+    apf::MeshIterator* it = m->begin(d);
+    while ((e = m->iterate(it))){
+      if(m->isOwned(e))
+        convertInterpolationPoints(m,e,n,ne,c);
+    }
+    m->end(it);
+  }
+  // if we have a full representation, we need to place internal nodes on
+  // triangles and tetrahedra
+  for(int d = 2; d <= md; ++d){
+    if(!fs->hasNodesIn(d) ||
+        getBlendingOrder(apf::Mesh::simplexTypes[d])) continue;
+    int n = fs->getEntityShape(apf::Mesh::simplexTypes[d])->countNodes();
+    int ne = fs->countNodesOn(apf::Mesh::simplexTypes[d]);
+    apf::NewArray<double> c;
+    getInternalBezierTransformationCoefficients(m,order,1,
+        apf::Mesh::simplexTypes[d],c);
+    apf::MeshEntity* e;
+    apf::MeshIterator* it = m->begin(d);
+    while ((e = m->iterate(it))){
+      if(!isBoundaryEntity(m,e) && m->isOwned(e))
+        convertInterpolationPoints(m,e,n-ne,ne,c);
+    }
+    m->end(it);
+  }
+  apf::synchronize(m->getCoordinateField());
+
+}
+
 void snapToInterpolate(apf::Mesh2* m, apf::MeshEntity* e, bool isNew)
 {
   PCU_ALWAYS_ASSERT(m->canSnap());
@@ -125,49 +172,50 @@ bool InterpolatingCurver::run()
 
 void BezierCurver::convertInterpolatingToBezier()
 {
-  apf::FieldShape * fs = m_mesh->getShape();
-  int order = fs->getOrder();
-
-  int md = m_mesh->getDimension();
-  int blendingOrder = getBlendingOrder(apf::Mesh::simplexTypes[md]);
-  // go downward, and convert interpolating to control points
-  int startDim = md - (blendingOrder > 0);
-
-  for(int d = startDim; d >= 1; --d){
-    if(!fs->hasNodesIn(d)) continue;
-    int n = fs->getEntityShape(apf::Mesh::simplexTypes[d])->countNodes();
-    int ne = fs->countNodesOn(apf::Mesh::simplexTypes[d]);
-    apf::NewArray<double> c;
-    getBezierTransformationCoefficients(order,
-        apf::Mesh::simplexTypes[d],c);
-    apf::MeshEntity* e;
-    apf::MeshIterator* it = m_mesh->begin(d);
-    while ((e = m_mesh->iterate(it))){
-      if(m_mesh->isOwned(e))
-        convertInterpolationPoints(m_mesh,e,n,ne,c);
-    }
-    m_mesh->end(it);
-  }
-  // if we have a full representation, we need to place internal nodes on
-  // triangles and tetrahedra
-  for(int d = 2; d <= md; ++d){
-    if(!fs->hasNodesIn(d) ||
-        getBlendingOrder(apf::Mesh::simplexTypes[d])) continue;
-    int n = fs->getEntityShape(apf::Mesh::simplexTypes[d])->countNodes();
-    int ne = fs->countNodesOn(apf::Mesh::simplexTypes[d]);
-    apf::NewArray<double> c;
-    getInternalBezierTransformationCoefficients(m_mesh,order,1,
-        apf::Mesh::simplexTypes[d],c);
-    apf::MeshEntity* e;
-    apf::MeshIterator* it = m_mesh->begin(d);
-    while ((e = m_mesh->iterate(it))){
-      if(!isBoundaryEntity(m_mesh,e) && m_mesh->isOwned(e))
-        convertInterpolationPoints(m_mesh,e,n-ne,ne,c);
-    }
-    m_mesh->end(it);
-  }
-
-  synchronize();
+  interpolatingToBezier(m_mesh);
+  /* apf::FieldShape * fs = m_mesh->getShape(); */
+  /* int order = fs->getOrder(); */
+
+  /* int md = m_mesh->getDimension(); */
+  /* int blendingOrder = getBlendingOrder(apf::Mesh::simplexTypes[md]); */
+  /* // go downward, and convert interpolating to control points */
+  /* int startDim = md - (blendingOrder > 0); */
+
+  /* for(int d = startDim; d >= 1; --d){ */
+  /*   if(!fs->hasNodesIn(d)) continue; */
+  /*   int n = fs->getEntityShape(apf::Mesh::simplexTypes[d])->countNodes(); */
+  /*   int ne = fs->countNodesOn(apf::Mesh::simplexTypes[d]); */
+  /*   apf::NewArray<double> c; */
+  /*   getBezierTransformationCoefficients(order, */
+  /*       apf::Mesh::simplexTypes[d],c); */
+  /*   apf::MeshEntity* e; */
+  /*   apf::MeshIterator* it = m_mesh->begin(d); */
+  /*   while ((e = m_mesh->iterate(it))){ */
+  /*     if(m_mesh->isOwned(e)) */
+  /*       convertInterpolationPoints(m_mesh,e,n,ne,c); */
+  /*   } */
+  /*   m_mesh->end(it); */
+  /* } */
+  /* // if we have a full representation, we need to place internal nodes on */
+  /* // triangles and tetrahedra */
+  /* for(int d = 2; d <= md; ++d){ */
+  /*   if(!fs->hasNodesIn(d) || */
+  /*       getBlendingOrder(apf::Mesh::simplexTypes[d])) continue; */
+  /*   int n = fs->getEntityShape(apf::Mesh::simplexTypes[d])->countNodes(); */
+  /*   int ne = fs->countNodesOn(apf::Mesh::simplexTypes[d]); */
+  /*   apf::NewArray<double> c; */
+  /*   getInternalBezierTransformationCoefficients(m_mesh,order,1, */
+  /*       apf::Mesh::simplexTypes[d],c); */
+  /*   apf::MeshEntity* e; */
+  /*   apf::MeshIterator* it = m_mesh->begin(d); */
+  /*   while ((e = m_mesh->iterate(it))){ */
+  /*     if(!isBoundaryEntity(m_mesh,e) && m_mesh->isOwned(e)) */
+  /*       convertInterpolationPoints(m_mesh,e,n-ne,ne,c); */
+  /*   } */
+  /*   m_mesh->end(it); */
+  /* } */
+
+  /* synchronize(); */
 }
 
 bool BezierCurver::run()

From 80b8484c50f1b6040d5f79d251b4999ba09157ec Mon Sep 17 00:00:00 2001
From: Avinash <moharana.avin@gmail.com>
Date: Thu, 18 Mar 2021 11:53:43 -0400
Subject: [PATCH 316/555] Removed commented lines.

---
 crv/crvCurveMesh.cc | 43 -------------------------------------------
 1 file changed, 43 deletions(-)

diff --git a/crv/crvCurveMesh.cc b/crv/crvCurveMesh.cc
index 2f4bcc148..2180856ee 100644
--- a/crv/crvCurveMesh.cc
+++ b/crv/crvCurveMesh.cc
@@ -173,49 +173,6 @@ bool InterpolatingCurver::run()
 void BezierCurver::convertInterpolatingToBezier()
 {
   interpolatingToBezier(m_mesh);
-  /* apf::FieldShape * fs = m_mesh->getShape(); */
-  /* int order = fs->getOrder(); */
-
-  /* int md = m_mesh->getDimension(); */
-  /* int blendingOrder = getBlendingOrder(apf::Mesh::simplexTypes[md]); */
-  /* // go downward, and convert interpolating to control points */
-  /* int startDim = md - (blendingOrder > 0); */
-
-  /* for(int d = startDim; d >= 1; --d){ */
-  /*   if(!fs->hasNodesIn(d)) continue; */
-  /*   int n = fs->getEntityShape(apf::Mesh::simplexTypes[d])->countNodes(); */
-  /*   int ne = fs->countNodesOn(apf::Mesh::simplexTypes[d]); */
-  /*   apf::NewArray<double> c; */
-  /*   getBezierTransformationCoefficients(order, */
-  /*       apf::Mesh::simplexTypes[d],c); */
-  /*   apf::MeshEntity* e; */
-  /*   apf::MeshIterator* it = m_mesh->begin(d); */
-  /*   while ((e = m_mesh->iterate(it))){ */
-  /*     if(m_mesh->isOwned(e)) */
-  /*       convertInterpolationPoints(m_mesh,e,n,ne,c); */
-  /*   } */
-  /*   m_mesh->end(it); */
-  /* } */
-  /* // if we have a full representation, we need to place internal nodes on */
-  /* // triangles and tetrahedra */
-  /* for(int d = 2; d <= md; ++d){ */
-  /*   if(!fs->hasNodesIn(d) || */
-  /*       getBlendingOrder(apf::Mesh::simplexTypes[d])) continue; */
-  /*   int n = fs->getEntityShape(apf::Mesh::simplexTypes[d])->countNodes(); */
-  /*   int ne = fs->countNodesOn(apf::Mesh::simplexTypes[d]); */
-  /*   apf::NewArray<double> c; */
-  /*   getInternalBezierTransformationCoefficients(m_mesh,order,1, */
-  /*       apf::Mesh::simplexTypes[d],c); */
-  /*   apf::MeshEntity* e; */
-  /*   apf::MeshIterator* it = m_mesh->begin(d); */
-  /*   while ((e = m_mesh->iterate(it))){ */
-  /*     if(!isBoundaryEntity(m_mesh,e) && m_mesh->isOwned(e)) */
-  /*       convertInterpolationPoints(m_mesh,e,n-ne,ne,c); */
-  /*   } */
-  /*   m_mesh->end(it); */
-  /* } */
-
-  /* synchronize(); */
 }
 
 bool BezierCurver::run()

From 439d9a856f36d6149e04b5c38aa7739300f1b5d1 Mon Sep 17 00:00:00 2001
From: Jacob Merson <mersoj@rpi.edu>
Date: Fri, 19 Mar 2021 03:57:41 -0400
Subject: [PATCH 317/555] fixes issue #334

fixes the compiler error associated with the strnlen on the wrong
input of strcpy
---
 phasta/phIO.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/phasta/phIO.c b/phasta/phIO.c
index 4352dadbb..11e31f4c5 100644
--- a/phasta/phIO.c
+++ b/phasta/phIO.c
@@ -81,7 +81,7 @@ static int find_header(FILE* f, const char* name, char* found, char header[PH_LI
     tmp[PH_LINE-1] = '\0';
     parse_header(tmp, &hname, &bytes, 0, NULL);
     if (!strncmp(name, hname, strlen(name))) {
-      strncpy(found, hname, strlen(hname));
+      strncpy(found, hname, strlen(found));
       found[strlen(hname)] = '\0';
       return 1;
     }

From 4d73368f3a91e19d33c0bae1246342830cdaae47 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 19 Mar 2021 08:10:11 -0400
Subject: [PATCH 318/555] fix overflow

reported by gcc10
---
 test/xgc_split.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/xgc_split.cc b/test/xgc_split.cc
index deaaddc75..2417362c2 100644
--- a/test/xgc_split.cc
+++ b/test/xgc_split.cc
@@ -76,7 +76,7 @@ int main(int argc, char** argv)
   char without_extension[256];
   snprintf(without_extension,strlen(argv[3])-3,"%s",argv[3]);
 
-  char vtk_fname[32];
+  char vtk_fname[512];
   sprintf(vtk_fname,"%s",without_extension); 
   pumi_mesh_write(m, vtk_fname, "vtk");
 

From 0a2d9ea1d2559b67c233ae9d4582a21997155be7 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 19 Mar 2021 08:16:20 -0400
Subject: [PATCH 319/555] initialize var and assert

#329
---
 ree/reeResidualFunctionals.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ree/reeResidualFunctionals.cc b/ree/reeResidualFunctionals.cc
index 9f5d33f97..f26adea88 100644
--- a/ree/reeResidualFunctionals.cc
+++ b/ree/reeResidualFunctionals.cc
@@ -710,12 +710,13 @@ static void getOrderedTetsandFaces(apf::Mesh* mesh, apf::MeshEntity* edge,
   else {
     apf::Up up;
     mesh->getUp(edge, up);
-    apf::MeshEntity* firstFace;
+    apf::MeshEntity* firstFace = nullptr;
     for (int i = 0; i < up.n; i++) {
       if ( isOnDomainBoundary(mesh, up.e[i]) ) {
         firstFace = up.e[i]; break;
       }
     }
+    PCU_ALWAYS_ASSERT(firstFace);
     faces.push_back(firstFace);
     mesh->getUp(firstFace, up);
     PCU_ALWAYS_ASSERT(up.n == 1);

From 897ffdad5afb65194d769ef3d290a31eb6987a70 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Sat, 20 Mar 2021 10:06:49 -0400
Subject: [PATCH 320/555] Revert "fixes issue #334"

This reverts commit 439d9a856f36d6149e04b5c38aa7739300f1b5d1.

causes failures in ph_adapt and chef7... not sure why
https://my.cdash.org/test/28207484
https://my.cdash.org/test/28207381
---
 phasta/phIO.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/phasta/phIO.c b/phasta/phIO.c
index 11e31f4c5..4352dadbb 100644
--- a/phasta/phIO.c
+++ b/phasta/phIO.c
@@ -81,7 +81,7 @@ static int find_header(FILE* f, const char* name, char* found, char header[PH_LI
     tmp[PH_LINE-1] = '\0';
     parse_header(tmp, &hname, &bytes, 0, NULL);
     if (!strncmp(name, hname, strlen(name))) {
-      strncpy(found, hname, strlen(found));
+      strncpy(found, hname, strlen(hname));
       found[strlen(hname)] = '\0';
       return 1;
     }

From 501a3c289a29e9796adb3dd6e74ccc2a4479a075 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Thu, 15 Apr 2021 11:08:53 -0400
Subject: [PATCH 321/555] update pumi meshes submodule

---
 pumi-meshes | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pumi-meshes b/pumi-meshes
index 4d293fe6d..d8d3f38b4 160000
--- a/pumi-meshes
+++ b/pumi-meshes
@@ -1 +1 @@
-Subproject commit 4d293fe6d329ef255ea90b08ed97884219a4547f
+Subproject commit d8d3f38b4d2c64245dbcf432251eedd51a671036

From 184835851b5cb6399f5c4dbe1886b6448bcd271f Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Thu, 15 Apr 2021 11:45:51 -0400
Subject: [PATCH 322/555] test requires parasolid

---
 test/testing.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/testing.cmake b/test/testing.cmake
index 4e5845ec4..ca97fea61 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -298,7 +298,7 @@ mpi_test(tet_serial 1
   "${MDIR}/pipe.${GXT}"
   "pipe.smb"
   "tet.smb")
-if(ENABLE_SIMMETRIX)
+if(ENABLE_SIMMETRIX AND SIM_PARASOLID)
   mpi_test(test_residual_error_estimate 1
     ./residualErrorEstimation_test
     "${MESHES}/electromagnetic/fichera.x_t"

From 4a4586907606d0a151a232611c1ab5563d21385a Mon Sep 17 00:00:00 2001
From: "E. Seegyoung Seol" <seols@rpi.edu>
Date: Mon, 24 May 2021 12:09:41 -0400
Subject: [PATCH 323/555] adding apf::freezeFields as needed

---
 apf/apfMesh.cc | 10 ++++++++++
 apf/apfMesh.h  |  4 ++++
 2 files changed, 14 insertions(+)

diff --git a/apf/apfMesh.cc b/apf/apfMesh.cc
index 02ff25118..34f17f2a8 100644
--- a/apf/apfMesh.cc
+++ b/apf/apfMesh.cc
@@ -752,6 +752,16 @@ void unfreezeFields(Mesh* m) {
   m->hasFrozenFields = false;
 }
 
+void freezeFields(Mesh* m) {
+  Field* f;
+  for (int i=0; i<m->countFields(); i++) {
+    f = m->getField(i);
+    if (!isFrozen(f))
+      freeze(f);
+  }
+  m->hasFrozenFields = true;
+}
+
 Copy getOtherCopy(Mesh* m, MeshEntity* s)
 {
   Copies remotes;
diff --git a/apf/apfMesh.h b/apf/apfMesh.h
index 052f6584b..ac60009f5 100644
--- a/apf/apfMesh.h
+++ b/apf/apfMesh.h
@@ -573,6 +573,10 @@ void changeMeshShape(Mesh* m, FieldShape* newShape, bool project = true);
   \details see apf::unfreezeField */
 void unfreezeFields(Mesh* m);
 
+/** \brief freeze all associated fields
+  \details see apf::freezeField */
+void freezeFields(Mesh* m);
+
 /** \brief count the number of mesh entities classified on a model entity */
 int countEntitiesOn(Mesh* m, ModelEntity* me, int dim);
 

From 0f773d20b256c0531c2c879638fcbaab8c590438 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Tue, 25 May 2021 06:54:02 -0600
Subject: [PATCH 324/555] small file format changes to receive data written
 from MGEN to avoid the write3D step with MATLAB

---
 test/matchedNodeElmReader.cc | 56 ++++++++++++++++++++----------------
 1 file changed, 32 insertions(+), 24 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 0061873cc..e8405d52a 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -584,14 +584,15 @@ bool skipLine(char* line) {
 
 void getNumVerts(FILE* f, unsigned& verts) {
   rewind(f);
-  verts = 0;
-  size_t linelimit = 1024;
-  char* line = new char[linelimit];
-  while( gmi_getline(&line,&linelimit,f) != -1 ) {
-    if( ! skipLine(line) )
-      verts++;
-  }
-  delete [] line;
+  gmi_fscanf(f, 1, "%d",  &verts);
+//  verts = 0;
+//  size_t linelimit = 1024;
+//  char* line = new char[linelimit];
+//  while( gmi_getline(&line,&linelimit,f) != -1 ) {
+//    if( ! skipLine(line) )
+//      verts++;
+//  }
+//  delete [] line;
 }
 
 void readClassification(FILE* f, unsigned numVtx, int** classification) {
@@ -622,12 +623,13 @@ void readCoords(FILE* f, unsigned numvtx, unsigned& localnumvtx, double** coordi
   long firstVtx, lastVtx;
   getLocalRange(numvtx, localnumvtx,firstVtx,lastVtx);
   *coordinates = new double[localnumvtx*3];
-  rewind(f);
+//  rewind(f);
   int vidx = 0;
   for(unsigned i=0; i<numvtx; i++) {
-    int id;
+//    int id;
     double pos[3];
-    gmi_fscanf(f, 4, "%d %lf %lf %lf", &id, pos+0, pos+1, pos+2);
+//    gmi_fscanf(f, 4, "%d %lf %lf %lf", &id, pos+0, pos+1, pos+2);
+    gmi_fscanf(f, 3, "%lf %lf %lf", pos+0, pos+1, pos+2);
     if( i >= firstVtx && i < lastVtx ) {
       for(unsigned j=0; j<3; j++)
         (*coordinates)[vidx*3+j] = pos[j];
@@ -671,10 +673,10 @@ void readMatches(FILE* f, unsigned numvtx, int** matches) {
           ( matchedVtx >= 1 && matchedVtx <= static_cast<int>(numvtx) ));
       if( matchedVtx != -1 )
         --matchedVtx;
-      if( matchedVtx == 66350 || matchedVtx == 65075 ) {
-        fprintf(stderr, "%d reader found match %d at gid %d i %d vidx %d\n",
-            PCU_Comm_Self(), matchedVtx, gid, i, vidx);
-      }
+//      if( matchedVtx == 66350 || matchedVtx == 65075 ) {
+//        fprintf(stderr, "%d reader found match %d at gid %d i %d vidx %d\n",
+ //           PCU_Comm_Self(), matchedVtx, gid, i, vidx);
+//      }
       (*matches)[vidx] = matchedVtx;
       vidx++;
     }
@@ -682,14 +684,15 @@ void readMatches(FILE* f, unsigned numvtx, int** matches) {
   }
 }
 
-void readElements(FILE* f, unsigned &dim, unsigned& numElms,
+void readElements(FILE* f, FILE* fh, unsigned &dim, unsigned& numElms,
     unsigned& numVtxPerElm, unsigned& localNumElms, int** elements) {
   rewind(f);
+  rewind(fh);
   int dimHeader[2];
-  gmi_fscanf(f, 2, "%u %u", dimHeader, dimHeader+1);
+  gmi_fscanf(fh, 2, "%u %u", dimHeader, dimHeader+1);
   assert( dimHeader[0] == 1 && dimHeader[1] == 1);
-  gmi_fscanf(f, 1, "%u", &dim);
-  gmi_fscanf(f, 2, "%u %u", &numElms, &numVtxPerElm);
+  gmi_fscanf(fh, 1, "%u", &dim);
+  gmi_fscanf(fh, 2, "%u %u", &numElms, &numVtxPerElm);
   long firstElm, lastElm;
   getLocalRange(numElms, localNumElms, firstElm, lastElm);
   *elements = new int[localNumElms*numVtxPerElm];
@@ -734,10 +737,12 @@ void readMesh(const char* meshfilename,
     const char* classfilename,
     const char* fathers2Dfilename,
     const char* solutionfilename,
+    const char* connHeadfilename,
     MeshInfo& mesh) {
   FILE* fc = fopen(coordfilename, "r");
   PCU_ALWAYS_ASSERT(fc);
   getNumVerts(fc,mesh.numVerts);
+  
   if(!PCU_Comm_Self())
     fprintf(stderr, "numVerts %u\n", mesh.numVerts);
   readCoords(fc, mesh.numVerts, mesh.localNumVerts, &(mesh.coords));
@@ -769,8 +774,10 @@ void readMesh(const char* meshfilename,
   }
 
   FILE* f = fopen(meshfilename, "r");
+  FILE* fh = fopen(connHeadfilename, "r");
   PCU_ALWAYS_ASSERT(f);
-  readElements(f, mesh.dim, mesh.numElms, mesh.numVtxPerElm,
+  PCU_ALWAYS_ASSERT(fh);
+  readElements(f,fh, mesh.dim, mesh.numElms, mesh.numVtxPerElm,
       mesh.localNumElms, &(mesh.elements));
   mesh.elementType = getElmType(mesh.dim, mesh.numVtxPerElm);
   fclose(f);
@@ -782,7 +789,7 @@ int main(int argc, char** argv)
   MPI_Init(&argc,&argv);
   PCU_Comm_Init();
   lion_set_verbosity(1);
-  if( argc != 9 ) {
+  if( argc != 11 ) {
     if( !PCU_Comm_Self() ) {
       printf("Usage: %s <ascii mesh connectivity .cnn> "
           "<ascii vertex coordinates .crd> "
@@ -790,6 +797,7 @@ int main(int argc, char** argv)
           "<ascii vertex classification flag .class> "
           "<ascii vertex fathers2D flag .fathers2D> "
           "<ascii solution flag .soln> "
+          "<ascii conn header> "
           "<output model .dmg> <output mesh .smb>\n",
           argv[0]);
     }
@@ -802,7 +810,7 @@ int main(int argc, char** argv)
 
   double t0 = PCU_Time();
   MeshInfo m;
-  readMesh(argv[1],argv[2],argv[3],argv[4],argv[5],argv[6],m);
+  readMesh(argv[1],argv[2],argv[3],argv[4],argv[5],argv[6],argv[7],m);
 
   bool isMatched = true;
   if( !strcmp(argv[3], "NULL") )
@@ -866,8 +874,8 @@ int main(int argc, char** argv)
   mesh->verify();
 
   outMap.clear();
-  gmi_write_dmg(model, argv[7]);
-  mesh->writeNative(argv[8]);
+  gmi_write_dmg(model, argv[8]);
+  mesh->writeNative(argv[9]);
   apf::writeVtkFiles("rendered",mesh);
 
   mesh->destroyNative();

From c1d05c1a5336549bf6d85f8be8f5d88c373336cc Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Thu, 27 May 2021 12:52:25 -0600
Subject: [PATCH 325/555] ditched all of the unused row numbers for every file
 that is read in which saves a lot of disk space and probably some read time
 here and write time from upstream codes.  We are ALWAYS reading this in a
 loop so that index is available.  I understand having it there during
 development probably helped for debugging

---
 test/matchedNodeElmReader.cc | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index e8405d52a..8f10f9a7d 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -603,9 +603,10 @@ void readClassification(FILE* f, unsigned numVtx, int** classification) {
   rewind(f);
   int vidx = 0;
   for(unsigned i=0; i<numVtx; i++) {
-    int id;
+//    int id;
     int mdlId;
-    gmi_fscanf(f, 2, "%d %d", &id, &mdlId);
+//    gmi_fscanf(f, 2, "%d %d", &id, &mdlId);
+    gmi_fscanf(f, 1, "%d",  &mdlId);
     //std::cout<<"read id is "<<id<<std::endl;
     //std::cout<<"read model id is "<<mdlId<<std::endl;
     if( i >= firstVtx && i < lastVtx ) {
@@ -665,9 +666,9 @@ void readMatches(FILE* f, unsigned numvtx, int** matches) {
   *matches = new int[localnumvtx];
   rewind(f);
   int vidx = 0;
-  int gid, matchedVtx;
+  int matchedVtx;
   int i = 0;
-  while( 2 == fscanf(f, "%d %d", &gid, &matchedVtx) ) {
+  while( 1 == fscanf(f, "%d", &matchedVtx) ) {
     if( i >= firstVtx && i < lastVtx ) {
       PCU_ALWAYS_ASSERT( matchedVtx == -1 ||
           ( matchedVtx >= 1 && matchedVtx <= static_cast<int>(numvtx) ));
@@ -700,8 +701,6 @@ void readElements(FILE* f, FILE* fh, unsigned &dim, unsigned& numElms,
   unsigned elmIdx = 0;
   int* elmVtx = new int[numVtxPerElm];
   for (i = 0; i < numElms; i++) {
-    int ignored;
-    gmi_fscanf(f, 1, "%u", &ignored);
     for (j = 0; j < numVtxPerElm; j++)
       gmi_fscanf(f, 1, "%u", elmVtx+j);
     if (i >= firstElm && i < lastElm) {
@@ -789,7 +788,7 @@ int main(int argc, char** argv)
   MPI_Init(&argc,&argv);
   PCU_Comm_Init();
   lion_set_verbosity(1);
-  if( argc != 11 ) {
+  if( argc != 10 ) {
     if( !PCU_Comm_Self() ) {
       printf("Usage: %s <ascii mesh connectivity .cnn> "
           "<ascii vertex coordinates .crd> "

From c26c7500341ce815ee20b54e9faf3faa6fbe35de Mon Sep 17 00:00:00 2001
From: Morteza HS <hakimi1364@gmail.com>
Date: Fri, 28 May 2021 08:31:36 -0400
Subject: [PATCH 326/555] Adds gmi_null to serialize.cc

So that the serialize can be called on partitioned meshes with a null
model, as well.
---
 test/serialize.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/serialize.cc b/test/serialize.cc
index 87f1b7be1..b84a4baa0 100644
--- a/test/serialize.cc
+++ b/test/serialize.cc
@@ -3,6 +3,7 @@
 #include <lionPrint.h>
 #include <apfMDS.h>
 #include <gmi_mesh.h>
+#include <gmi_null.h>
 #include <crv.h>
 #ifdef HAVE_SIMMETRIX
 #include <gmi_sim.h>
@@ -42,6 +43,7 @@ int main( int argc, char* argv[])
   gmi_register_sim();
 #endif
   gmi_register_mesh();
+  gmi_register_null();
   crv::getBezier(2);//hack to make sure curved meshes can be serialized!
   GroupCode code;
   code.mesh = apf::loadMdsMesh(argv[1], argv[2]);

From fe15d56fbf51487d0b50d23873e938079b234aa9 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Fri, 28 May 2021 07:40:07 -0700
Subject: [PATCH 327/555] start converting ints to longs for conversion

not sure if this will compile
---
 apf/apfConstruct.cc          | 6 ++----
 apf/apfConvert.h             | 6 ++++--
 test/matchedNodeElmReader.cc | 8 ++++----
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index 400d36f4c..87c72697f 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -8,8 +8,6 @@
 
 namespace apf {
 
-typedef int Gid;
-
 static void constructVerts(
     Mesh2* m, const Gid* conn, int nelem, int etype,
     GlobalToVert& result)
@@ -32,7 +30,7 @@ static void constructElements(
     int irep=0;
     Downward verts;
     int offset = i * nev;
-    int vCur=conn[offset];
+    Gid vCur=conn[offset];
     int vNext=-1;
     int uniqueVerts=1;
     for (int j = 0; j < nev; ++j) {
@@ -58,7 +56,7 @@ static Gid getMax(const GlobalToVert& globalToVert)
   Gid max = -1;
   APF_CONST_ITERATE(GlobalToVert, globalToVert, it)
     max = std::max(max, it->first);
-  return PCU_Max_Int(max); // this is type-dependent
+  return PCU_Max_Long(max); // this is type-dependent
 }
 
 
diff --git a/apf/apfConvert.h b/apf/apfConvert.h
index 2d517c03e..6fb3a68ea 100644
--- a/apf/apfConvert.h
+++ b/apf/apfConvert.h
@@ -29,8 +29,10 @@ class MeshEntity;
 void convert(Mesh *in, Mesh2 *out,
              MeshEntity** nodes=NULL, MeshEntity** elems=NULL);
 
+typedef long Gid;
+
 /** \brief a map from global ids to vertex objects */
-typedef std::map<int, MeshEntity*> GlobalToVert;
+typedef std::map<Gid, MeshEntity*> GlobalToVert;
 
 /** \brief construct a mesh from just a connectivity array
   \details this function is here to interface with very
@@ -46,7 +48,7 @@ typedef std::map<int, MeshEntity*> GlobalToVert;
 
   Note that all vertices will have zero coordinates, so
   it is often good to use apf::setCoords after this. */
-void construct(Mesh2* m, const int* conn, int nelem, int etype,
+void construct(Mesh2* m, const Gid* conn, int nelem, int etype,
     GlobalToVert& globalToVert);
 
 /** \brief Assign coordinates to the mesh
diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 8f10f9a7d..84519c6ee 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -686,7 +686,7 @@ void readMatches(FILE* f, unsigned numvtx, int** matches) {
 }
 
 void readElements(FILE* f, FILE* fh, unsigned &dim, unsigned& numElms,
-    unsigned& numVtxPerElm, unsigned& localNumElms, int** elements) {
+    unsigned& numVtxPerElm, unsigned& localNumElms, Gid** elements) {
   rewind(f);
   rewind(fh);
   int dimHeader[2];
@@ -696,10 +696,10 @@ void readElements(FILE* f, FILE* fh, unsigned &dim, unsigned& numElms,
   gmi_fscanf(fh, 2, "%u %u", &numElms, &numVtxPerElm);
   long firstElm, lastElm;
   getLocalRange(numElms, localNumElms, firstElm, lastElm);
-  *elements = new int[localNumElms*numVtxPerElm];
+  *elements = new Gid[localNumElms*numVtxPerElm];
   unsigned i, j;
   unsigned elmIdx = 0;
-  int* elmVtx = new int[numVtxPerElm];
+  Gid* elmVtx = new Gid[numVtxPerElm];
   for (i = 0; i < numElms; i++) {
     for (j = 0; j < numVtxPerElm; j++)
       gmi_fscanf(f, 1, "%u", elmVtx+j);
@@ -717,7 +717,7 @@ void readElements(FILE* f, FILE* fh, unsigned &dim, unsigned& numElms,
 struct MeshInfo {
   double* coords;
   double* solution;
-  int* elements;
+  Gid* elements;
   int* matches;
   int* classification;
   int* fathers2D;

From bed77e7a00c4f6b01ffa2bcdf46437315349cdb4 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Fri, 28 May 2021 10:05:08 -0700
Subject: [PATCH 328/555] chasing out int long conflicts...still won't compile

---
 apf/apfConstruct.cc |  6 +++---
 mds/apfMDS.cc       |  6 +++---
 mds/apfMDS.h        |  2 +-
 pcu/PCU.h           |  2 ++
 pcu/pcu.c           | 16 ++++++++++++++++
 pcu/pcu_coll.c      |  9 +++++++++
 pcu/pcu_coll.h      |  1 +
 7 files changed, 35 insertions(+), 7 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index 87c72697f..d70e55d4d 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -9,7 +9,7 @@
 namespace apf {
 
 static void constructVerts(
-    Mesh2* m, const Gid* conn, int nelem, int etype,
+    Mesh2* m, const int* conn, int nelem, int etype,
     GlobalToVert& result)
 {
   ModelEntity* interior = m->findModelEntity(m->getDimension(), 0);
@@ -20,7 +20,7 @@ static void constructVerts(
 }
 
 static void constructElements(
-    Mesh2* m, const Gid* conn, int nelem, int etype,
+    Mesh2* m, const int* conn, int nelem, int etype,
     GlobalToVert& globalToVert)
 {
   ModelEntity* interior = m->findModelEntity(m->getDimension(), 0);
@@ -507,7 +507,7 @@ void destruct(Mesh2* m, int*& conn, int& nelem, int &etype)
     Downward verts;
     int nverts = m->getDownward(e, 0, verts);
     if (!conn)
-      conn = new Gid[nelem * nverts];
+      conn = new int[nelem * nverts];
     for (int j = 0; j < nverts; ++j)
       conn[i++] = getNumber(global, Node(verts[j], 0));
   }
diff --git a/mds/apfMDS.cc b/mds/apfMDS.cc
index 1913ef8cc..71229c00c 100644
--- a/mds/apfMDS.cc
+++ b/mds/apfMDS.cc
@@ -1050,8 +1050,8 @@ void deriveMdlFromManifold(Mesh2* mesh, bool* isModelVert,
           "MeshTag name \"_vert_id\" is used internally in this method\n");
   apf::MeshTag* vIDTag = mesh->createIntTag("_vert_id", 1);
   for (apf::GlobalToVert::iterator vit = globalToVert.begin();
-       vit != globalToVert.end(); vit++) {
-    mesh->setIntTag(vit->second, vIDTag, &(vit->first));
+       vit !=  globalToVert.end(); vit++) {
+    mesh->setIntTag((int)vit->second, vIDTag, (int)&(vit->first));
   }
 
   // Reserve tags used for model faces
@@ -1167,7 +1167,7 @@ void derive2DMdlFromManifold(Mesh2* mesh, bool* isModelVert,
   apf::MeshTag* vIDTag = mesh->createIntTag("_vert_id", 1);
   for (apf::GlobalToVert::iterator vit = globalToVert.begin();
        vit != globalToVert.end(); vit++) {
-    mesh->setIntTag(vit->second, vIDTag, &(vit->first));
+    mesh->setIntTag((int)vit->second, vIDTag, (int)&(vit->first));
   }
 
   // Reserve tags used for model edges
diff --git a/mds/apfMDS.h b/mds/apfMDS.h
index 97176ef29..443aeb398 100644
--- a/mds/apfMDS.h
+++ b/mds/apfMDS.h
@@ -43,7 +43,7 @@ class MeshEntity;
 class Migration;
 
 /** \brief a map from global ids to vertex objects */
-typedef std::map<int, MeshEntity*> GlobalToVert;
+typedef std::map<long, MeshEntity*> GlobalToVert;
 
 /** \brief create an empty MDS part
   \param model the geometric model interface
diff --git a/pcu/PCU.h b/pcu/PCU.h
index 67b2ecccd..70ab43478 100644
--- a/pcu/PCU.h
+++ b/pcu/PCU.h
@@ -77,6 +77,8 @@ void PCU_Min_Ints(int* p, size_t n);
 int PCU_Min_Int(int x);
 void PCU_Max_Ints(int* p, size_t n);
 int PCU_Max_Int(int x);
+void PCU_Max_Longss(long* p, size_t n);
+long PCU_Max_Long(long x);
 int PCU_Or(int c);
 int PCU_And(int c);
 
diff --git a/pcu/pcu.c b/pcu/pcu.c
index e600b6fdf..0412f8d17 100644
--- a/pcu/pcu.c
+++ b/pcu/pcu.c
@@ -469,6 +469,22 @@ int PCU_Max_Int(int x)
   PCU_Max_Ints(a, 1);
   return a[0];
 }
+/** \brief Performs an Allreduce maximum of long arrays.
+  */
+void PCU_Max_Longs(long* p, size_t n)
+{
+  if (global_state == uninit)
+    reel_fail("Max_Longs called before Comm_Init");
+  pcu_allreduce(&(get_msg()->coll),pcu_max_longs,p,n*sizeof(long));
+}
+
+long PCU_Max_Long(long x)
+{
+  long a[1];
+  a[0] = x;
+  PCU_Max_Longs(a, 1);
+  return a[0];
+}
 
 /** \brief Performs a parallel logical OR reduction
   */
diff --git a/pcu/pcu_coll.c b/pcu/pcu_coll.c
index 836a69346..7bdf8a221 100644
--- a/pcu/pcu_coll.c
+++ b/pcu/pcu_coll.c
@@ -88,6 +88,15 @@ void pcu_max_ints(void* local, void* incoming, size_t size)
     a[i] = MAX(a[i],b[i]);
 }
 
+void pcu_max_longs(void* local, void* incoming, size_t size)
+{
+  long* a = local;
+  long* b= incoming;
+  size_t n = size/sizeof(long);
+  for (size_t i=0; i < n; ++i)
+    a[i] = MAX(a[i],b[i]);
+}
+
 void pcu_min_sizets(void* local, void* incoming, size_t size)
 {
   size_t* a = local;
diff --git a/pcu/pcu_coll.h b/pcu/pcu_coll.h
index 83ca77daa..11f204ee8 100644
--- a/pcu/pcu_coll.h
+++ b/pcu/pcu_coll.h
@@ -33,6 +33,7 @@ void pcu_min_doubles(void* local, void* incoming, size_t size);
 void pcu_add_ints(void* local, void* incoming, size_t size);
 void pcu_min_ints(void* local, void* incoming, size_t size);
 void pcu_max_ints(void* local, void* incoming, size_t size);
+void pcu_max_longs(void* local, void* incoming, size_t size);
 void pcu_add_longs(void* local, void* incoming, size_t size);
 void pcu_add_sizets(void* local, void* incoming, size_t size);
 void pcu_min_sizets(void* local, void* incoming, size_t size);

From 4717eef01eca7639738a51f358a69c9125e766c4 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 28 May 2021 15:01:13 -0400
Subject: [PATCH 329/555] use long tags to store gids

---
 mds/apfMDS.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/mds/apfMDS.cc b/mds/apfMDS.cc
index 71229c00c..2b8f4f780 100644
--- a/mds/apfMDS.cc
+++ b/mds/apfMDS.cc
@@ -1048,10 +1048,10 @@ void deriveMdlFromManifold(Mesh2* mesh, bool* isModelVert,
 
   PCU_ALWAYS_ASSERT_VERBOSE(!mesh->findTag("_vert_id"),
           "MeshTag name \"_vert_id\" is used internally in this method\n");
-  apf::MeshTag* vIDTag = mesh->createIntTag("_vert_id", 1);
+  apf::MeshTag* vIDTag = mesh->createLongTag("_vert_id", 1);
   for (apf::GlobalToVert::iterator vit = globalToVert.begin();
        vit !=  globalToVert.end(); vit++) {
-    mesh->setIntTag((int)vit->second, vIDTag, (int)&(vit->first));
+    mesh->setLongTag(vit->second, vIDTag, &(vit->first));
   }
 
   // Reserve tags used for model faces
@@ -1164,10 +1164,10 @@ void derive2DMdlFromManifold(Mesh2* mesh, bool* isModelVert,
 
   PCU_ALWAYS_ASSERT_VERBOSE(!mesh->findTag("_vert_id"),
           "MeshTag name \"_vert_id\" is used internally in this method\n");
-  apf::MeshTag* vIDTag = mesh->createIntTag("_vert_id", 1);
+  apf::MeshTag* vIDTag = mesh->createLongTag("_vert_id", 1);
   for (apf::GlobalToVert::iterator vit = globalToVert.begin();
        vit != globalToVert.end(); vit++) {
-    mesh->setIntTag((int)vit->second, vIDTag, (int)&(vit->first));
+    mesh->setLongTag(vit->second, vIDTag, &(vit->first));
   }
 
   // Reserve tags used for model edges

From 6faee07b51e4b23c9c180de33eba42e921b0932b Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 28 May 2021 15:26:10 -0400
Subject: [PATCH 330/555] use long ints for construct

---
 apf/apfConstruct.cc          | 10 +++++-----
 apf/apfConvert.h             |  2 +-
 apf/apfConvertTags.h         |  9 ++++-----
 test/construct.cc            |  2 +-
 test/icesheet.cc             |  8 ++++----
 test/matchedNodeElmReader.cc |  8 ++++----
 6 files changed, 19 insertions(+), 20 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index d70e55d4d..49d2680df 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -9,7 +9,7 @@
 namespace apf {
 
 static void constructVerts(
-    Mesh2* m, const int* conn, int nelem, int etype,
+    Mesh2* m, const Gid* conn, int nelem, int etype,
     GlobalToVert& result)
 {
   ModelEntity* interior = m->findModelEntity(m->getDimension(), 0);
@@ -20,7 +20,7 @@ static void constructVerts(
 }
 
 static void constructElements(
-    Mesh2* m, const int* conn, int nelem, int etype,
+    Mesh2* m, const Gid* conn, int nelem, int etype,
     GlobalToVert& globalToVert)
 {
   ModelEntity* interior = m->findModelEntity(m->getDimension(), 0);
@@ -160,7 +160,7 @@ static void constructRemotes(Mesh2* m, GlobalToVert& globalToVert)
   }
 }
 
-void construct(Mesh2* m, const int* conn, int nelem, int etype,
+void construct(Mesh2* m, const Gid* conn, int nelem, int etype,
     GlobalToVert& globalToVert)
 {
   constructVerts(m, conn, nelem, etype, globalToVert);
@@ -492,7 +492,7 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
   m->destroyTag(matchGidTag);
 }
 
-void destruct(Mesh2* m, int*& conn, int& nelem, int &etype)
+void destruct(Mesh2* m, Gid*& conn, int& nelem, int &etype)
 {
   int dim = m->getDimension();
   nelem = m->count(dim);
@@ -507,7 +507,7 @@ void destruct(Mesh2* m, int*& conn, int& nelem, int &etype)
     Downward verts;
     int nverts = m->getDownward(e, 0, verts);
     if (!conn)
-      conn = new int[nelem * nverts];
+      conn = new Gid[nelem * nverts];
     for (int j = 0; j < nverts; ++j)
       conn[i++] = getNumber(global, Node(verts[j], 0));
   }
diff --git a/apf/apfConvert.h b/apf/apfConvert.h
index 6fb3a68ea..63ad80a38 100644
--- a/apf/apfConvert.h
+++ b/apf/apfConvert.h
@@ -77,7 +77,7 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
 
 /** \brief convert an apf::Mesh2 object into a connectivity array
   \details this is useful for debugging the apf::convert function */
-void destruct(Mesh2* m, int*& conn, int& nelem, int &etype);
+void destruct(Mesh2* m, Gid*& conn, int& nelem, int &etype);
 
 /** \brief get a contiguous set of global vertex coordinates
   \details this is used for debugging apf::setCoords */
diff --git a/apf/apfConvertTags.h b/apf/apfConvertTags.h
index eff252129..375bdef85 100644
--- a/apf/apfConvertTags.h
+++ b/apf/apfConvertTags.h
@@ -6,10 +6,9 @@
 #include "apfConvert.h"
 
 namespace {
-  typedef int Gid;
-  static Gid getMax(const apf::GlobalToVert& globalToVert)
+  static apf::Gid getMax(const apf::GlobalToVert& globalToVert)
   {
-    Gid max = -1;
+    apf::Gid max = -1;
     APF_CONST_ITERATE(apf::GlobalToVert, globalToVert, it)
       max = std::max(max, it->first);
     return PCU_Max_Int(max); // this is type-dependent
@@ -62,8 +61,8 @@ apf::MeshTag* setMappedTag(Mesh2* m, const char* tagName,
     const T* vals, const int entries,
     int nverts, GlobalToVert& globalToVert)
 {
-  Gid max = getMax(globalToVert);
-  Gid total = max + 1;
+  apf::Gid max = getMax(globalToVert);
+  apf::Gid total = max + 1;
   int peers = PCU_Comm_Peers();
   int quotient = total / peers;
   int remainder = total % peers;
diff --git a/test/construct.cc b/test/construct.cc
index 962316162..a702b973d 100644
--- a/test/construct.cc
+++ b/test/construct.cc
@@ -16,7 +16,7 @@ int main(int argc, char** argv)
   lion_set_verbosity(1);
   gmi_register_mesh();
   gmi_register_null();
-  int* conn;
+  apf::Gid* conn;
   double* coords;
   int nelem;
   int etype;
diff --git a/test/icesheet.cc b/test/icesheet.cc
index 7dd6db281..4b7345c7f 100644
--- a/test/icesheet.cc
+++ b/test/icesheet.cc
@@ -66,9 +66,9 @@ void readCoords(FILE* f, unsigned numvtx, double* coordinates) {
 }
 
 void readElements(FILE* f, unsigned numelms, int numVtxPerElm,
-    unsigned numVerts, int* elements) {
+    unsigned numVerts, apf::Gid* elements) {
   unsigned i;
-  std::map<int, int> count;
+  std::map<apf::Gid, int> count;
   for (i = 0; i < numelms*numVtxPerElm; i++) {
     int vtxid;
     gmi_fscanf(f, 1, "%u", &vtxid);
@@ -80,7 +80,7 @@ void readElements(FILE* f, unsigned numelms, int numVtxPerElm,
 
 struct MeshInfo {
   double* coords;
-  int* elements;
+  apf::Gid* elements;
   unsigned elementType;
   unsigned numVerts;
   unsigned numElms;
@@ -95,7 +95,7 @@ void readMesh(const char* meshfilename, MeshInfo& mesh) {
       mesh.numVerts, mesh.numElms, mesh.numVtxPerElm);
   mesh.coords = new double[mesh.numVerts*3];
   readCoords(f, mesh.numVerts, mesh.coords);
-  mesh.elements = new int [mesh.numElms*mesh.numVtxPerElm];
+  mesh.elements = new apf::Gid [mesh.numElms*mesh.numVtxPerElm];
   readElements(f, mesh.numElms, mesh.numVtxPerElm, mesh.numVerts, mesh.elements);
   mesh.elementType = getElmType(mesh.numVtxPerElm);
   fclose(f);
diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 84519c6ee..1ab26c29e 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -686,7 +686,7 @@ void readMatches(FILE* f, unsigned numvtx, int** matches) {
 }
 
 void readElements(FILE* f, FILE* fh, unsigned &dim, unsigned& numElms,
-    unsigned& numVtxPerElm, unsigned& localNumElms, Gid** elements) {
+    unsigned& numVtxPerElm, unsigned& localNumElms, apf::Gid** elements) {
   rewind(f);
   rewind(fh);
   int dimHeader[2];
@@ -696,10 +696,10 @@ void readElements(FILE* f, FILE* fh, unsigned &dim, unsigned& numElms,
   gmi_fscanf(fh, 2, "%u %u", &numElms, &numVtxPerElm);
   long firstElm, lastElm;
   getLocalRange(numElms, localNumElms, firstElm, lastElm);
-  *elements = new Gid[localNumElms*numVtxPerElm];
+  *elements = new apf::Gid[localNumElms*numVtxPerElm];
   unsigned i, j;
   unsigned elmIdx = 0;
-  Gid* elmVtx = new Gid[numVtxPerElm];
+  apf::Gid* elmVtx = new apf::Gid[numVtxPerElm];
   for (i = 0; i < numElms; i++) {
     for (j = 0; j < numVtxPerElm; j++)
       gmi_fscanf(f, 1, "%u", elmVtx+j);
@@ -717,7 +717,7 @@ void readElements(FILE* f, FILE* fh, unsigned &dim, unsigned& numElms,
 struct MeshInfo {
   double* coords;
   double* solution;
-  Gid* elements;
+  apf::Gid* elements;
   int* matches;
   int* classification;
   int* fathers2D;

From 6a212fbfda9078bc73d6a9cbade2f59be311a149 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Fri, 28 May 2021 15:01:46 -0600
Subject: [PATCH 331/555] debugging

---
 apf/apfConstruct.cc | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index 49d2680df..67f23325e 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -32,6 +32,7 @@ static void constructElements(
     int offset = i * nev;
     Gid vCur=conn[offset];
     int vNext=-1;
+    int uniqueVerts2=1;
     int uniqueVerts=1;
     for (int j = 0; j < nev; ++j) {
       if(irep ==0){
@@ -47,7 +48,12 @@ static void constructElements(
     if(uniqueVerts==5) etypeL=apf::Mesh::PYRAMID;
     if(uniqueVerts==6) etypeL=apf::Mesh::PRISM;
     if(uniqueVerts==8) etypeL=apf::Mesh::HEX;
+    if(m==NULL) {
+      uniqueVerts2=2;
+    }
+    if(uniqueVerts/uniqueVerts2 > 0){
     buildElement(m, interior, etypeL, verts);
+    }
   }
 }
 
@@ -480,7 +486,7 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
         int rightPart = cp->first;
         apf::MeshEntity* right = cp->second;
         m->addMatch(left, rightPart, right);
-        PCU_Debug_Print("%d add remote copy match ptr to %d gid %d\n",
+        PCU_Debug_Print("%d add remote copy match ptr to %d gid %ld\n",
           self, rightPart, it->first);
       }
     }

From f39e81579cd6e76027bdd37d1fb8a102cf2ee91f Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Fri, 28 May 2021 17:15:48 -0600
Subject: [PATCH 332/555] typo of no consequence

---
 pcu/PCU.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pcu/PCU.h b/pcu/PCU.h
index 70ab43478..daecc2634 100644
--- a/pcu/PCU.h
+++ b/pcu/PCU.h
@@ -77,7 +77,7 @@ void PCU_Min_Ints(int* p, size_t n);
 int PCU_Min_Int(int x);
 void PCU_Max_Ints(int* p, size_t n);
 int PCU_Max_Int(int x);
-void PCU_Max_Longss(long* p, size_t n);
+void PCU_Max_Longs(long* p, size_t n);
 long PCU_Max_Long(long x);
 int PCU_Or(int c);
 int PCU_And(int c);

From 6dd96dbb378cac7815d41da34f0ee23a983da4f9 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Fri, 28 May 2021 18:34:00 -0600
Subject: [PATCH 333/555]  long format read of connectivity into a long
 variable

---
 test/matchedNodeElmReader.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 1ab26c29e..49fb5ef0e 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -702,7 +702,7 @@ void readElements(FILE* f, FILE* fh, unsigned &dim, unsigned& numElms,
   apf::Gid* elmVtx = new apf::Gid[numVtxPerElm];
   for (i = 0; i < numElms; i++) {
     for (j = 0; j < numVtxPerElm; j++)
-      gmi_fscanf(f, 1, "%u", elmVtx+j);
+      gmi_fscanf(f, 1, "%ld", elmVtx+j);
     if (i >= firstElm && i < lastElm) {
       for (j = 0; j < numVtxPerElm; j++) {
         const unsigned elmVtxIdx = elmIdx*numVtxPerElm+j;

From d7c20b62efbcd52043e3449c77b004f8c09d3e93 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Sat, 29 May 2021 07:22:21 -0700
Subject: [PATCH 334/555] attempt to lion_eprint produced nothing

---
 apf/apfConstruct.cc | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index 67f23325e..154943d59 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -5,6 +5,7 @@
 #include "apf.h"
 #include "apfNumbering.h"
 #include <map>
+#include <lionPrint.h>
 
 namespace apf {
 
@@ -200,6 +201,14 @@ void setCoords(Mesh2* m, const double* coords, int nverts,
   PCU_Comm_Begin();
   int to = std::min(peers - 1, start / quotient);
   int n = std::min((to+1)*quotient-start, nverts);
+  if(n > 100000000) {
+     lion_eprint(1, "setCoords int overflow of: self=%d,mySize=%d,total=%ld, n=%d,to=%d, quotient=%d, remainder=%d start=%d, peers=%d \n",self,mySize,total,n,to,quotient,remainder,start,peers);
+  Gid peersG = PCU_Comm_Peers();
+    Gid quotientG = total / peersG;
+  Gid remainderG = total % peersG;
+     lion_eprint(1, "setCoords Gid0test: self=%d,mySize=%d,total=%ld, quotientG=%ld, peers=%ld \n",self,mySize,total,quotientG,remainderG,peersG);
+}
+
   while (nverts > 0) {
     PCU_COMM_PACK(to, start);
     PCU_COMM_PACK(to, n);

From 67bef252b982b8320adc5a7f161fcdb30cb23461 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Sat, 29 May 2021 10:01:51 -0700
Subject: [PATCH 335/555] changing all variables upstream of the crash to long
 using Gid

---
 apf/apfConstruct.cc | 45 +++++++++++++++++++++++----------------------
 1 file changed, 23 insertions(+), 22 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index 154943d59..0857715cf 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -181,26 +181,27 @@ void construct(Mesh2* m, const Gid* conn, int nelem, int etype,
 void setCoords(Mesh2* m, const double* coords, int nverts,
     GlobalToVert& globalToVert)
 {
+  Gid nvertsG=nverts;
   Gid max = getMax(globalToVert);
   Gid total = max + 1;
-  int peers = PCU_Comm_Peers();
-  int quotient = total / peers;
-  int remainder = total % peers;
-  int mySize = quotient;
-  int self = PCU_Comm_Self();
+  Gid peers = PCU_Comm_Peers();
+  Gid quotient = total / peers;
+  Gid remainder = total % peers;
+  Gid mySize = quotient;
+  Gid self = PCU_Comm_Self();
   if (self == (peers - 1))
     mySize += remainder;
-  int myOffset = self * quotient;
+  Gid myOffset = self * quotient;
 
   /* Force each peer to have exactly mySize verts.
      This means we might need to send and recv some coords */
   double* c = new double[mySize*3];
 
-  int start = PCU_Exscan_Int(nverts);
+  Gid start = PCU_Exscan_Int(nverts);
 
   PCU_Comm_Begin();
-  int to = std::min(peers - 1, start / quotient);
-  int n = std::min((to+1)*quotient-start, nverts);
+  Gid to = std::min(peers - 1, start / quotient);
+  Gid n = std::min((to+1)*quotient-start, nvertsG);
   if(n > 100000000) {
      lion_eprint(1, "setCoords int overflow of: self=%d,mySize=%d,total=%ld, n=%d,to=%d, quotient=%d, remainder=%d start=%d, peers=%d \n",self,mySize,total,n,to,quotient,remainder,start,peers);
   Gid peersG = PCU_Comm_Peers();
@@ -209,16 +210,16 @@ void setCoords(Mesh2* m, const double* coords, int nverts,
      lion_eprint(1, "setCoords Gid0test: self=%d,mySize=%d,total=%ld, quotientG=%ld, peers=%ld \n",self,mySize,total,quotientG,remainderG,peersG);
 }
 
-  while (nverts > 0) {
+  while (nvertsG > 0) {
     PCU_COMM_PACK(to, start);
     PCU_COMM_PACK(to, n);
     PCU_Comm_Pack(to, coords, n*3*sizeof(double));
 
-    nverts -= n;
+    nvertsG -= n;
     start += n;
     coords += n*3;
     to = std::min(peers - 1, to + 1);
-    n = std::min(quotient, nverts);
+    n = std::min(quotient, nvertsG);
   }
   PCU_Comm_Send();
   while (PCU_Comm_Receive()) {
@@ -228,36 +229,36 @@ void setCoords(Mesh2* m, const double* coords, int nverts,
   }
 
   /* Tell all the owners of the coords what we need */
-  typedef std::vector< std::vector<int> > TmpParts;
+  typedef std::vector< std::vector<Gid> > TmpParts;
   TmpParts tmpParts(mySize);
   PCU_Comm_Begin();
   APF_CONST_ITERATE(GlobalToVert, globalToVert, it) {
-    int gid = it->first;
-    int to = std::min(peers - 1, gid / quotient);
+    Gid gid = it->first;
+    Gid to = std::min(peers - 1, gid / quotient);
     PCU_COMM_PACK(to, gid);
   }
   PCU_Comm_Send();
   while (PCU_Comm_Receive()) {
-    int gid;
+    Gid gid;
     PCU_COMM_UNPACK(gid);
-    int from = PCU_Comm_Sender();
+    Gid from = PCU_Comm_Sender();
     tmpParts.at(gid - myOffset).push_back(from);
   }
 
   /* Send the coords to everybody who want them */
   PCU_Comm_Begin();
-  for (int i = 0; i < mySize; ++i) {
-    std::vector<int>& parts = tmpParts[i];
+  for (Gid i = 0; i < mySize; ++i) {
+    std::vector<Gid>& parts = tmpParts[i];
     for (size_t j = 0; j < parts.size(); ++j) {
-      int to = parts[j];
-      int gid = i + myOffset;
+      Gid to = parts[j];
+      Gid gid = i + myOffset;
       PCU_COMM_PACK(to, gid);
       PCU_Comm_Pack(to, &c[i*3], 3*sizeof(double));
     }
   }
   PCU_Comm_Send();
   while (PCU_Comm_Receive()) {
-    int gid;
+    Gid gid;
     PCU_COMM_UNPACK(gid);
     double v[3];
     PCU_Comm_Unpack(v, sizeof(v));

From 36713b76173a6aa1fc6936e985491cbfbf104a7c Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Sat, 29 May 2021 12:27:06 -0600
Subject: [PATCH 336/555] debug code fix

---
 apf/apfConstruct.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index 0857715cf..0b9294d3a 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -202,12 +202,12 @@ void setCoords(Mesh2* m, const double* coords, int nverts,
   PCU_Comm_Begin();
   Gid to = std::min(peers - 1, start / quotient);
   Gid n = std::min((to+1)*quotient-start, nvertsG);
-  if(n > 100000000) {
+  if(n > 1000) {
      lion_eprint(1, "setCoords int overflow of: self=%d,mySize=%d,total=%ld, n=%d,to=%d, quotient=%d, remainder=%d start=%d, peers=%d \n",self,mySize,total,n,to,quotient,remainder,start,peers);
   Gid peersG = PCU_Comm_Peers();
     Gid quotientG = total / peersG;
   Gid remainderG = total % peersG;
-     lion_eprint(1, "setCoords Gid0test: self=%d,mySize=%d,total=%ld, quotientG=%ld, peers=%ld \n",self,mySize,total,quotientG,remainderG,peersG);
+     lion_eprint(1, "setCoords Gid0test: self=%d,mySize=%d,total=%ld, quotientG=%ld, remainderG=%ld,peers=%ld \n",self,mySize,total,quotientG,remainderG,peersG);
 }
 
   while (nvertsG > 0) {

From 9b42069ff8d25256ee60479bfa855c0408fdbc81 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Sat, 29 May 2021 16:11:54 -0700
Subject: [PATCH 337/555] modified the debug statement to report all the
 numbers computed in setCoords

---
 apf/apfConstruct.cc | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index 0b9294d3a..26c08c05f 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -203,11 +203,12 @@ void setCoords(Mesh2* m, const double* coords, int nverts,
   Gid to = std::min(peers - 1, start / quotient);
   Gid n = std::min((to+1)*quotient-start, nvertsG);
   if(n > 1000) {
-     lion_eprint(1, "setCoords int overflow of: self=%d,mySize=%d,total=%ld, n=%d,to=%d, quotient=%d, remainder=%d start=%d, peers=%d \n",self,mySize,total,n,to,quotient,remainder,start,peers);
-  Gid peersG = PCU_Comm_Peers();
-    Gid quotientG = total / peersG;
-  Gid remainderG = total % peersG;
-     lion_eprint(1, "setCoords Gid0test: self=%d,mySize=%d,total=%ld, quotientG=%ld, remainderG=%ld,peers=%ld \n",self,mySize,total,quotientG,remainderG,peersG);
+     Gid sizeToSend=n*3*sizeof(double);
+     lion_eprint(1, "setCoords int overflow of: self=%ld,mySize=%ld,total=%ld, n=%ld,to=%ld, quotient=%ld, remainder=%ld start=%ld, peers=%ld, sizeToSend=%ld, nvertsG=%ld, nverts=%u \n",self,mySize,total,n,to,quotient,remainder,start,peers,sizeToSend,nvertsG,nverts);
+//  Gid peersG = PCU_Comm_Peers();
+//    Gid quotientG = total / peersG;
+//  Gid remainderG = total % peersG;
+//     lion_eprint(1, "setCoords Gid0test: self=%d,mySize=%d,total=%ld, quotientG=%ld, remainderG=%ld,peers=%ld \n",self,mySize,total,quotientG,remainderG,peersG);
 }
 
   while (nvertsG > 0) {

From 0800b2c4ec60179da7d493ba9f3604f06c2358be Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Sun, 30 May 2021 21:12:37 -0700
Subject: [PATCH 338/555] vertices and elements, global, local, start, end all
 made Gid

---
 apf/apfConstruct.cc          |  1 +
 test/matchedNodeElmReader.cc | 54 +++++++++++++++++++-----------------
 2 files changed, 29 insertions(+), 26 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index 26c08c05f..228f43fb6 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -210,6 +210,7 @@ void setCoords(Mesh2* m, const double* coords, int nverts,
 //  Gid remainderG = total % peersG;
 //     lion_eprint(1, "setCoords Gid0test: self=%d,mySize=%d,total=%ld, quotientG=%ld, remainderG=%ld,peers=%ld \n",self,mySize,total,quotientG,remainderG,peersG);
 }
+  PCU_Barrier();
 
   while (nvertsG > 0) {
     PCU_COMM_PACK(to, start);
diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 49fb5ef0e..d03047a28 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -531,8 +531,8 @@ void setClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* t) {
   mesh->acceptChanges();
 }
 
-void getLocalRange(unsigned total, unsigned& local,
-    long& first, long& last) {
+void getLocalRange(apf::Gid total, apf::Gid& local,
+    apf::Gid& first, apf::Gid& last) {
   const int self = PCU_Comm_Self();
   const int peers = PCU_Comm_Peers();
   local = total/peers;
@@ -582,9 +582,9 @@ bool skipLine(char* line) {
   return (line[0] == '#' || line[0] == ' ' );
 }
 
-void getNumVerts(FILE* f, unsigned& verts) {
+void getNumVerts(FILE* f, apf::Gid& verts) {
   rewind(f);
-  gmi_fscanf(f, 1, "%d",  &verts);
+  gmi_fscanf(f, 1, "%ld",  &verts);
 //  verts = 0;
 //  size_t linelimit = 1024;
 //  char* line = new char[linelimit];
@@ -595,14 +595,14 @@ void getNumVerts(FILE* f, unsigned& verts) {
 //  delete [] line;
 }
 
-void readClassification(FILE* f, unsigned numVtx, int** classification) {
-  long firstVtx, lastVtx;
-  unsigned localNumVtx;
+void readClassification(FILE* f, apf::Gid numVtx, int** classification) {
+  apf::Gid firstVtx, lastVtx;
+  apf::Gid localNumVtx;
   getLocalRange(numVtx,localNumVtx,firstVtx,lastVtx);
   *classification = new int[localNumVtx];
   rewind(f);
   int vidx = 0;
-  for(unsigned i=0; i<numVtx; i++) {
+  for(apf::Gid i=0; i<numVtx; i++) {
 //    int id;
     int mdlId;
 //    gmi_fscanf(f, 2, "%d %d", &id, &mdlId);
@@ -620,13 +620,13 @@ void readClassification(FILE* f, unsigned numVtx, int** classification) {
   }*/
 }
 
-void readCoords(FILE* f, unsigned numvtx, unsigned& localnumvtx, double** coordinates) {
-  long firstVtx, lastVtx;
+void readCoords(FILE* f, apf::Gid numvtx, apf::Gid& localnumvtx, double** coordinates) {
+  apf::Gid firstVtx, lastVtx;
   getLocalRange(numvtx, localnumvtx,firstVtx,lastVtx);
   *coordinates = new double[localnumvtx*3];
 //  rewind(f);
   int vidx = 0;
-  for(unsigned i=0; i<numvtx; i++) {
+  for(apf::Gid i=0; i<numvtx; i++) {
 //    int id;
     double pos[3];
 //    gmi_fscanf(f, 4, "%d %lf %lf %lf", &id, pos+0, pos+1, pos+2);
@@ -639,13 +639,13 @@ void readCoords(FILE* f, unsigned numvtx, unsigned& localnumvtx, double** coordi
   }
 }
 
-void readSolution(FILE* f, unsigned numvtx, unsigned& localnumvtx, double** solution) {
-  long firstVtx, lastVtx;
+void readSolution(FILE* f, apf::Gid numvtx, apf::Gid& localnumvtx, double** solution) {
+  apf::Gid firstVtx, lastVtx;
   getLocalRange(numvtx, localnumvtx,firstVtx,lastVtx);
   *solution = new double[localnumvtx*5];
   rewind(f);
   int vidx = 0;
-  for(unsigned i=0; i<numvtx; i++) {
+  for(apf::Gid i=0; i<numvtx; i++) {
     double pos[5];
     pos[4]=0; //temperature
     gmi_fscanf(f, 4, "%lf %lf %lf %lf", pos+0, pos+1, pos+2, pos+3);
@@ -657,9 +657,9 @@ void readSolution(FILE* f, unsigned numvtx, unsigned& localnumvtx, double** solu
   }
 }
 
-void readMatches(FILE* f, unsigned numvtx, int** matches) {
-  long firstVtx, lastVtx;
-  unsigned localnumvtx;
+void readMatches(FILE* f, apf::Gid numvtx, int** matches) {
+  apf::Gid firstVtx, lastVtx;
+  apf::Gid localnumvtx;
   getLocalRange(numvtx, localnumvtx, firstVtx, lastVtx);
   fprintf(stderr, "%d readMatches numvtx %d localnumvtx %u firstVtx %ld lastVtx %ld\n",
       PCU_Comm_Self(), numvtx, localnumvtx, firstVtx, lastVtx);
@@ -667,7 +667,7 @@ void readMatches(FILE* f, unsigned numvtx, int** matches) {
   rewind(f);
   int vidx = 0;
   int matchedVtx;
-  int i = 0;
+  apf::Gid i = 0;
   while( 1 == fscanf(f, "%d", &matchedVtx) ) {
     if( i >= firstVtx && i < lastVtx ) {
       PCU_ALWAYS_ASSERT( matchedVtx == -1 ||
@@ -685,19 +685,21 @@ void readMatches(FILE* f, unsigned numvtx, int** matches) {
   }
 }
 
-void readElements(FILE* f, FILE* fh, unsigned &dim, unsigned& numElms,
-    unsigned& numVtxPerElm, unsigned& localNumElms, apf::Gid** elements) {
+//void readElements(FILE* f, FILE* fh, unsigned &dim, unsigned& numElms,
+void readElements(FILE* f, FILE* fh, unsigned &dim, apf::Gid& numElms,
+    unsigned& numVtxPerElm, apf::Gid& localNumElms, apf::Gid** elements) {
   rewind(f);
   rewind(fh);
   int dimHeader[2];
   gmi_fscanf(fh, 2, "%u %u", dimHeader, dimHeader+1);
   assert( dimHeader[0] == 1 && dimHeader[1] == 1);
   gmi_fscanf(fh, 1, "%u", &dim);
-  gmi_fscanf(fh, 2, "%u %u", &numElms, &numVtxPerElm);
+  gmi_fscanf(fh, 2, "%ld %u", &numElms, &numVtxPerElm);
   long firstElm, lastElm;
   getLocalRange(numElms, localNumElms, firstElm, lastElm);
   *elements = new apf::Gid[localNumElms*numVtxPerElm];
-  unsigned i, j;
+  apf::Gid i;
+  unsigned j;
   unsigned elmIdx = 0;
   apf::Gid* elmVtx = new apf::Gid[numVtxPerElm];
   for (i = 0; i < numElms; i++) {
@@ -723,10 +725,10 @@ struct MeshInfo {
   int* fathers2D;
   unsigned dim;
   unsigned elementType;
-  unsigned numVerts;
-  unsigned localNumVerts;
-  unsigned numElms;
-  unsigned localNumElms;
+  apf::Gid numVerts;
+  apf::Gid localNumVerts;
+  apf::Gid numElms;
+  apf::Gid localNumElms;
   unsigned numVtxPerElm;
 };
 

From 014b5eb53d0507eb15899af33ef81042f35dd2fe Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Sun, 30 May 2021 22:27:23 -0600
Subject: [PATCH 339/555] write format issues

---
 test/matchedNodeElmReader.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index d03047a28..4a7c05a52 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -661,7 +661,7 @@ void readMatches(FILE* f, apf::Gid numvtx, int** matches) {
   apf::Gid firstVtx, lastVtx;
   apf::Gid localnumvtx;
   getLocalRange(numvtx, localnumvtx, firstVtx, lastVtx);
-  fprintf(stderr, "%d readMatches numvtx %d localnumvtx %u firstVtx %ld lastVtx %ld\n",
+  fprintf(stderr, "%d readMatches numvtx %ld localnumvtx %ld firstVtx %ld lastVtx %ld\n",
       PCU_Comm_Self(), numvtx, localnumvtx, firstVtx, lastVtx);
   *matches = new int[localnumvtx];
   rewind(f);
@@ -745,7 +745,7 @@ void readMesh(const char* meshfilename,
   getNumVerts(fc,mesh.numVerts);
   
   if(!PCU_Comm_Self())
-    fprintf(stderr, "numVerts %u\n", mesh.numVerts);
+    fprintf(stderr, "numVerts %ld\n", mesh.numVerts);
   readCoords(fc, mesh.numVerts, mesh.localNumVerts, &(mesh.coords));
   fclose(fc);
  

From c89c45de2fc4a0a551b12e8b4583e3cab38b596d Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Tue, 1 Jun 2021 08:27:25 -0600
Subject: [PATCH 340/555] CWS changes to try to trap bad messages at time of
 packing

---
 pcu/pcu.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/pcu/pcu.c b/pcu/pcu.c
index 0412f8d17..b6c30f901 100644
--- a/pcu/pcu.c
+++ b/pcu/pcu.c
@@ -40,6 +40,8 @@
 #include <sys/types.h> /*required for mode_t for mkdir on some systems*/
 #include <sys/stat.h> /*using POSIX mkdir call for SMB "foo/" path*/
 #include <errno.h> /* for checking the error from mkdir */
+#include <limits.h> /*INT_MAX*/
+#include <stdlib.h> /*abort*/
 
 enum state { uninit, init };
 static enum state global_state = uninit;
@@ -138,6 +140,10 @@ int PCU_Comm_Pack(int to_rank, const void* data, size_t size)
     reel_fail("Comm_Pack called before Comm_Init");
   if ((to_rank < 0)||(to_rank >= pcu_mpi_size()))
     reel_fail("Invalid rank in Comm_Pack");
+  if ( size > (size_t)INT_MAX ) {
+	  fprintf(stderr, "ERROR Attempting to pack a PCU message whose size exceeds INT_MAX... exiting\n");
+	  abort();
+  }
   memcpy(pcu_msg_pack(get_msg(),to_rank,size),data,size);
   return PCU_SUCCESS;
 }

From a94aa3b880448dd1968cc2ff811e84458eda3d5c Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Tue, 1 Jun 2021 12:50:25 -0600
Subject: [PATCH 341/555] lots of places were missed in int->Gid change

---
 apf/apfConstruct.cc          | 182 +++++++++++++++++++----------------
 test/matchedNodeElmReader.cc |  31 +++---
 2 files changed, 115 insertions(+), 98 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index 228f43fb6..f36791337 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -32,8 +32,7 @@ static void constructElements(
     Downward verts;
     int offset = i * nev;
     Gid vCur=conn[offset];
-    int vNext=-1;
-    int uniqueVerts2=1;
+    Gid vNext=-1;
     int uniqueVerts=1;
     for (int j = 0; j < nev; ++j) {
       if(irep ==0){
@@ -49,12 +48,7 @@ static void constructElements(
     if(uniqueVerts==5) etypeL=apf::Mesh::PYRAMID;
     if(uniqueVerts==6) etypeL=apf::Mesh::PRISM;
     if(uniqueVerts==8) etypeL=apf::Mesh::HEX;
-    if(m==NULL) {
-      uniqueVerts2=2;
-    }
-    if(uniqueVerts/uniqueVerts2 > 0){
     buildElement(m, interior, etypeL, verts);
-    }
   }
 }
 
@@ -73,11 +67,11 @@ static Gid getMax(const GlobalToVert& globalToVert)
    I didn't think to use it here, so credit is given. */
 static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
 {
-  Gid max = getMax(globalToVert);
+  Gid max = getMax(globalToVert);  // seems like we read this and know it already on every rank so why compute with global comm?
   Gid total = max + 1;
   int peers = PCU_Comm_Peers();
-  int quotient = total / peers;
-  int remainder = total % peers;
+  int quotient = total / peers; // this seems to work as C++ is doing the math in 64 bit and the result is assumed not to overflow 32
+  int remainder = total % peers; // same
   int mySize = quotient;
   int self = PCU_Comm_Self();
   if (self == (peers - 1))
@@ -88,19 +82,22 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
      broker for that global id */
   PCU_Comm_Begin();
   APF_ITERATE(GlobalToVert, globalToVert, it) {
-    int gid = it->first;
-    int to = std::min(peers - 1, gid / quotient);
+    Gid gid = it->first;  
+    int tmpI=gid / quotient;
+    int to = std::min(peers - 1,tmpI); 
     PCU_COMM_PACK(to, gid);
   }
   PCU_Comm_Send();
-  int myOffset = self * quotient;
+  Gid myOffset = self * quotient;
   /* brokers store all the part ids that sent messages
      for each global id */
   while (PCU_Comm_Receive()) {
-    int gid;
+    Gid gid;
     PCU_COMM_UNPACK(gid);
     int from = PCU_Comm_Sender();
-    tmpParts.at(gid - myOffset).push_back(from);
+    Gid tmpL=gid - myOffset; // forcing 64 bit difference until we know it is safe
+    int tmpI=tmpL;
+    tmpParts.at(tmpI).push_back(from);
   }
   /* for each global id, send all associated part ids
      to all associated parts */
@@ -109,7 +106,7 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
     std::vector<int>& parts = tmpParts[i];
     for (size_t j = 0; j < parts.size(); ++j) {
       int to = parts[j];
-      int gid = i + myOffset;
+      Gid gid = i + myOffset;
       int nparts = parts.size();
       PCU_COMM_PACK(to, gid);
       PCU_COMM_PACK(to, nparts);
@@ -122,7 +119,7 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
      lookup the vertex and classify it on the partition
      model entity for that set of parts */
   while (PCU_Comm_Receive()) {
-    int gid;
+    Gid gid;
     PCU_COMM_UNPACK(gid);
     int nparts;
     PCU_COMM_UNPACK(nparts);
@@ -145,7 +142,7 @@ static void constructRemotes(Mesh2* m, GlobalToVert& globalToVert)
   int self = PCU_Comm_Self();
   PCU_Comm_Begin();
   APF_ITERATE(GlobalToVert, globalToVert, it) {
-    int gid = it->first;
+    Gid gid = it->first;
     MeshEntity* vert = it->second;
     Parts residence;
     m->getResidence(vert, residence);
@@ -157,7 +154,7 @@ static void constructRemotes(Mesh2* m, GlobalToVert& globalToVert)
   }
   PCU_Comm_Send();
   while (PCU_Comm_Receive()) {
-    int gid;
+    Gid gid;
     PCU_COMM_UNPACK(gid);
     MeshEntity* remote;
     PCU_COMM_UNPACK(remote);
@@ -181,14 +178,13 @@ void construct(Mesh2* m, const Gid* conn, int nelem, int etype,
 void setCoords(Mesh2* m, const double* coords, int nverts,
     GlobalToVert& globalToVert)
 {
-  Gid nvertsG=nverts;
   Gid max = getMax(globalToVert);
   Gid total = max + 1;
-  Gid peers = PCU_Comm_Peers();
-  Gid quotient = total / peers;
-  Gid remainder = total % peers;
-  Gid mySize = quotient;
-  Gid self = PCU_Comm_Self();
+  int peers = PCU_Comm_Peers();
+  int quotient = total / peers;
+  int remainder = total % peers;
+  int mySize = quotient;
+  int self = PCU_Comm_Self();
   if (self == (peers - 1))
     mySize += remainder;
   Gid myOffset = self * quotient;
@@ -197,14 +193,18 @@ void setCoords(Mesh2* m, const double* coords, int nverts,
      This means we might need to send and recv some coords */
   double* c = new double[mySize*3];
 
-  Gid start = PCU_Exscan_Int(nverts);
+  Gid start = PCU_Exscan_Long(nverts);
 
-  PCU_Comm_Begin();
-  Gid to = std::min(peers - 1, start / quotient);
-  Gid n = std::min((to+1)*quotient-start, nvertsG);
+  PCU_Comm_Begin();  // the forced 64 bit math below may not be necessary 
+  Gid tmpL=start / quotient; 
+  int tmpInt=tmpL;
+  int to = std::min(peers - 1, tmpInt);
+  tmpL=(to+1)*quotient-start; 
+  tmpInt=tmpL;
+  int n = std::min(tmpInt, nverts);
   if(n > 1000) {
      Gid sizeToSend=n*3*sizeof(double);
-     lion_eprint(1, "setCoords int overflow of: self=%ld,mySize=%ld,total=%ld, n=%ld,to=%ld, quotient=%ld, remainder=%ld start=%ld, peers=%ld, sizeToSend=%ld, nvertsG=%ld, nverts=%u \n",self,mySize,total,n,to,quotient,remainder,start,peers,sizeToSend,nvertsG,nverts);
+     lion_eprint(1, "setCoords int overflow of: self=%ld,mySize=%ld,total=%ld, n=%ld,to=%ld, quotient=%ld, remainder=%ld start=%ld, peers=%ld, sizeToSend=%ld, nverts=%u \n",self,mySize,total,n,to,quotient,remainder,start,peers,sizeToSend,nverts);
 //  Gid peersG = PCU_Comm_Peers();
 //    Gid quotientG = total / peersG;
 //  Gid remainderG = total % peersG;
@@ -212,31 +212,33 @@ void setCoords(Mesh2* m, const double* coords, int nverts,
 }
   PCU_Barrier();
 
-  while (nvertsG > 0) {
+  while (nverts > 0) {
     PCU_COMM_PACK(to, start);
     PCU_COMM_PACK(to, n);
     PCU_Comm_Pack(to, coords, n*3*sizeof(double));
 
-    nvertsG -= n;
+    nverts -= n;
     start += n;
     coords += n*3;
     to = std::min(peers - 1, to + 1);
-    n = std::min(quotient, nvertsG);
+    n = std::min(quotient, nverts);
   }
   PCU_Comm_Send();
   while (PCU_Comm_Receive()) {
     PCU_COMM_UNPACK(start);
-    PCU_COMM_UNPACK(n);
+    PCU_COMM_UNPACK(n); // |||||| more in-place 64 bit math
     PCU_Comm_Unpack(&c[(start - myOffset) * 3], n*3*sizeof(double));
   }
 
   /* Tell all the owners of the coords what we need */
-  typedef std::vector< std::vector<Gid> > TmpParts;
+  typedef std::vector< std::vector<int> > TmpParts;
   TmpParts tmpParts(mySize);
   PCU_Comm_Begin();
   APF_CONST_ITERATE(GlobalToVert, globalToVert, it) {
     Gid gid = it->first;
-    Gid to = std::min(peers - 1, gid / quotient);
+    Gid tmpL=gid / quotient;
+    int tmpInt=tmpL;
+    int to = std::min(peers - 1, tmpInt);
     PCU_COMM_PACK(to, gid);
   }
   PCU_Comm_Send();
@@ -244,15 +246,17 @@ void setCoords(Mesh2* m, const double* coords, int nverts,
     Gid gid;
     PCU_COMM_UNPACK(gid);
     Gid from = PCU_Comm_Sender();
-    tmpParts.at(gid - myOffset).push_back(from);
+    Gid tmpL=gid - myOffset;
+    int tmpInt=tmpL;
+    tmpParts.at(tmpInt).push_back(from);
   }
 
   /* Send the coords to everybody who want them */
   PCU_Comm_Begin();
-  for (Gid i = 0; i < mySize; ++i) {
-    std::vector<Gid>& parts = tmpParts[i];
+  for (int i = 0; i < mySize; ++i) {
+    std::vector<int>& parts = tmpParts[i];
     for (size_t j = 0; j < parts.size(); ++j) {
-      Gid to = parts[j];
+      int to = parts[j];
       Gid gid = i + myOffset;
       PCU_COMM_PACK(to, gid);
       PCU_Comm_Pack(to, &c[i*3], 3*sizeof(double));
@@ -284,23 +288,28 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
   int self = PCU_Comm_Self();
   if (self == (peers - 1))
     mySize += remainder;
-  int myOffset = self * quotient;
+  Gid myOffset = self * quotient;
 
   /* Force each peer to have exactly mySize verts.
      This means we might need to send and recv some matches */
   int* c = new int[mySize];
   PCU_Debug_Print("%d mysize %d\n", self, mySize);
 
-  int start = PCU_Exscan_Int(nverts);
+  Gid start = PCU_Exscan_Long(nverts);
 
   PCU_Comm_Begin();
-  int to = std::min(peers - 1, start / quotient);
-  int n = std::min((to+1)*quotient-start, nverts);
+
+  Gid tmpL=start / quotient; 
+  int tmpInt=tmpL;
+  int to = std::min(peers - 1, tmpInt);
+  tmpL=(to+1)*quotient-start; 
+  tmpInt=tmpL;
+  int n = std::min(tmpInt, nverts);
   while (nverts > 0) {
     PCU_COMM_PACK(to, start);
     PCU_COMM_PACK(to, n);
     PCU_Comm_Pack(to, matches, n*sizeof(int));
-    PCU_Debug_Print("%d sending start %d n %d to %d\n",
+    PCU_Debug_Print("%d sending start %ld n %d to %d\n",
         self, start, n, to);
 
     nverts -= n;
@@ -312,16 +321,16 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
   PCU_Comm_Send();
   while (PCU_Comm_Receive()) {
     PCU_COMM_UNPACK(start);
-    PCU_COMM_UNPACK(n);
+    PCU_COMM_UNPACK(n); /// in-place 64
     PCU_Comm_Unpack(&c[(start - myOffset)], n*sizeof(int));
-    PCU_Debug_Print("%d reciving start %d n %d from %d\n",
+    PCU_Debug_Print("%d receiving start %ld n %d from %d\n",
         self, start, n, PCU_Comm_Sender());
   }
 
   for (int i = 0; i < mySize; ++i) {
     int match = c[i];
     if( match != -1 )
-      PCU_Debug_Print("%d found match %d at gid %d\n",
+      PCU_Debug_Print("%d found match %d at gid %ld\n",
           self, match, i+myOffset);
   }
 
@@ -330,15 +339,16 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
   TmpParts tmpParts(mySize);
   PCU_Comm_Begin();
   APF_CONST_ITERATE(GlobalToVert, globalToVert, it) {
-    int gid = it->first;
-    int to = std::min(peers - 1, gid / quotient);
+    Gid gid = it->first;  
+    int tmpI=gid / quotient;
+    int to = std::min(peers - 1,tmpI); 
     PCU_COMM_PACK(to, gid);
-    PCU_Debug_Print("%d requesting matches of gid %d isShared %d isOwned %d from %d\n",
+    PCU_Debug_Print("%d requesting matches of gid %ld isShared %d isOwned %d from %d\n",
         self, gid, m->isShared(it->second), m->isOwned(it->second), to);
   }
   PCU_Comm_Send();
   while (PCU_Comm_Receive()) {
-    int gid;
+    Gid gid;
     PCU_COMM_UNPACK(gid);
     int from = PCU_Comm_Sender();
     tmpParts.at(gid - myOffset).push_back(from);
@@ -351,27 +361,27 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
     std::vector<int>& parts = tmpParts[i];
     for (size_t j = 0; j < parts.size(); ++j) {
       int to = parts[j];
-      int gid = i + myOffset;
-      int matchGid = c[i];
+      Gid gid = i + myOffset;
+      Gid matchGid = c[i];
       PCU_COMM_PACK(to, gid);
       PCU_COMM_PACK(to, matchGid);
       if( matchGid != -1 ) {
-        PCU_Debug_Print("%d packing i %d gid %d matchGid %d to %d\n",
+        PCU_Debug_Print("%d packing i %d gid %ld matchGid %ld to %d\n",
             self, i, gid, matchGid, to);
       }
     }
   }
   PCU_Comm_Send();
   while (PCU_Comm_Receive()) {
-    int gid;
+    Gid gid;
     PCU_COMM_UNPACK(gid);
-    int match;
+    Gid match;
     PCU_COMM_UNPACK(match);
     PCU_ALWAYS_ASSERT(gid != match);
     PCU_ALWAYS_ASSERT(globalToVert.count(gid));
-    m->setIntTag(globalToVert[gid], matchGidTag, &match);
+    m->setLongTag(globalToVert[gid], matchGidTag, &match);
     if( match != -1 ) {
-      PCU_Debug_Print("%d attaching match %d to gid %d\n",
+      PCU_Debug_Print("%d attaching match %ld to gid %ld\n",
           self, match, gid);
     }
   }
@@ -384,55 +394,57 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
   PCU_Comm_Begin();
   APF_CONST_ITERATE(GlobalToVert, globalToVert, it) {
     MeshEntity* e = it->second;
-    int gid = it->first;
-    int to = std::min(peers - 1, gid / quotient);
+    Gid gid = it->first;
+    int tmpI=gid / quotient;
+    int to = std::min(peers - 1,tmpI); 
     PCU_COMM_PACK(to, gid);
     PCU_COMM_PACK(to, e);
-    PCU_Debug_Print("%d packing pointer to %d gid %d vert %p\n",
+    PCU_Debug_Print("%d packing pointer to %d gid %ld vert %p\n",
         self, to, gid, (void*)e);
   }
   PCU_Comm_Send();
   typedef std::pair< int, apf::MeshEntity* > EntOwnerPtrs;
-  typedef std::map< int, std::vector< EntOwnerPtrs > > GidPtrs;
+  typedef std::map< Gid, std::vector< EntOwnerPtrs > > GidPtrs;
   GidPtrs gidPtrs;
   while (PCU_Comm_Receive()) {
-    int gid;
+    Gid gid;
     PCU_COMM_UNPACK(gid);
     MeshEntity* vert;
     PCU_COMM_UNPACK(vert);
     int owner = PCU_Comm_Sender();
     gidPtrs[gid-myOffset].push_back(EntOwnerPtrs(owner,vert));
-    PCU_Debug_Print("%d unpacking pointer from %d gid %d vert %p\n",
+    PCU_Debug_Print("%d unpacking pointer from %d gid %ld vert %p\n",
         self, owner, gid, (void*)vert);
   }
 
   /* Tell the brokers of the matches we need */
-  typedef std::pair<int,int> MatchingPair;
-  typedef std::map< MatchingPair, std::vector<int> > MatchMap;
+  typedef std::pair<Gid,Gid> MatchingPair;
+  typedef std::map< MatchingPair, std::vector<Gid> > MatchMap;
   MatchMap matchParts;
   PCU_Comm_Begin();
   APF_CONST_ITERATE(GlobalToVert, globalToVert, it) { //loop over local verts
-    int gid = it->first;
-    int matchGid;
-    m->getIntTag(it->second, matchGidTag, &matchGid); //get the matched ent gid
+    Gid gid = it->first;
+    Gid matchGid;
+    m->getLongTag(it->second, matchGidTag, &matchGid); //get the matched ent gid
     if( matchGid != -1 ) {  // marker for an unmatched vertex
-      int to = std::min(peers - 1, matchGid / quotient); // broker
+      int tmpI=matchGid / quotient;
+      int to = std::min(peers - 1,tmpI);  //broker
       PCU_COMM_PACK(to, gid); // send the local vert gid
       PCU_COMM_PACK(to, matchGid); // and the match gid needed
-      PCU_Debug_Print("%d packing req ptr to %d gid %d matchGid %d\n",
+      PCU_Debug_Print("%d packing req ptr to %d gid %ld matchGid %ld\n",
           self, to, gid, matchGid);
     }
   }
   PCU_Comm_Send();
   while (PCU_Comm_Receive()) {
-    int gid;
+    Gid gid;
     PCU_COMM_UNPACK(gid); // request from entity gid
-    int matchGid;
+    Gid matchGid;
     PCU_COMM_UNPACK(matchGid); // requesting matched entity gid 
     MatchingPair mp(gid,matchGid);
     int from = PCU_Comm_Sender();
     matchParts[mp].push_back(from); // store a list of the proceses that need the pair (entity gid, match gid)
-    PCU_Debug_Print("%d unpacking ptr req from %d gid %d matchGid %d\n",
+    PCU_Debug_Print("%d unpacking ptr req from %d gid %ld matchGid %ld\n",
         self, from, gid, matchGid);
   }
 
@@ -441,9 +453,9 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
   PCU_Comm_Begin();
   APF_ITERATE(MatchMap,matchParts,it) {
     MatchingPair mp = it->first;
-    int gid = mp.first;
-    int matchGid = mp.second;
-    std::vector<int> parts = it->second;
+    Gid gid = mp.first;
+    Gid matchGid = mp.second;
+    std::vector<Gid> parts = it->second;
     for(size_t i=0; i<parts.size(); i++) {
       const int to = parts[i];
       PCU_COMM_PACK(to, gid);
@@ -456,16 +468,16 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
         PCU_COMM_PACK(to, owner);
         apf::MeshEntity* ent = eop.second;
         PCU_COMM_PACK(to, ent);
-        PCU_Debug_Print("%d packing match ptr to %d gid %d matchGid %d vert %p owner %d\n",
+        PCU_Debug_Print("%d packing match ptr to %d gid %ld matchGid %ld vert %p owner %d\n",
             self, to, gid, matchGid, (void*)ent, owner);
       }
     }
   }
   PCU_Comm_Send();
   while (PCU_Comm_Receive()) {
-    int gid;
+    Gid gid;
     PCU_COMM_UNPACK(gid);
-    int matchGid;
+    Gid matchGid;
     PCU_COMM_UNPACK(matchGid);
     size_t numMatches;
     PCU_COMM_UNPACK(numMatches);
@@ -474,7 +486,7 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
       PCU_COMM_UNPACK(owner);
       MeshEntity* match;
       PCU_COMM_UNPACK(match);
-      PCU_Debug_Print("%d unpacked match ptr from %d gid %d matchGid %d matchPtr %p owner %d\n",
+      PCU_Debug_Print("%d unpacked match ptr from %d gid %ld matchGid %ld matchPtr %p owner %d\n",
           self, PCU_Comm_Sender(), gid, matchGid, (void*)match, owner);
       PCU_ALWAYS_ASSERT(globalToVert.count(gid));
       MeshEntity* partner = globalToVert[gid];
@@ -489,8 +501,8 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
 
   APF_CONST_ITERATE(GlobalToVert, globalToVert, it) { //loop over local verts
     apf::MeshEntity* left = it->second;
-    int matchGid;
-    m->getIntTag(left, matchGidTag, &matchGid); //get the matched ent gid
+    Gid matchGid;
+    m->getLongTag(left, matchGidTag, &matchGid); //get the matched ent gid
     if( matchGid != -1 ) {  // a matched vtx
       apf::Copies copies;
       m->getRemotes(left,copies);
@@ -499,7 +511,7 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
         apf::MeshEntity* right = cp->second;
         m->addMatch(left, rightPart, right);
         PCU_Debug_Print("%d add remote copy match ptr to %d gid %ld\n",
-          self, rightPart, it->first);
+          self, rightPart, it->first); // is this last argument correct
       }
     }
   }
diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 4a7c05a52..8581ed360 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -531,14 +531,19 @@ void setClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* t) {
   mesh->acceptChanges();
 }
 
-void getLocalRange(apf::Gid total, apf::Gid& local,
+void getLocalRange(apf::Gid total, int& local,
     apf::Gid& first, apf::Gid& last) {
   const int self = PCU_Comm_Self();
   const int peers = PCU_Comm_Peers();
-  local = total/peers;
-  if( self == peers-1 ) //last rank
-    if( local*peers < total )
-      local += total - local*peers;
+  local = total/peers; 
+  if( self == peers-1 ) {  //last rank
+    apf::Gid lp=local*peers;
+    if( lp < total ){
+      apf::Gid lpd;
+      lpd= total - lp;
+      local += lpd;
+    }
+  }
   first = PCU_Exscan_Long(local);
   last = first+local;
 }
@@ -597,7 +602,7 @@ void getNumVerts(FILE* f, apf::Gid& verts) {
 
 void readClassification(FILE* f, apf::Gid numVtx, int** classification) {
   apf::Gid firstVtx, lastVtx;
-  apf::Gid localNumVtx;
+  int localNumVtx;
   getLocalRange(numVtx,localNumVtx,firstVtx,lastVtx);
   *classification = new int[localNumVtx];
   rewind(f);
@@ -620,7 +625,7 @@ void readClassification(FILE* f, apf::Gid numVtx, int** classification) {
   }*/
 }
 
-void readCoords(FILE* f, apf::Gid numvtx, apf::Gid& localnumvtx, double** coordinates) {
+void readCoords(FILE* f, apf::Gid numvtx, int& localnumvtx, double** coordinates) {
   apf::Gid firstVtx, lastVtx;
   getLocalRange(numvtx, localnumvtx,firstVtx,lastVtx);
   *coordinates = new double[localnumvtx*3];
@@ -639,7 +644,7 @@ void readCoords(FILE* f, apf::Gid numvtx, apf::Gid& localnumvtx, double** coordi
   }
 }
 
-void readSolution(FILE* f, apf::Gid numvtx, apf::Gid& localnumvtx, double** solution) {
+void readSolution(FILE* f, apf::Gid numvtx, int& localnumvtx, double** solution) {
   apf::Gid firstVtx, lastVtx;
   getLocalRange(numvtx, localnumvtx,firstVtx,lastVtx);
   *solution = new double[localnumvtx*5];
@@ -659,9 +664,9 @@ void readSolution(FILE* f, apf::Gid numvtx, apf::Gid& localnumvtx, double** solu
 
 void readMatches(FILE* f, apf::Gid numvtx, int** matches) {
   apf::Gid firstVtx, lastVtx;
-  apf::Gid localnumvtx;
+  int localnumvtx;
   getLocalRange(numvtx, localnumvtx, firstVtx, lastVtx);
-  fprintf(stderr, "%d readMatches numvtx %ld localnumvtx %ld firstVtx %ld lastVtx %ld\n",
+  fprintf(stderr, "%d readMatches numvtx %ld localnumvtx %d firstVtx %ld lastVtx %ld\n",
       PCU_Comm_Self(), numvtx, localnumvtx, firstVtx, lastVtx);
   *matches = new int[localnumvtx];
   rewind(f);
@@ -687,7 +692,7 @@ void readMatches(FILE* f, apf::Gid numvtx, int** matches) {
 
 //void readElements(FILE* f, FILE* fh, unsigned &dim, unsigned& numElms,
 void readElements(FILE* f, FILE* fh, unsigned &dim, apf::Gid& numElms,
-    unsigned& numVtxPerElm, apf::Gid& localNumElms, apf::Gid** elements) {
+    unsigned& numVtxPerElm, int& localNumElms, apf::Gid** elements) {
   rewind(f);
   rewind(fh);
   int dimHeader[2];
@@ -726,9 +731,9 @@ struct MeshInfo {
   unsigned dim;
   unsigned elementType;
   apf::Gid numVerts;
-  apf::Gid localNumVerts;
+  int localNumVerts;
   apf::Gid numElms;
-  apf::Gid localNumElms;
+  int localNumElms;
   unsigned numVtxPerElm;
 };
 

From 15bb9279d7e00886deec45691e482e8cbc29ea7d Mon Sep 17 00:00:00 2001
From: Morteza HS <hakimi1364@gmail.com>
Date: Tue, 1 Jun 2021 15:54:01 -0400
Subject: [PATCH 342/555] Adds a new functionality to maDBG

this debug functionality allows the uniform refinement of entities
classified on a specific model entity as identified by a pair of
integers (model dim, model tag)
---
 ma/maDBG.cc | 45 +++++++++++++++++++++++++++++++++++++++++++++
 ma/maDBG.h  |  6 +++++-
 2 files changed, 50 insertions(+), 1 deletion(-)

diff --git a/ma/maDBG.cc b/ma/maDBG.cc
index 808bedf3d..71d4459ea 100644
--- a/ma/maDBG.cc
+++ b/ma/maDBG.cc
@@ -10,6 +10,7 @@
 #include "maDBG.h"
 #include "maShape.h"
 #include "maAdapt.h"
+#include "maRefine.h"
 #include <gmi.h>
 #include <gmi_null.h>
 #include <PCU.h>
@@ -386,4 +387,48 @@ void visualizeSizeField(
   apf::destroyMesh(msf);
 }
 
+struct SplitByTag : public ma::Predicate
+{
+  SplitByTag(ma::Adapt* a_, int type_, int tag_) :
+    a(a_), type(type_), tag(tag_) {}
+  bool operator() (ma::Entity* e) {
+    ma::Mesh* m = a->mesh;
+    int mtype = m->getModelType(m->toModel(e));
+    int mtag  = m->getModelTag(m->toModel(e));
+    if (mtype != type) return false;
+    if (mtag  != tag ) return false;
+    return true;
+  }
+  ma::Adapt* a;
+  int type;
+  int tag;
+};
+
+void uniformAdaptByModelTag(
+    apf::Mesh2* m,
+    int mtype,
+    int mtag,
+    int level)
+{
+  ma::Input* in = ma::configureUniformRefine(m, 0);
+  ma::validateInput(in);
+  ma::Adapt* a = new ma::Adapt(in);
+  for(int i = 0; i < level; i++) {
+    SplitByTag p(a, mtype, mtag);
+    ma::markEntities(a, 1, p, ma::SPLIT, ma::NEED_NOT_SPLIT,
+    	ma::DONT_SPLIT | ma::NEED_NOT_SPLIT);
+    PCU_ALWAYS_ASSERT(ma::checkFlagConsistency(a,1,ma::SPLIT));
+    ma::Refine* r = a->refine;
+    ma::resetCollection(r);
+    ma::collectForTransfer(r);
+    ma::collectForMatching(r);
+    ma::addAllMarkedEdges(r);
+    ma::splitElements(r);
+    ma::processNewElements(r);
+    ma::destroySplitElements(r);
+    ma::forgetNewEntities(r);
+  }
+  delete(a);
+}
+
 }
diff --git a/ma/maDBG.h b/ma/maDBG.h
index 751450aa3..027a245fd 100644
--- a/ma/maDBG.h
+++ b/ma/maDBG.h
@@ -67,6 +67,10 @@ void visualizeSizeField(
     int smapleSize[2],
     double userScale,
     const char* OutputPrefix);
-
+void uniformAdaptByModelTag(
+    apf::Mesh2* m,
+    int mtype,
+    int mtag,
+    int level);
 }
 #endif

From 00b46ab261472e2e6cfe7711605cd230f69e604e Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Tue, 1 Jun 2021 19:34:19 -0600
Subject: [PATCH 343/555] attempt to get nodes better aligned with streamed in
 element partitions by accounting for multitopology

---
 test/matchedNodeElmReader.cc | 40 +++++++++++++++++++++++++++++++++---
 1 file changed, 37 insertions(+), 3 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 8581ed360..64dfaa282 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -531,6 +531,35 @@ void setClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* t) {
   mesh->acceptChanges();
 }
 
+void getLocalRangeMT(apf::Gid total, apf::Gid& numWedge, apf::Gid& numTet, int& local,
+    apf::Gid& first, apf::Gid& last) {
+  const int self = PCU_Comm_Self();
+  const int peers = PCU_Comm_Peers();
+  apf::Gid Nverte=numWedge/2+numTet/6;
+  int nvertpp=Nverte/peers;
+  float xnwr=peers/(1.0+numTet/(3.0*numWedge));
+  int nAllWedge=xnwr;
+  int nWpp=numWedge/xnwr;
+  int localw= numWedge - nAllWedge*nWpp;
+  int ntetpartial=6*(nvertpp-localw/2);
+  if(self < nAllWedge) 
+     local=nWpp;
+  else if( self==nAllWedge)  // mixed
+      local=localw+ntetpartial;
+  else 
+     local=(numTet-ntetpartial)/(peers-nAllWedge-1);
+  apf::Gid lp=PCU_Exscan_Long(local)+local;
+  if( self == peers-1 ) {  //last rank
+    if( lp < total ){
+      apf::Gid lpd;
+      lpd= total - lp;
+      local += lpd;
+    }
+  }
+  first = PCU_Exscan_Long(local);
+  last = first+local;
+}
+
 void getLocalRange(apf::Gid total, int& local,
     apf::Gid& first, apf::Gid& last) {
   const int self = PCU_Comm_Self();
@@ -697,11 +726,16 @@ void readElements(FILE* f, FILE* fh, unsigned &dim, apf::Gid& numElms,
   rewind(fh);
   int dimHeader[2];
   gmi_fscanf(fh, 2, "%u %u", dimHeader, dimHeader+1);
-  assert( dimHeader[0] == 1 && dimHeader[1] == 1);
+//  assert( dimHeader[0] == 1 && dimHeader[1] == 1);
   gmi_fscanf(fh, 1, "%u", &dim);
   gmi_fscanf(fh, 2, "%ld %u", &numElms, &numVtxPerElm);
-  long firstElm, lastElm;
-  getLocalRange(numElms, localNumElms, firstElm, lastElm);
+  apf::Gid numWedge, numTet;
+  int ijunk;
+  gmi_fscanf(fh, 2, "%ld %u", &numWedge, &ijunk);
+  gmi_fscanf(fh, 2, "%ld %u", &numTet, &ijunk);
+  apf::Gid firstElm, lastElm;
+//  getLocalRange(numElms, localNumElms, firstElm, lastElm);
+  getLocalRangeMT(numElms, numWedge, numTet, localNumElms, firstElm, lastElm);
   *elements = new apf::Gid[localNumElms*numVtxPerElm];
   apf::Gid i;
   unsigned j;

From f739597386af6de172bd6162b10aca2f0760c140 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Wed, 2 Jun 2021 09:54:07 -0700
Subject: [PATCH 344/555] still chasing out missed Gid=> int issues

---
 test/matchedNodeElmReader.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 64dfaa282..e2dcfa06f 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -700,12 +700,12 @@ void readMatches(FILE* f, apf::Gid numvtx, int** matches) {
   *matches = new int[localnumvtx];
   rewind(f);
   int vidx = 0;
-  int matchedVtx;
+  apf::Gid matchedVtx;
   apf::Gid i = 0;
-  while( 1 == fscanf(f, "%d", &matchedVtx) ) {
+  while( 1 == fscanf(f, "%ld", &matchedVtx) ) {
     if( i >= firstVtx && i < lastVtx ) {
       PCU_ALWAYS_ASSERT( matchedVtx == -1 ||
-          ( matchedVtx >= 1 && matchedVtx <= static_cast<int>(numvtx) ));
+          ( matchedVtx >= 1 && matchedVtx <= numvtx ));
       if( matchedVtx != -1 )
         --matchedVtx;
 //      if( matchedVtx == 66350 || matchedVtx == 65075 ) {

From 935a77c00cb08f686233aade10fa7204aab87f63 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Wed, 2 Jun 2021 19:52:53 -0600
Subject: [PATCH 345/555] read parted files from MGEN

---
 test/matchedNodeElmReader.cc | 119 ++++++++++++++++-------------------
 1 file changed, 55 insertions(+), 64 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index e2dcfa06f..51d152c38 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -1,3 +1,4 @@
+//  char* filename;
 #include <gmi_mesh.h>
 #include <gmi_null.h>
 #include <apfMDS.h>
@@ -13,6 +14,9 @@
 #include <cassert>
 #include <algorithm>
 #include <apfBox.h>
+#include <sstream>
+#include <iostream>
+
 
 /* from https://github.com/SCOREC/core/issues/205
 0=fully interior of the volume
@@ -654,103 +658,78 @@ void readClassification(FILE* f, apf::Gid numVtx, int** classification) {
   }*/
 }
 
-void readCoords(FILE* f, apf::Gid numvtx, int& localnumvtx, double** coordinates) {
-  apf::Gid firstVtx, lastVtx;
-  getLocalRange(numvtx, localnumvtx,firstVtx,lastVtx);
+void readCoords(FILE* f, int& localnumvtx, double** coordinates) {
   *coordinates = new double[localnumvtx*3];
-//  rewind(f);
   int vidx = 0;
-  for(apf::Gid i=0; i<numvtx; i++) {
-//    int id;
+  for(int i=0; i<localnumvtx; i++) {
     double pos[3];
-//    gmi_fscanf(f, 4, "%d %lf %lf %lf", &id, pos+0, pos+1, pos+2);
     gmi_fscanf(f, 3, "%lf %lf %lf", pos+0, pos+1, pos+2);
-    if( i >= firstVtx && i < lastVtx ) {
-      for(unsigned j=0; j<3; j++)
+    for(unsigned j=0; j<3; j++)
         (*coordinates)[vidx*3+j] = pos[j];
-      vidx++;
-    }
+    vidx++;
   }
 }
 
-void readSolution(FILE* f, apf::Gid numvtx, int& localnumvtx, double** solution) {
-  apf::Gid firstVtx, lastVtx;
-  getLocalRange(numvtx, localnumvtx,firstVtx,lastVtx);
+void readSolution(FILE* f, int& localnumvtx, double** solution) {
   *solution = new double[localnumvtx*5];
   rewind(f);
   int vidx = 0;
-  for(apf::Gid i=0; i<numvtx; i++) {
+  for(int i=0; i<localnumvtx; i++) {
     double pos[5];
     pos[4]=0; //temperature
     gmi_fscanf(f, 4, "%lf %lf %lf %lf", pos+0, pos+1, pos+2, pos+3);
-    if( i >= firstVtx && i < lastVtx ) {
-      for(unsigned j=0; j<5; j++)
-        (*solution)[vidx*5+j] = pos[j];
-      vidx++;
-    }
+    for(unsigned j=0; j<5; j++)
+      (*solution)[vidx*5+j] = pos[j];
+    vidx++;
   }
 }
 
-void readMatches(FILE* f, apf::Gid numvtx, int** matches) {
-  apf::Gid firstVtx, lastVtx;
-  int localnumvtx;
-  getLocalRange(numvtx, localnumvtx, firstVtx, lastVtx);
-  fprintf(stderr, "%d readMatches numvtx %ld localnumvtx %d firstVtx %ld lastVtx %ld\n",
-      PCU_Comm_Self(), numvtx, localnumvtx, firstVtx, lastVtx);
-  *matches = new int[localnumvtx];
+void readMatches(FILE* f, int localnumvtx, apf::Gid** matches) {
+  fprintf(stderr, "%d readMatches localnumvtx \n",
+      PCU_Comm_Self(), localnumvtx);
+  *matches = new apf::Gid[localnumvtx];
   rewind(f);
   int vidx = 0;
   apf::Gid matchedVtx;
   apf::Gid i = 0;
   while( 1 == fscanf(f, "%ld", &matchedVtx) ) {
-    if( i >= firstVtx && i < lastVtx ) {
-      PCU_ALWAYS_ASSERT( matchedVtx == -1 ||
-          ( matchedVtx >= 1 && matchedVtx <= numvtx ));
-      if( matchedVtx != -1 )
+    PCU_ALWAYS_ASSERT( matchedVtx == -1 ||
+       ( matchedVtx >= 1 && matchedVtx <= numvtx ));
+    if( matchedVtx != -1 )
         --matchedVtx;
-//      if( matchedVtx == 66350 || matchedVtx == 65075 ) {
-//        fprintf(stderr, "%d reader found match %d at gid %d i %d vidx %d\n",
- //           PCU_Comm_Self(), matchedVtx, gid, i, vidx);
-//      }
-      (*matches)[vidx] = matchedVtx;
-      vidx++;
-    }
+    (*matches)[vidx] = matchedVtx;
+    vidx++;
     i++;
   }
 }
 
-//void readElements(FILE* f, FILE* fh, unsigned &dim, unsigned& numElms,
-void readElements(FILE* f, FILE* fh, unsigned &dim, apf::Gid& numElms,
+void readElements(FILE* f, FILE* fh, unsigned &dim,  apf::Gid& numElms,
     unsigned& numVtxPerElm, int& localNumElms, apf::Gid** elements) {
   rewind(f);
   rewind(fh);
   int dimHeader[2];
+  apf::Gid numElms;  // not used but save anyway
   gmi_fscanf(fh, 2, "%u %u", dimHeader, dimHeader+1);
 //  assert( dimHeader[0] == 1 && dimHeader[1] == 1);
   gmi_fscanf(fh, 1, "%u", &dim);
   gmi_fscanf(fh, 2, "%ld %u", &numElms, &numVtxPerElm);
-  apf::Gid numWedge, numTet;
-  int ijunk;
-  gmi_fscanf(fh, 2, "%ld %u", &numWedge, &ijunk);
-  gmi_fscanf(fh, 2, "%ld %u", &numTet, &ijunk);
-  apf::Gid firstElm, lastElm;
-//  getLocalRange(numElms, localNumElms, firstElm, lastElm);
-  getLocalRangeMT(numElms, numWedge, numTet, localNumElms, firstElm, lastElm);
+  int self = PCU_Comm_Self();
+  const int peers = PCU_Comm_Peers();
+  for (int j=0; j<= self,j++)
+     gmi_fscanf(fh, 2, "%d %u", &localNumElms, &numVtxPerElm);
   *elements = new apf::Gid[localNumElms*numVtxPerElm];
-  apf::Gid i;
+  int i;
   unsigned j;
   unsigned elmIdx = 0;
   apf::Gid* elmVtx = new apf::Gid[numVtxPerElm];
   for (i = 0; i < numElms; i++) {
     for (j = 0; j < numVtxPerElm; j++)
       gmi_fscanf(f, 1, "%ld", elmVtx+j);
-    if (i >= firstElm && i < lastElm) {
-      for (j = 0; j < numVtxPerElm; j++) {
-        const unsigned elmVtxIdx = elmIdx*numVtxPerElm+j;
-        (*elements)[elmVtxIdx] = --(elmVtx[j]); //export from matlab using 1-based indices
-      }
-      elmIdx++;
+    for (j = 0; j < numVtxPerElm; j++) {
+      const unsigned elmVtxIdx = elmIdx*numVtxPerElm+j;
+      (*elements)[elmVtxIdx] = --(elmVtx[j]); //export from matlab using 1-based indices
     }
+    elmIdx++;
   }
   delete [] elmVtx;
 }
@@ -779,37 +758,49 @@ void readMesh(const char* meshfilename,
     const char* solutionfilename,
     const char* connHeadfilename,
     MeshInfo& mesh) {
-  FILE* fc = fopen(coordfilename, "r");
+
+  int self = PCU_Comm_Self();
+
+  char filename[64];
+//  sprintf(filename, "coords.%d",self);
+  sprintf(filename, "%s.%d",coordfilename,self);
+    
+  FILE* fc = fopen(filename , "r");
   PCU_ALWAYS_ASSERT(fc);
   getNumVerts(fc,mesh.numVerts);
+  mesh.localNumVerts=mesh.numVerts;
+  mesh.numVerts=PCU_Add_Long(mesh.numVerts);
   
   if(!PCU_Comm_Self())
     fprintf(stderr, "numVerts %ld\n", mesh.numVerts);
-  readCoords(fc, mesh.numVerts, mesh.localNumVerts, &(mesh.coords));
+  readCoords(fc, mesh.localNumVerts, &(mesh.coords));
   fclose(fc);
  
   if(0==1) {
   FILE* fs = fopen(solutionfilename, "r");
   PCU_ALWAYS_ASSERT(fs);
-  readSolution(fs, mesh.numVerts, mesh.localNumVerts, &(mesh.solution));
+  readSolution(fs, mesh.localNumVerts, &(mesh.solution));
   fclose(fs);
   }
 
-  FILE* ff = fopen(classfilename, "r");
+  sprintf(filename, "%s.%d",classfilename,self);
+  FILE* ff = fopen(filename, "r");
   PCU_ALWAYS_ASSERT(ff);
-  readClassification(ff, mesh.numVerts, &(mesh.classification));
+  readClassification(ff, mesh.localNumVerts, &(mesh.classification));
   fclose(ff);
 
   //add an argument to readMesh for the fathers2D
-  FILE* fff = fopen(fathers2Dfilename, "r");
+  sprintf(filename, "%s.%d",fathers2Dfilename,self);
+  FILE* fff = fopen(filename, "r");
   PCU_ALWAYS_ASSERT(fff);
-  readClassification(fff, mesh.numVerts, &(mesh.fathers2D)); // note we re-use classification reader
+  readClassification(fff, mesh.localNumVerts, &(mesh.fathers2D)); // note we re-use classification reader
   fclose(fff);
 
   if( strcmp(matchfilename, "NULL") ) {
-    FILE* fm = fopen(matchfilename, "r");
+    sprintf(filename, "%s.%d",matchfilename,self);
+    FILE* fm = fopen(filename, "r");
     PCU_ALWAYS_ASSERT(fm);
-    readMatches(fm, mesh.numVerts, &(mesh.matches));
+    readMatches(fm, mesh.localNumVerts, &(mesh.matches));
     fclose(fm);
   }
 
@@ -817,7 +808,7 @@ void readMesh(const char* meshfilename,
   FILE* fh = fopen(connHeadfilename, "r");
   PCU_ALWAYS_ASSERT(f);
   PCU_ALWAYS_ASSERT(fh);
-  readElements(f,fh, mesh.dim, mesh.numElms, mesh.numVtxPerElm,
+  readElements(f,fh, mesh.dim,  mesh.numElms, mesh.numVtxPerElm,
       mesh.localNumElms, &(mesh.elements));
   mesh.elementType = getElmType(mesh.dim, mesh.numVtxPerElm);
   fclose(f);

From 9f98fe940ccb605bbe36a60dafa378eae9b99ab2 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Thu, 3 Jun 2021 05:16:18 -0600
Subject: [PATCH 346/555] broken, possibly  just do to parted file ingestion

---
 apf/apfConstruct.cc          | 12 ++++----
 apf/apfConvert.h             |  2 +-
 test/matchedNodeElmReader.cc | 60 +++++++++---------------------------
 3 files changed, 21 insertions(+), 53 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index f36791337..0a4fcd862 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -275,7 +275,7 @@ void setCoords(Mesh2* m, const double* coords, int nverts,
   delete [] c;
 }
 
-void setMatches(Mesh2* m, const int* matches, int nverts,
+void setMatches(Mesh2* m, const Gid* matches, int nverts,
     GlobalToVert& globalToVert)
 {
   PCU_Debug_Open();
@@ -308,7 +308,7 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
   while (nverts > 0) {
     PCU_COMM_PACK(to, start);
     PCU_COMM_PACK(to, n);
-    PCU_Comm_Pack(to, matches, n*sizeof(int));
+    PCU_Comm_Pack(to, matches, n*sizeof(Gid));
     PCU_Debug_Print("%d sending start %ld n %d to %d\n",
         self, start, n, to);
 
@@ -328,9 +328,9 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
   }
 
   for (int i = 0; i < mySize; ++i) {
-    int match = c[i];
+    Gid match = c[i];
     if( match != -1 )
-      PCU_Debug_Print("%d found match %d at gid %ld\n",
+      PCU_Debug_Print("%d found match %ld at gid %ld\n",
           self, match, i+myOffset);
   }
 
@@ -354,7 +354,7 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
     tmpParts.at(gid - myOffset).push_back(from);
   }
  
-  MeshTag* matchGidTag = m->createIntTag("matchGids", 1);
+  MeshTag* matchGidTag = m->createLongTag("matchGids", 1);
   /* Send the matches to everybody who wants them */
   PCU_Comm_Begin();
   for (int i = 0; i < mySize; ++i) {
@@ -393,7 +393,7 @@ void setMatches(Mesh2* m, const int* matches, int nverts,
    */
   PCU_Comm_Begin();
   APF_CONST_ITERATE(GlobalToVert, globalToVert, it) {
-    MeshEntity* e = it->second;
+    MeshEntity* e = it->second;  // KEJ does not follow this
     Gid gid = it->first;
     int tmpI=gid / quotient;
     int to = std::min(peers - 1,tmpI); 
diff --git a/apf/apfConvert.h b/apf/apfConvert.h
index 63ad80a38..b6b121db8 100644
--- a/apf/apfConvert.h
+++ b/apf/apfConvert.h
@@ -71,7 +71,7 @@ void setCoords(Mesh2* m, const double* coords, int nverts,
   * After this call, all vertices in the apf::Mesh2 object have correct
   * coordinates assigned.
   */
-void setMatches(Mesh2* m, const int* matches, int nverts,
+void setMatches(Mesh2* m, const Gid* matches, int nverts,
     GlobalToVert& globalToVert);
 
 
diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 51d152c38..b51232995 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -1,4 +1,3 @@
-//  char* filename;
 #include <gmi_mesh.h>
 #include <gmi_null.h>
 #include <apfMDS.h>
@@ -14,9 +13,6 @@
 #include <cassert>
 #include <algorithm>
 #include <apfBox.h>
-#include <sstream>
-#include <iostream>
-
 
 /* from https://github.com/SCOREC/core/issues/205
 0=fully interior of the volume
@@ -535,34 +531,7 @@ void setClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* t) {
   mesh->acceptChanges();
 }
 
-void getLocalRangeMT(apf::Gid total, apf::Gid& numWedge, apf::Gid& numTet, int& local,
-    apf::Gid& first, apf::Gid& last) {
-  const int self = PCU_Comm_Self();
-  const int peers = PCU_Comm_Peers();
-  apf::Gid Nverte=numWedge/2+numTet/6;
-  int nvertpp=Nverte/peers;
-  float xnwr=peers/(1.0+numTet/(3.0*numWedge));
-  int nAllWedge=xnwr;
-  int nWpp=numWedge/xnwr;
-  int localw= numWedge - nAllWedge*nWpp;
-  int ntetpartial=6*(nvertpp-localw/2);
-  if(self < nAllWedge) 
-     local=nWpp;
-  else if( self==nAllWedge)  // mixed
-      local=localw+ntetpartial;
-  else 
-     local=(numTet-ntetpartial)/(peers-nAllWedge-1);
-  apf::Gid lp=PCU_Exscan_Long(local)+local;
-  if( self == peers-1 ) {  //last rank
-    if( lp < total ){
-      apf::Gid lpd;
-      lpd= total - lp;
-      local += lpd;
-    }
-  }
-  first = PCU_Exscan_Long(local);
-  last = first+local;
-}
+
 
 void getLocalRange(apf::Gid total, int& local,
     apf::Gid& first, apf::Gid& last) {
@@ -684,9 +653,9 @@ void readSolution(FILE* f, int& localnumvtx, double** solution) {
   }
 }
 
-void readMatches(FILE* f, int localnumvtx, apf::Gid** matches) {
-  fprintf(stderr, "%d readMatches localnumvtx \n",
-      PCU_Comm_Self(), localnumvtx);
+void readMatches(FILE* f, apf::Gid numvtx, int localnumvtx, apf::Gid** matches) {
+  fprintf(stderr, "%d readMatches numvtx %ld localnumvtx %d \n",
+      PCU_Comm_Self(), numvtx, localnumvtx);
   *matches = new apf::Gid[localnumvtx];
   rewind(f);
   int vidx = 0;
@@ -708,21 +677,19 @@ void readElements(FILE* f, FILE* fh, unsigned &dim,  apf::Gid& numElms,
   rewind(f);
   rewind(fh);
   int dimHeader[2];
-  apf::Gid numElms;  // not used but save anyway
   gmi_fscanf(fh, 2, "%u %u", dimHeader, dimHeader+1);
 //  assert( dimHeader[0] == 1 && dimHeader[1] == 1);
   gmi_fscanf(fh, 1, "%u", &dim);
   gmi_fscanf(fh, 2, "%ld %u", &numElms, &numVtxPerElm);
-  int self = PCU_Comm_Self();
-  const int peers = PCU_Comm_Peers();
-  for (int j=0; j<= self,j++)
+  int self = PCU_Comm_Self();;
+  for (int j=0; j< self+1;j++)
      gmi_fscanf(fh, 2, "%d %u", &localNumElms, &numVtxPerElm);
   *elements = new apf::Gid[localNumElms*numVtxPerElm];
   int i;
   unsigned j;
   unsigned elmIdx = 0;
   apf::Gid* elmVtx = new apf::Gid[numVtxPerElm];
-  for (i = 0; i < numElms; i++) {
+  for (i = 0; i < localNumElms; i++) {
     for (j = 0; j < numVtxPerElm; j++)
       gmi_fscanf(f, 1, "%ld", elmVtx+j);
     for (j = 0; j < numVtxPerElm; j++) {
@@ -738,7 +705,7 @@ struct MeshInfo {
   double* coords;
   double* solution;
   apf::Gid* elements;
-  int* matches;
+  apf::Gid* matches;
   int* classification;
   int* fathers2D;
   unsigned dim;
@@ -762,7 +729,6 @@ void readMesh(const char* meshfilename,
   int self = PCU_Comm_Self();
 
   char filename[64];
-//  sprintf(filename, "coords.%d",self);
   sprintf(filename, "%s.%d",coordfilename,self);
     
   FILE* fc = fopen(filename , "r");
@@ -777,7 +743,8 @@ void readMesh(const char* meshfilename,
   fclose(fc);
  
   if(0==1) {
-  FILE* fs = fopen(solutionfilename, "r");
+  sprintf(filename, "%s.%d",solutionfilename,self);
+  FILE* fs = fopen(filename, "r");
   PCU_ALWAYS_ASSERT(fs);
   readSolution(fs, mesh.localNumVerts, &(mesh.solution));
   fclose(fs);
@@ -800,15 +767,16 @@ void readMesh(const char* meshfilename,
     sprintf(filename, "%s.%d",matchfilename,self);
     FILE* fm = fopen(filename, "r");
     PCU_ALWAYS_ASSERT(fm);
-    readMatches(fm, mesh.localNumVerts, &(mesh.matches));
+    readMatches(fm, mesh.numVerts, mesh.localNumVerts, &(mesh.matches));
     fclose(fm);
   }
 
-  FILE* f = fopen(meshfilename, "r");
+  sprintf(filename, "%s.%d",meshfilename,self);
+  FILE* f = fopen(filename, "r");
   FILE* fh = fopen(connHeadfilename, "r");
   PCU_ALWAYS_ASSERT(f);
   PCU_ALWAYS_ASSERT(fh);
-  readElements(f,fh, mesh.dim,  mesh.numElms, mesh.numVtxPerElm,
+  readElements(f,fh, mesh.dim, mesh.numElms, mesh.numVtxPerElm,
       mesh.localNumElms, &(mesh.elements));
   mesh.elementType = getElmType(mesh.dim, mesh.numVtxPerElm);
   fclose(f);

From cc74af6aada4f6f80dea330d99c748792f486328 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Thu, 3 Jun 2021 07:57:21 -0600
Subject: [PATCH 347/555] more int->Gid issues and also some bugs on file
 ingestion but passes small viz node case that does not test int overflow

---
 apf/apfConstruct.cc          |  4 +--
 apf/apfConvertTags.h         | 25 ++++++++++------
 test/matchedNodeElmReader.cc | 58 ++++++++++++------------------------
 3 files changed, 37 insertions(+), 50 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index 0a4fcd862..e8507a39f 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -292,7 +292,7 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts,
 
   /* Force each peer to have exactly mySize verts.
      This means we might need to send and recv some matches */
-  int* c = new int[mySize];
+  Gid* c = new Gid[mySize];
   PCU_Debug_Print("%d mysize %d\n", self, mySize);
 
   Gid start = PCU_Exscan_Long(nverts);
@@ -322,7 +322,7 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts,
   while (PCU_Comm_Receive()) {
     PCU_COMM_UNPACK(start);
     PCU_COMM_UNPACK(n); /// in-place 64
-    PCU_Comm_Unpack(&c[(start - myOffset)], n*sizeof(int));
+    PCU_Comm_Unpack(&c[(start - myOffset)], n*sizeof(Gid));
     PCU_Debug_Print("%d receiving start %ld n %d from %d\n",
         self, start, n, PCU_Comm_Sender());
   }
diff --git a/apf/apfConvertTags.h b/apf/apfConvertTags.h
index 375bdef85..515e7b6b4 100644
--- a/apf/apfConvertTags.h
+++ b/apf/apfConvertTags.h
@@ -70,17 +70,21 @@ apf::MeshTag* setMappedTag(Mesh2* m, const char* tagName,
   int self = PCU_Comm_Self();
   if (self == (peers - 1))
     mySize += remainder;
-  int myOffset = self * quotient;
+  apf::Gid myOffset = self * quotient;
 
   /* Force each peer to have exactly mySize verts.
      This means we might need to send and recv some coords */
   T* c = new T[mySize*entries];
 
-  int start = PCU_Exscan_Int(nverts);
+  apf::Gid start = PCU_Exscan_Int(nverts);
 
   PCU_Comm_Begin();
-  int to = std::min(peers - 1, start / quotient);
-  int n = std::min((to+1)*quotient-start, nverts);
+  apf::Gid tmpL=start / quotient;
+  int tmpI=tmpL;
+  int to = std::min(peers - 1, tmpI);
+  tmpL=(to+1)*quotient-start;
+  tmpI=tmpL;
+  int n = std::min(tmpI, nverts);
   while (nverts > 0) {
     PCU_COMM_PACK(to, start);
     PCU_COMM_PACK(to, n);
@@ -104,13 +108,16 @@ apf::MeshTag* setMappedTag(Mesh2* m, const char* tagName,
   TmpParts tmpParts(mySize);
   PCU_Comm_Begin();
   APF_CONST_ITERATE(GlobalToVert, globalToVert, it) {
-    int gid = it->first;
-    int to = std::min(peers - 1, gid / quotient);
+    apf::Gid gid = it->first;
+    tmpL=gid / quotient;
+    tmpI=tmpL;
+    int to = std::min(peers - 1, tmpI);
+// replaced     int to = std::min(peers - 1, gid / quotient);
     PCU_COMM_PACK(to, gid);
   }
   PCU_Comm_Send();
   while (PCU_Comm_Receive()) {
-    int gid;
+    apf::Gid gid;
     PCU_COMM_UNPACK(gid);
     int from = PCU_Comm_Sender();
     tmpParts.at(gid - myOffset).push_back(from);
@@ -122,7 +129,7 @@ apf::MeshTag* setMappedTag(Mesh2* m, const char* tagName,
     std::vector<int>& parts = tmpParts[i];
     for (size_t j = 0; j < parts.size(); ++j) {
       int to = parts[j];
-      int gid = i + myOffset;
+      apf::Gid gid = i + myOffset;
       PCU_COMM_PACK(to, gid);
       PCU_Comm_Pack(to, &c[i*entries], entries*sizeof(T));
     }
@@ -131,7 +138,7 @@ apf::MeshTag* setMappedTag(Mesh2* m, const char* tagName,
   apf::MeshTag* t = createTag<T>(m,tagName,entries);
   T* v = new T[entries];
   while (PCU_Comm_Receive()) {
-    int gid;
+    apf::Gid gid;
     PCU_COMM_UNPACK(gid);
     PCU_Comm_Unpack(v, entries*sizeof(T));
     setEntTag(m,t,globalToVert[gid],v);
diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index b51232995..1356c7ddc 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -592,64 +592,37 @@ bool skipLine(char* line) {
 void getNumVerts(FILE* f, apf::Gid& verts) {
   rewind(f);
   gmi_fscanf(f, 1, "%ld",  &verts);
-//  verts = 0;
-//  size_t linelimit = 1024;
-//  char* line = new char[linelimit];
-//  while( gmi_getline(&line,&linelimit,f) != -1 ) {
-//    if( ! skipLine(line) )
-//      verts++;
-//  }
-//  delete [] line;
 }
 
-void readClassification(FILE* f, apf::Gid numVtx, int** classification) {
-  apf::Gid firstVtx, lastVtx;
-  int localNumVtx;
-  getLocalRange(numVtx,localNumVtx,firstVtx,lastVtx);
+void readClassification(FILE* f, int localNumVtx, int** classification) {
   *classification = new int[localNumVtx];
   rewind(f);
-  int vidx = 0;
-  for(apf::Gid i=0; i<numVtx; i++) {
-//    int id;
-    int mdlId;
-//    gmi_fscanf(f, 2, "%d %d", &id, &mdlId);
+  int mdlId;
+  for(int i=0; i<localNumVtx; i++) {
     gmi_fscanf(f, 1, "%d",  &mdlId);
-    //std::cout<<"read id is "<<id<<std::endl;
-    //std::cout<<"read model id is "<<mdlId<<std::endl;
-    if( i >= firstVtx && i < lastVtx ) {
-      (*classification)[vidx] = mdlId;
-      vidx++;
-    }
+    (*classification)[i] = mdlId;
   }
-  /*std::cout<<"numVtx is "<<numVtx<<std::endl;
-  for (int i=0;i<numVtx;i++) {
-      std::cout<<"vtx num "<<i<<" has class "<<(*classification)[i]<<std::endl;
-  }*/
 }
 
 void readCoords(FILE* f, int& localnumvtx, double** coordinates) {
   *coordinates = new double[localnumvtx*3];
-  int vidx = 0;
   for(int i=0; i<localnumvtx; i++) {
     double pos[3];
     gmi_fscanf(f, 3, "%lf %lf %lf", pos+0, pos+1, pos+2);
     for(unsigned j=0; j<3; j++)
-        (*coordinates)[vidx*3+j] = pos[j];
-    vidx++;
+        (*coordinates)[i*3+j] = pos[j];
   }
 }
 
 void readSolution(FILE* f, int& localnumvtx, double** solution) {
   *solution = new double[localnumvtx*5];
   rewind(f);
-  int vidx = 0;
   for(int i=0; i<localnumvtx; i++) {
     double pos[5];
     pos[4]=0; //temperature
     gmi_fscanf(f, 4, "%lf %lf %lf %lf", pos+0, pos+1, pos+2, pos+3);
     for(unsigned j=0; j<5; j++)
-      (*solution)[vidx*5+j] = pos[j];
-    vidx++;
+      (*solution)[i*5+j] = pos[j];
   }
 }
 
@@ -658,18 +631,25 @@ void readMatches(FILE* f, apf::Gid numvtx, int localnumvtx, apf::Gid** matches)
       PCU_Comm_Self(), numvtx, localnumvtx);
   *matches = new apf::Gid[localnumvtx];
   rewind(f);
-  int vidx = 0;
   apf::Gid matchedVtx;
-  apf::Gid i = 0;
-  while( 1 == fscanf(f, "%ld", &matchedVtx) ) {
+  for(int i=0; i<localnumvtx; i++) {
+    gmi_fscanf(f, 1, "%ld",  &matchedVtx);
     PCU_ALWAYS_ASSERT( matchedVtx == -1 ||
        ( matchedVtx >= 1 && matchedVtx <= numvtx ));
     if( matchedVtx != -1 )
         --matchedVtx;
-    (*matches)[vidx] = matchedVtx;
-    vidx++;
-    i++;
+    (*matches)[i] = matchedVtx;
   }
+// I think the above will perform better than the code commented out below
+//  int vidx = 0;
+//  while( 1 == fscanf(f, "%ld", &matchedVtx) ) {
+//    PCU_ALWAYS_ASSERT( matchedVtx == -1 ||
+//       ( matchedVtx >= 1 && matchedVtx <= numvtx ));
+//    if( matchedVtx != -1 )
+//        --matchedVtx;
+//    (*matches)[vidx] = matchedVtx;
+//    vidx++;
+//  }
 }
 
 void readElements(FILE* f, FILE* fh, unsigned &dim,  apf::Gid& numElms,

From 20cdfc0f4c64d6aee0b9af1a45478c2b7f53b4a5 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Thu, 3 Jun 2021 14:38:31 -0700
Subject: [PATCH 348/555] fixes Jacob caught

---
 apf/apfConvertTags.h | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/apf/apfConvertTags.h b/apf/apfConvertTags.h
index 515e7b6b4..44da7db60 100644
--- a/apf/apfConvertTags.h
+++ b/apf/apfConvertTags.h
@@ -11,13 +11,13 @@ namespace {
     apf::Gid max = -1;
     APF_CONST_ITERATE(apf::GlobalToVert, globalToVert, it)
       max = std::max(max, it->first);
-    return PCU_Max_Int(max); // this is type-dependent
+    return PCU_Max_Long(max); // this is type-dependent
   }
 
   template <class T> inline
   apf::MeshTag* createTag(apf::Mesh*,
       const char*, const int) {
-//    exit(EXIT_FAILURE);
+    exit(EXIT_FAILURE);
     return 0;
   }
   
@@ -27,12 +27,23 @@ namespace {
     return m->createIntTag(name,entries);
   }
   
+  template <> inline
+  apf::MeshTag* createTag<long>(apf::Mesh* m,
+      const char* name, const long entries) {
+    return m->createLongTag(name,entries);
+  }
+  
   template <> inline
   apf::MeshTag* createTag<double>(apf::Mesh* m,
       const char* name, const int entries) {
     return m->createDoubleTag(name,entries);
   }
   
+  inline void setEntTag(apf::Mesh* m, apf::MeshTag* t,
+      apf::MeshEntity* e, long* vals) {
+    return m->setLongTag(e,t,vals);
+  }
+  
   inline void setEntTag(apf::Mesh* m, apf::MeshTag* t,
       apf::MeshEntity* e, int* vals) {
     return m->setIntTag(e,t,vals);
@@ -76,7 +87,7 @@ apf::MeshTag* setMappedTag(Mesh2* m, const char* tagName,
      This means we might need to send and recv some coords */
   T* c = new T[mySize*entries];
 
-  apf::Gid start = PCU_Exscan_Int(nverts);
+  apf::Gid start = PCU_Exscan_Long(nverts);
 
   PCU_Comm_Begin();
   apf::Gid tmpL=start / quotient;

From bb9c085b866f90554f14b0e774c55b6b9035b3dc Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Thu, 3 Jun 2021 14:49:40 -0700
Subject: [PATCH 349/555] rollback long tags as I am out of skill

---
 apf/apfConvertTags.h | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/apf/apfConvertTags.h b/apf/apfConvertTags.h
index 44da7db60..181999188 100644
--- a/apf/apfConvertTags.h
+++ b/apf/apfConvertTags.h
@@ -27,11 +27,11 @@ namespace {
     return m->createIntTag(name,entries);
   }
   
-  template <> inline
-  apf::MeshTag* createTag<long>(apf::Mesh* m,
-      const char* name, const long entries) {
-    return m->createLongTag(name,entries);
-  }
+//  template <> inline
+//  apf::MeshTag* createTag<long>(apf::Mesh* m,
+//      const char* name, const long entries) {
+//    return m->createLongTag(name,entries);
+//  }
   
   template <> inline
   apf::MeshTag* createTag<double>(apf::Mesh* m,
@@ -39,10 +39,10 @@ namespace {
     return m->createDoubleTag(name,entries);
   }
   
-  inline void setEntTag(apf::Mesh* m, apf::MeshTag* t,
-      apf::MeshEntity* e, long* vals) {
-    return m->setLongTag(e,t,vals);
-  }
+//  inline void setEntTag(apf::Mesh* m, apf::MeshTag* t,
+//      apf::MeshEntity* e, long* vals) {
+//    return m->setLongTag(e,t,vals);
+//  }
   
   inline void setEntTag(apf::Mesh* m, apf::MeshTag* t,
       apf::MeshEntity* e, int* vals) {

From 91685fa51d22131112b55b160b2634436681a919 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Thu, 3 Jun 2021 19:10:19 -0700
Subject: [PATCH 350/555] crazy print around max computation

---
 apf/apfConstruct.cc | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index e8507a39f..5ac3507b9 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -67,7 +67,24 @@ static Gid getMax(const GlobalToVert& globalToVert)
    I didn't think to use it here, so credit is given. */
 static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
 {
+  Gid ifirst=0;
+  int self2 = PCU_Comm_Self();
+  APF_ITERATE(GlobalToVert, globalToVert, it) {
+    Gid gid = it->first;  
+    if(ifirst==0 || ifirst==13437400 ) {
+        lion_eprint(1, "constructResidence: self=%d,gid=%ld,ifirst=%ld  \n",self2,gid,ifirst);
+    }
+    ifirst++;
+  }
   Gid max = getMax(globalToVert);  // seems like we read this and know it already on every rank so why compute with global comm?
+  ifirst=0;
+  APF_ITERATE(GlobalToVert, globalToVert, it) {
+    Gid gid = it->first;  
+    if(ifirst==0 || ifirst==13437400 ) {
+        lion_eprint(1, "constructResidence: self=%d,gid=%ld,ifirst=%ld,max=%ld  \n",self2,gid,ifirst,max);
+    }
+    ifirst++;
+  }
   Gid total = max + 1;
   int peers = PCU_Comm_Peers();
   int quotient = total / peers; // this seems to work as C++ is doing the math in 64 bit and the result is assumed not to overflow 32

From ba64b1d1574f43c77cb1df4c6aa99540c51d59f6 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Thu, 3 Jun 2021 23:27:32 -0700
Subject: [PATCH 351/555]  digging

---
 apf/apfConstruct.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index 5ac3507b9..7c4352809 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -76,7 +76,9 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
     }
     ifirst++;
   }
+  PCU_Barrier();
   Gid max = getMax(globalToVert);  // seems like we read this and know it already on every rank so why compute with global comm?
+  PCU_Barrier();
   ifirst=0;
   APF_ITERATE(GlobalToVert, globalToVert, it) {
     Gid gid = it->first;  

From 81fe4b9cc8713722c8878c3d866231c1fe2086bd Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Sat, 5 Jun 2021 07:14:44 -0700
Subject: [PATCH 352/555] New try at long tags  uncommented and changes entries
 back to an int which compiles

---
 apf/apfConvertTags.h | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/apf/apfConvertTags.h b/apf/apfConvertTags.h
index 181999188..020916fa5 100644
--- a/apf/apfConvertTags.h
+++ b/apf/apfConvertTags.h
@@ -27,11 +27,11 @@ namespace {
     return m->createIntTag(name,entries);
   }
   
-//  template <> inline
-//  apf::MeshTag* createTag<long>(apf::Mesh* m,
-//      const char* name, const long entries) {
-//    return m->createLongTag(name,entries);
-//  }
+  template <> inline
+  apf::MeshTag* createTag<long>(apf::Mesh* m,
+      const char* name, const int entries) {
+    return m->createLongTag(name,entries);
+  }
   
   template <> inline
   apf::MeshTag* createTag<double>(apf::Mesh* m,
@@ -39,10 +39,10 @@ namespace {
     return m->createDoubleTag(name,entries);
   }
   
-//  inline void setEntTag(apf::Mesh* m, apf::MeshTag* t,
-//      apf::MeshEntity* e, long* vals) {
-//    return m->setLongTag(e,t,vals);
-//  }
+  inline void setEntTag(apf::Mesh* m, apf::MeshTag* t,
+      apf::MeshEntity* e, long* vals) {
+    return m->setLongTag(e,t,vals);
+  }
   
   inline void setEntTag(apf::Mesh* m, apf::MeshTag* t,
       apf::MeshEntity* e, int* vals) {

From f6cc76163785c56d4e358a1be1f72df4fdb77ded Mon Sep 17 00:00:00 2001
From: Sebastian Wolf <wolf.sebastian@in.tum.de>
Date: Mon, 7 Jun 2021 11:57:30 +0200
Subject: [PATCH 353/555] Fixes for compilation with gcc 11.1.1

---
 ma/maStats.cc    | 2 +-
 phasta/phIO.c    | 2 +-
 pumi/pumi_sys.cc | 2 +-
 test/describe.cc | 4 ++--
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/ma/maStats.cc b/ma/maStats.cc
index eb27b7add..479dcce21 100644
--- a/ma/maStats.cc
+++ b/ma/maStats.cc
@@ -51,7 +51,7 @@ void getLinearQualitiesInPhysicalSpace(ma::Mesh* m,
   ma::Iterator* it;
   it = m->begin(m->getDimension());
   while( (e = m->iterate(it)) ) {
-    double lq;
+    double lq = 0;
     if (m->getType(e) == apf::Mesh::TRIANGLE) {
       ma::Vector p[3];
       ma::getVertPoints(m, e, p);
diff --git a/phasta/phIO.c b/phasta/phIO.c
index 4352dadbb..11e31f4c5 100644
--- a/phasta/phIO.c
+++ b/phasta/phIO.c
@@ -81,7 +81,7 @@ static int find_header(FILE* f, const char* name, char* found, char header[PH_LI
     tmp[PH_LINE-1] = '\0';
     parse_header(tmp, &hname, &bytes, 0, NULL);
     if (!strncmp(name, hname, strlen(name))) {
-      strncpy(found, hname, strlen(hname));
+      strncpy(found, hname, strlen(found));
       found[strlen(hname)] = '\0';
       return 1;
     }
diff --git a/pumi/pumi_sys.cc b/pumi/pumi_sys.cc
index 6b620a151..37a1145bf 100644
--- a/pumi/pumi_sys.cc
+++ b/pumi/pumi_sys.cc
@@ -103,7 +103,7 @@ double pumi_getMem()
   Kernel_GetMemorySize(KERNEL_MEMSIZE_HEAP, &heap);
   return (double)heap/M;
 #else
-  struct mallinfo meminfo_now = mallinfo();
+  struct mallinfo2 meminfo_now = mallinfo2();
   return ((double)meminfo_now.arena)/M;
 #endif
 }
diff --git a/test/describe.cc b/test/describe.cc
index 73e79a56b..b4eeae055 100644
--- a/test/describe.cc
+++ b/test/describe.cc
@@ -32,7 +32,7 @@ static double get_peak()
 
 static double get_peak()
 {
-  return mallinfo().arena;
+  return mallinfo2().arena;
 }
 
 #else
@@ -62,7 +62,7 @@ static void print_stats(const char* name, double value)
 
 static double get_chunks()
 {
-  struct mallinfo m = mallinfo();
+  struct mallinfo2 m = mallinfo2();
   return m.uordblks + m.hblkhd;
 }
 

From 4c5455ef64d420d634054656fb97603223473b74 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Tue, 8 Jun 2021 13:03:40 -0400
Subject: [PATCH 354/555] use mallinfo2 when available

---
 CMakeLists.txt   | 13 +++++++++++++
 pumi/pumi_sys.cc |  5 ++++-
 test/describe.cc |  8 ++++++++
 3 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 630db24fc..3c3d9a3d5 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -148,6 +148,19 @@ add_library(core INTERFACE)
 target_link_libraries(core INTERFACE ${SCOREC_EXPORTED_TARGETS})
 scorec_export_library(core)
 
+#check for mallinfo2
+if(${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU")
+  include(CheckCXXSymbolExists)
+  check_cxx_symbol_exists(mallinfo2 "malloc" HAS_MALLINFO2)
+  message(STATUS "mallinfo2: ${PUMI_HAS_MALLINFO2}")
+  if(PUMI_HAS_MALLINFO2)
+    message(STATUS "has mallinfo2")
+    target_compile_definitions(pumi PUBLIC -DPUMI_HAS_MALLINFO2)
+  else()
+    message(STATUS "does not have mallinfo2")
+  endif()
+endif()
+
 if(BUILD_EXES)
   add_subdirectory(test)
 endif()
diff --git a/pumi/pumi_sys.cc b/pumi/pumi_sys.cc
index 37a1145bf..897e8dcde 100644
--- a/pumi/pumi_sys.cc
+++ b/pumi/pumi_sys.cc
@@ -102,9 +102,12 @@ double pumi_getMem()
   size_t heap;
   Kernel_GetMemorySize(KERNEL_MEMSIZE_HEAP, &heap);
   return (double)heap/M;
-#else
+#elif defined(__GNUG__) && defined(PUMI_HAS_MALLINFO2)
   struct mallinfo2 meminfo_now = mallinfo2();
   return ((double)meminfo_now.arena)/M;
+#elif defined(__GNUG__)
+  struct mallinfo meminfo_now = mallinfo();
+  return ((double)meminfo_now.arena)/M;
 #endif
 }
 
diff --git a/test/describe.cc b/test/describe.cc
index b4eeae055..bc2a8684c 100644
--- a/test/describe.cc
+++ b/test/describe.cc
@@ -32,7 +32,11 @@ static double get_peak()
 
 static double get_peak()
 {
+#if defined(__GNUG__) && defined(PUMI_HAS_MALLINFO2)
   return mallinfo2().arena;
+#elif defined(__GNUG__)
+  return mallinfo().arena;
+#endif
 }
 
 #else
@@ -62,7 +66,11 @@ static void print_stats(const char* name, double value)
 
 static double get_chunks()
 {
+#if defined(__GNUG__) && defined(PUMI_HAS_MALLINFO2)
   struct mallinfo2 m = mallinfo2();
+#elif defined(__GNUG__)
+  struct mallinfo m = mallinfo();
+#endif
   return m.uordblks + m.hblkhd;
 }
 

From f1d533da026e644ccd4326870b73c4a4ccf0bed6 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Tue, 8 Jun 2021 14:03:17 -0400
Subject: [PATCH 355/555] fix symbol detection logic

---
 CMakeLists.txt | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3c3d9a3d5..eed1d6a32 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -151,13 +151,9 @@ scorec_export_library(core)
 #check for mallinfo2
 if(${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU")
   include(CheckCXXSymbolExists)
-  check_cxx_symbol_exists(mallinfo2 "malloc" HAS_MALLINFO2)
-  message(STATUS "mallinfo2: ${PUMI_HAS_MALLINFO2}")
+  check_cxx_symbol_exists(mallinfo2 "malloc.h" PUMI_HAS_MALLINFO2)
   if(PUMI_HAS_MALLINFO2)
-    message(STATUS "has mallinfo2")
-    target_compile_definitions(pumi PUBLIC -DPUMI_HAS_MALLINFO2)
-  else()
-    message(STATUS "does not have mallinfo2")
+    target_compile_definitions(core INTERFACE -DPUMI_HAS_MALLINFO2)
   endif()
 endif()
 

From 993ee2f445bdf778c85de8dfeada256ceafdf013 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Sun, 20 Jun 2021 07:30:27 -0400
Subject: [PATCH 356/555] simmodsuite max version to 15.0-210501

---
 cmake/FindSimModSuite.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/FindSimModSuite.cmake b/cmake/FindSimModSuite.cmake
index 64ad49986..244ea4ca1 100644
--- a/cmake/FindSimModSuite.cmake
+++ b/cmake/FindSimModSuite.cmake
@@ -84,7 +84,7 @@ string(REGEX REPLACE
   "${SIM_VERSION}")
 
 set(MIN_VALID_SIM_VERSION 12.0.190225)
-set(MAX_VALID_SIM_VERSION 15.0.200714)
+set(MAX_VALID_SIM_VERSION 15.0.210501)
 if( ${SKIP_SIMMETRIX_VERSION_CHECK} )
   message(STATUS "Skipping Simmetrix SimModSuite version check."
     " This may result in undefined behavior")

From b8b3681d3f5ebf6499f33b82224f513ff84efadd Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Mon, 21 Jun 2021 20:29:37 -0400
Subject: [PATCH 357/555] found was not null terminated, use strcpy

---
 phasta/phIO.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/phasta/phIO.c b/phasta/phIO.c
index 11e31f4c5..2806dd783 100644
--- a/phasta/phIO.c
+++ b/phasta/phIO.c
@@ -7,6 +7,7 @@
 #include <PCU.h>
 #include <lionPrint.h>
 #include <phiotimer.h>
+#include <assert.h>
 
 #define PH_LINE 1024
 #define MAGIC 362436
@@ -81,8 +82,8 @@ static int find_header(FILE* f, const char* name, char* found, char header[PH_LI
     tmp[PH_LINE-1] = '\0';
     parse_header(tmp, &hname, &bytes, 0, NULL);
     if (!strncmp(name, hname, strlen(name))) {
-      strncpy(found, hname, strlen(found));
-      found[strlen(hname)] = '\0';
+      assert(strlen(hname) < PH_LINE);
+      strcpy(found, hname);
       return 1;
     }
     fseek(f, bytes, SEEK_CUR);

From 4dd330e960b1921ae0d8d4039b8de8680a20d993 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 25 Jun 2021 08:58:47 -0400
Subject: [PATCH 358/555] pumi version 2.2.6

see issue #341
---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index eed1d6a32..ade7eb39f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -6,7 +6,7 @@ endif()
 # This is the top level CMake file for the SCOREC build
 cmake_minimum_required(VERSION 3.0)
 
-project(SCOREC VERSION 2.2.5 LANGUAGES CXX C)
+project(SCOREC VERSION 2.2.6 LANGUAGES CXX C)
 
 include(cmake/bob.cmake)
 include(cmake/xsdk.cmake)

From 4628e9130f2e3f372230a7e9f0a3c165405d159f Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Tue, 29 Jun 2021 08:39:55 -0400
Subject: [PATCH 359/555] parma: add imbalance definition

---
 parma/parma.h | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/parma/parma.h b/parma/parma.h
index c6a3bf284..065477fa7 100644
--- a/parma/parma.h
+++ b/parma/parma.h
@@ -16,6 +16,12 @@
 
 /**
  * @brief get entity imbalance
+ * @remark The imbalance of a given entity order (i.e., vtx, edge, face, rgn) 
+ * is defined as the maximum count of that entity order on a part, across all parts, 
+ * divided by the average entity count across all parts.
+ * For example if there are four parts and the parts have 5, 7, 12, and 8
+ * vertices, respectively, then the vertex imbalance is 50%;
+ * 12 / ((5+7+8+12)/4) = 1.5.
  * @param mesh (InOut) partitioned mesh
  * @param entImb (InOut) entity imbalance [vtx, edge, face, rgn]
  */
@@ -23,9 +29,12 @@ void Parma_GetEntImbalance(apf::Mesh* mesh, double (*entImb)[4]);
 
 /**
  * @brief see Parma_GetEntImbalance(...)
- * @remark On a part, if an entity order (vtx, edge, face, rgn) does not have
+ * @remark The weighted imbalance definition replaces the entity count with the 
+ * sum of entity weights.  If the weight for all the entities of a given order
+ * is one, then the two definitions are equivalent.
+ * On a part, if an entity order (vtx, edge, face, rgn) does not have
  * weights set on all its entities then a weight of one will be assigned to each
- * of the entities (of the given order)
+ * of the entities (of the given order).
  * @param mesh (InOut) partitioned mesh
  * @param weight (In) entity weights used for computing imbalance
  * @param entImb (InOut) entity imbalance [vtx, edge, face, rgn]

From 750c0212f432ac6b20f3ef919f32bf43739355d6 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 29 Jun 2021 16:26:29 -0400
Subject: [PATCH 360/555] Adds logic to take care of load-balancing options

When all the options are set to false for Pre, Mid, and Post balance,
we check the imbalance and if it is bigger than in->maximumImbalance, then
we choose the best option based on the following criteria

a) if compiled with zoltan and #peers < 16K, use graph Zoltan
b) if compiled with zoltan and #peers > 16K, use RIB Zoltan
c) if compiled without zoltan,  use parma
---
 CMakeLists.txt  |   4 ++
 ma/maBalance.cc | 121 ++++++++++++++++++++++++++++++++++++++++++++----
 ma/maInput.cc   |  12 +++++
 3 files changed, 128 insertions(+), 9 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index ade7eb39f..86359c804 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -120,6 +120,10 @@ if(ENABLE_OMEGA_H)
   bob_public_dep(Omega_h)
 endif()
 
+if(ENABLE_ZOLTAN)
+  add_definitions(-DHAVE_ZOLTAN)
+endif()
+
 # Include the SCOREC project packages
 add_subdirectory(lion)
 add_subdirectory(pcu)
diff --git a/ma/maBalance.cc b/ma/maBalance.cc
index ebc845f28..32cb4967b 100644
--- a/ma/maBalance.cc
+++ b/ma/maBalance.cc
@@ -121,17 +121,58 @@ void printEntityImbalance(Mesh* m)
   print("element imbalance %.0f%% of average",p);
 }
 
+double estimateWeightedImbalance(Adapt* a)
+{
+  Tag* w = getElementWeights(a);
+  double imb[4];
+  Parma_GetWeightedEntImbalance(a->mesh, w, &imb);
+  removeTagFromDimension(a->mesh, w, a->mesh->getDimension());
+  a->mesh->destroyTag(w);
+  return imb[a->mesh->getDimension()];
+}
+
 void preBalance(Adapt* a)
 {
   if (PCU_Comm_Peers()==1)
     return;
   Input* in = a->input;
-  if (in->shouldRunPreZoltan)
+  // First take care of user overrides. That is, if any of the three options
+  // is true, apply that balancer and return.
+  if (in->shouldRunPreZoltan) {
     runZoltan(a);
-  if (in->shouldRunPreZoltanRib)
+    return;
+  }
+  if (in->shouldRunPreZoltanRib) {
     runZoltan(a,apf::RIB);
-  if (in->shouldRunPreParma)
+    return;
+  }
+  if (in->shouldRunPreParma) {
     runParma(a);
+    return;
+  }
+
+  // Then, take care of the case where all the options are set to false.
+  // That is, if the default values have not changed by the user. In
+  // this case, we apply the best possible balancer, if weighted imbalance
+  // is bigger than in->maximumImbalance
+  if ((!in->shouldRunPreZoltan) &&
+      (!in->shouldRunPreZoltanRib) &&
+      (!in->shouldRunPreParma) &&
+      (estimateWeightedImbalance(a) > in->maximumImbalance)) {
+#ifdef HAVE_ZOLTAN
+    if (PCU_Comm_Peers() < 16000) {
+      runZoltan(a);
+      return;
+    }
+    else {
+      runZoltan(a, apf::RIB);
+      return;
+    }
+#else
+    runParma(a);
+    return;
+#endif
+  }
 }
 
 void midBalance(Adapt* a)
@@ -139,10 +180,37 @@ void midBalance(Adapt* a)
   if (PCU_Comm_Peers()==1)
     return;
   Input* in = a->input;
-  if (in->shouldRunMidZoltan)
+  // First take care of user overrides. That is, if any of the three options
+  // is true, apply that balancer and return.
+  if (in->shouldRunMidZoltan) {
     runZoltan(a);
-  if (in->shouldRunMidParma)
+    return;
+  }
+  if (in->shouldRunMidParma) {
     runParma(a);
+    return;
+  }
+  // Then, take care of the case where all the options are set to false.
+  // That is, if the default values have not changed by the user. In
+  // this case, we apply the best possible balancer, if weighted imbalance
+  // is bigger than in->maximumImbalance
+  if ((!in->shouldRunMidZoltan) &&
+      (!in->shouldRunMidParma) &&
+      (estimateWeightedImbalance(a) > in->maximumImbalance)) {
+#ifdef HAVE_ZOLTAN
+    if (PCU_Comm_Peers() < 16000) {
+      runZoltan(a);
+      return;
+    }
+    else {
+      runZoltan(a, apf::RIB);
+      return;
+    }
+#else
+    runParma(a);
+    return;
+#endif
+  }
 }
 
 void postBalance(Adapt* a)
@@ -150,13 +218,48 @@ void postBalance(Adapt* a)
   if (PCU_Comm_Peers()==1)
     return;
   Input* in = a->input;
-  if (in->shouldRunPostZoltan)
+  // First take care of user overrides. That is, if any of the three options
+  // is true, apply that balancer and return.
+  if (in->shouldRunPostZoltan) {
     runZoltan(a);
-  if (in->shouldRunPostZoltanRib)
+    printEntityImbalance(a->mesh);
+    return;
+  }
+  if (in->shouldRunPostZoltanRib) {
     runZoltan(a,apf::RIB);
-  if (in->shouldRunPostParma)
+    printEntityImbalance(a->mesh);
+    return;
+  }
+  if (in->shouldRunPostParma) {
     runParma(a);
-  printEntityImbalance(a->mesh);
+    printEntityImbalance(a->mesh);
+    return;
+  }
+  // Then, take care of the case where all the options are set to false.
+  // That is, if the default values have not changed by the user. In
+  // this case, we apply the best possible balancer, if weighted imbalance
+  // is bigger than in->maximumImbalance
+  if ((!in->shouldRunPostZoltan) &&
+      (!in->shouldRunPostZoltanRib) &&
+      (!in->shouldRunPostParma) &&
+      (estimateWeightedImbalance(a) > in->maximumImbalance)) {
+#ifdef HAVE_ZOLTAN
+    if (PCU_Comm_Peers() < 16000) {
+      runZoltan(a);
+      printEntityImbalance(a->mesh);
+      return;
+    }
+    else {
+      runZoltan(a, apf::RIB);
+      printEntityImbalance(a->mesh);
+      return;
+    }
+#else
+    runParma(a);
+    printEntityImbalance(a->mesh);
+    return;
+#endif
+  }
 }
 
 }
diff --git a/ma/maInput.cc b/ma/maInput.cc
index e384a8b26..f83bdefd9 100644
--- a/ma/maInput.cc
+++ b/ma/maInput.cc
@@ -78,6 +78,18 @@ void rejectInput(const char* str)
   abort();
 }
 
+// if more than 1 option is true return true
+static bool moreThanOneOptionIsTrue(bool op1, bool op2, bool op3)
+{
+  int cnt = 0;
+  if (op1) cnt++;
+  if (op2) cnt++;
+  if (op3) cnt++;
+  if (cnt > 1)
+    return true;
+  return false;
+}
+
 void validateInput(Input* in)
 {
   if ( ! in->sizeField)

From 558b8752b8eccc18f91202a05c8f5aef31d172db Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 29 Jun 2021 16:30:33 -0400
Subject: [PATCH 361/555] Adds logic to figure out # adapt iterations

Using the size field information this logic figures out the required number
of adapt iterations. If the number is larger than 10 we cap it to 10 and show a
warning message to the user.
---
 ma/maInput.cc | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)

diff --git a/ma/maInput.cc b/ma/maInput.cc
index f83bdefd9..0929cd20e 100644
--- a/ma/maInput.cc
+++ b/ma/maInput.cc
@@ -11,6 +11,7 @@
 #include <lionPrint.h>
 #include <apfShape.h>
 #include <cstdio>
+#include <PCU.h>
 #include <pcu_util.h>
 #include <cstdlib>
 
@@ -132,6 +133,54 @@ void validateInput(Input* in)
     rejectInput("maximum imbalance less than 1.0");
   if (in->maximumEdgeRatio < 1.0)
     rejectInput("maximum tet edge ratio less than one");
+  if (moreThanOneOptionIsTrue(
+  	in->shouldRunPreZoltan,
+  	in->shouldRunPreZoltanRib,
+  	in->shouldRunPreParma))
+    rejectInput("only one of Zoltan, ZoltanRib, and Parma PreBalance options can be set to true!");
+  if (moreThanOneOptionIsTrue(
+  	in->shouldRunPostZoltan,
+  	in->shouldRunPostZoltanRib,
+  	in->shouldRunPostParma))
+    rejectInput("only one of Zoltan, ZoltanRib, and Parma PostBalance options can be set to true!");
+  if (in->shouldRunMidZoltan && in->shouldRunMidParma)
+    rejectInput("only one of Zoltan and Parma MidBalance options can be set to true!");
+#ifndef HAVE_ZOLTAN
+  if (in->shouldRunPreZoltan ||
+      in->shouldRunPreZoltanRib ||
+      in->shouldRunMidZoltan)
+    rejectInput("core is not compiled with Zoltan. Use a different balancer or compile core with ENABLE_ZOLTAN=ON!");
+#endif
+}
+
+static void configPrint(const char* format, ...)
+{
+  if (PCU_Comm_Self())
+    return;
+  lion_oprint(1, "\nMeshAdaptConfigure: ");
+  va_list ap;
+  va_start(ap, format);
+  lion_voprint(1, format, ap);
+  va_end(ap);
+  lion_oprint(1, "\n");
+}
+
+static void updateInputBasedOnSize(Mesh* m, Input* in)
+{
+  // number of iterations
+  double maxMetricLength = getMaximumEdgeLength(m, in->sizeField);
+  int iter = std::ceil(std::log2(maxMetricLength));
+  if (iter > 10) {
+    configPrint("Based on requested sizefield, MeshAdapt requires at least %d iterations,\n"
+    	"                    which is larger than the maximum of 10 allowed.\n"
+    	"                    Setting the number of iteration to 10!", iter);
+    in->maximumIterations = 10;
+  }
+  else {
+    configPrint("Based on requested sizefield, MeshAdapt requires at least %d iterations.\n"
+    	"                    Setting the number of iteration to %d!", iter, iter);
+    in->maximumIterations = iter;
+  }
 }
 
 void setSolutionTransfer(Input* in, SolutionTransfer* s)
@@ -172,6 +221,7 @@ Input* configure(
    solution transfer */
   Input* in = configure(m,s);
   in->sizeField = makeSizeField(m, f, logInterpolation);
+  updateInputBasedOnSize(m, in);
   return in;
 }
 
@@ -182,6 +232,7 @@ Input* configure(
 {
   Input* in = configure(m,s);
   in->sizeField = makeSizeField(m, f);
+  updateInputBasedOnSize(m, in);
   return in;
 }
 
@@ -192,6 +243,7 @@ Input* configure(
 {
   Input* in = configure(m,s);
   in->sizeField = makeSizeField(m, f);
+  updateInputBasedOnSize(m, in);
   return in;
 }
 
@@ -204,6 +256,7 @@ Input* configure(
 {
   Input* in = configure(m,s);
   in->sizeField = makeSizeField(m, sizes, frames, logInterpolation);
+  updateInputBasedOnSize(m, in);
   return in;
 }
 

From f9005bb6c0f158a5b66c38953d4bd4318f01a320 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Tue, 29 Jun 2021 19:40:13 -0400
Subject: [PATCH 362/555] cleans up maInput.cc

---
 ma/maInput.cc | 39 +++++++++++++++------------------------
 1 file changed, 15 insertions(+), 24 deletions(-)

diff --git a/ma/maInput.cc b/ma/maInput.cc
index 0929cd20e..6cd4858d9 100644
--- a/ma/maInput.cc
+++ b/ma/maInput.cc
@@ -8,6 +8,7 @@
  
 *******************************************************************************/
 #include "maInput.h"
+#include "maAdapt.h"
 #include <lionPrint.h>
 #include <apfShape.h>
 #include <cstdio>
@@ -74,6 +75,8 @@ void setDefaultValues(Input* in)
 
 void rejectInput(const char* str)
 {
+  if (PCU_Comm_Self() != 0)
+    return;
   lion_eprint(1,"MeshAdapt input error:\n");
   lion_eprint(1,"%s\n",str);
   abort();
@@ -153,33 +156,21 @@ void validateInput(Input* in)
 #endif
 }
 
-static void configPrint(const char* format, ...)
-{
-  if (PCU_Comm_Self())
-    return;
-  lion_oprint(1, "\nMeshAdaptConfigure: ");
-  va_list ap;
-  va_start(ap, format);
-  lion_voprint(1, format, ap);
-  va_end(ap);
-  lion_oprint(1, "\n");
-}
-
-static void updateInputBasedOnSize(Mesh* m, Input* in)
+static void updateMaxIterBasedOnSize(Mesh* m, Input* in)
 {
   // number of iterations
   double maxMetricLength = getMaximumEdgeLength(m, in->sizeField);
   int iter = std::ceil(std::log2(maxMetricLength));
-  if (iter > 10) {
-    configPrint("Based on requested sizefield, MeshAdapt requires at least %d iterations,\n"
-    	"                    which is larger than the maximum of 10 allowed.\n"
-    	"                    Setting the number of iteration to 10!", iter);
+  if (iter >= 10) {
+    print("ma::configure:  Based on requested sizefield, MeshAdapt requires at least %d iterations,\n"
+    	"           which is equal to or larger than the maximum of 10 allowed.\n"
+    	"           Setting the number of iteration to 10!", iter);
     in->maximumIterations = 10;
   }
   else {
-    configPrint("Based on requested sizefield, MeshAdapt requires at least %d iterations.\n"
-    	"                    Setting the number of iteration to %d!", iter, iter);
-    in->maximumIterations = iter;
+    print("ma::configure:  Based on requested sizefield, MeshAdapt requires at least %d iterations.\n"
+    	"           Setting the number of iteration to %d!", iter, iter+1);
+    in->maximumIterations = iter+1;
   }
 }
 
@@ -221,7 +212,7 @@ Input* configure(
    solution transfer */
   Input* in = configure(m,s);
   in->sizeField = makeSizeField(m, f, logInterpolation);
-  updateInputBasedOnSize(m, in);
+  updateMaxIterBasedOnSize(m, in);
   return in;
 }
 
@@ -232,7 +223,7 @@ Input* configure(
 {
   Input* in = configure(m,s);
   in->sizeField = makeSizeField(m, f);
-  updateInputBasedOnSize(m, in);
+  updateMaxIterBasedOnSize(m, in);
   return in;
 }
 
@@ -243,7 +234,7 @@ Input* configure(
 {
   Input* in = configure(m,s);
   in->sizeField = makeSizeField(m, f);
-  updateInputBasedOnSize(m, in);
+  updateMaxIterBasedOnSize(m, in);
   return in;
 }
 
@@ -256,7 +247,7 @@ Input* configure(
 {
   Input* in = configure(m,s);
   in->sizeField = makeSizeField(m, sizes, frames, logInterpolation);
-  updateInputBasedOnSize(m, in);
+  updateMaxIterBasedOnSize(m, in);
   return in;
 }
 

From 76ad3c4c5b345fa9c94cf3f3d517dae0afcec459 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Wed, 30 Jun 2021 03:26:07 -0400
Subject: [PATCH 363/555] Updates torus_ma_test

---
 test/torus_ma_test.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/test/torus_ma_test.cc b/test/torus_ma_test.cc
index fe04b6beb..190ad63f7 100644
--- a/test/torus_ma_test.cc
+++ b/test/torus_ma_test.cc
@@ -56,10 +56,14 @@ int main(int argc, char** argv)
   apf::writeVtkFiles("torus_before",m);
   CylindricalShock sf(m);
   ma::Input* in = ma::configure(m, &sf);
+  // Note that the optimal number of iterations will be set
+  // inside the call to ma::configure above. However for this
+  // test we override this value to 3 to reduce the run time of
+  // test.
+  in->maximumIterations = 3;
   in->shouldRunPreZoltan = true;
   in->shouldRunMidParma = true;
   in->shouldRunPostParma = true;
-  in->shouldRefineLayer = true;
   ma::adapt(in);
   m->verify();
   apf::writeVtkFiles("torus_after",m);

From 2b541add6242dbe38e5a433486f8dabadbc29197 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Wed, 30 Jun 2021 03:30:54 -0400
Subject: [PATCH 364/555] Updates refine2x to use either ZoltanRib or Parma

---
 test/refine2x.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test/refine2x.cc b/test/refine2x.cc
index 6413ad24f..db3f61095 100644
--- a/test/refine2x.cc
+++ b/test/refine2x.cc
@@ -104,8 +104,11 @@ int main(int argc, char** argv)
   ma::Mesh* m = apf::loadMdsMesh(argv[1],argv[2]);
   AnisotropicX* ansx = new AnisotropicX(m, atoi(argv[3]));
   ma::Input* in = ma::configure(m, ansx);
+#ifdef HAVE_ZOLTAN
   in->shouldRunPreZoltanRib = true;
+#else
   in->shouldRunPreParma = true;
+#endif
   in->shouldRunMidParma = true;
   in->shouldRunPostParma = true;
   in->maximumIterations = 10;

From 3943ce4f6cc2fcd89adeb30d9903db18a756aad1 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 30 Jun 2021 09:07:59 -0400
Subject: [PATCH 365/555] using target_compile_definitions

#342
---
 CMakeLists.txt        | 3 ---
 ma/maBalance.cc       | 6 +++---
 ma/maInput.cc         | 2 +-
 test/refine2x.cc      | 2 +-
 zoltan/CMakeLists.txt | 1 +
 5 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 86359c804..efa6f89a0 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -120,9 +120,6 @@ if(ENABLE_OMEGA_H)
   bob_public_dep(Omega_h)
 endif()
 
-if(ENABLE_ZOLTAN)
-  add_definitions(-DHAVE_ZOLTAN)
-endif()
 
 # Include the SCOREC project packages
 add_subdirectory(lion)
diff --git a/ma/maBalance.cc b/ma/maBalance.cc
index 32cb4967b..8077531fb 100644
--- a/ma/maBalance.cc
+++ b/ma/maBalance.cc
@@ -159,7 +159,7 @@ void preBalance(Adapt* a)
       (!in->shouldRunPreZoltanRib) &&
       (!in->shouldRunPreParma) &&
       (estimateWeightedImbalance(a) > in->maximumImbalance)) {
-#ifdef HAVE_ZOLTAN
+#ifdef PUMI_HAS_ZOLTAN
     if (PCU_Comm_Peers() < 16000) {
       runZoltan(a);
       return;
@@ -197,7 +197,7 @@ void midBalance(Adapt* a)
   if ((!in->shouldRunMidZoltan) &&
       (!in->shouldRunMidParma) &&
       (estimateWeightedImbalance(a) > in->maximumImbalance)) {
-#ifdef HAVE_ZOLTAN
+#ifdef PUMI_HAS_ZOLTAN
     if (PCU_Comm_Peers() < 16000) {
       runZoltan(a);
       return;
@@ -243,7 +243,7 @@ void postBalance(Adapt* a)
       (!in->shouldRunPostZoltanRib) &&
       (!in->shouldRunPostParma) &&
       (estimateWeightedImbalance(a) > in->maximumImbalance)) {
-#ifdef HAVE_ZOLTAN
+#ifdef PUMI_HAS_ZOLTAN
     if (PCU_Comm_Peers() < 16000) {
       runZoltan(a);
       printEntityImbalance(a->mesh);
diff --git a/ma/maInput.cc b/ma/maInput.cc
index 6cd4858d9..9d2eae040 100644
--- a/ma/maInput.cc
+++ b/ma/maInput.cc
@@ -148,7 +148,7 @@ void validateInput(Input* in)
     rejectInput("only one of Zoltan, ZoltanRib, and Parma PostBalance options can be set to true!");
   if (in->shouldRunMidZoltan && in->shouldRunMidParma)
     rejectInput("only one of Zoltan and Parma MidBalance options can be set to true!");
-#ifndef HAVE_ZOLTAN
+#ifndef PUMI_HAS_ZOLTAN
   if (in->shouldRunPreZoltan ||
       in->shouldRunPreZoltanRib ||
       in->shouldRunMidZoltan)
diff --git a/test/refine2x.cc b/test/refine2x.cc
index db3f61095..96f138004 100644
--- a/test/refine2x.cc
+++ b/test/refine2x.cc
@@ -104,7 +104,7 @@ int main(int argc, char** argv)
   ma::Mesh* m = apf::loadMdsMesh(argv[1],argv[2]);
   AnisotropicX* ansx = new AnisotropicX(m, atoi(argv[3]));
   ma::Input* in = ma::configure(m, ansx);
-#ifdef HAVE_ZOLTAN
+#ifdef PUMI_HAS_ZOLTAN
   in->shouldRunPreZoltanRib = true;
 #else
   in->shouldRunPreParma = true;
diff --git a/zoltan/CMakeLists.txt b/zoltan/CMakeLists.txt
index 5450ee52a..0901f9df5 100644
--- a/zoltan/CMakeLists.txt
+++ b/zoltan/CMakeLists.txt
@@ -63,6 +63,7 @@ if(ENABLE_ZOLTAN)
     ${ZOLTAN_LIBRARIES}
     ${PARMETIS_LIBRARIES}
   )
+  target_compile_definitions(apf_zoltan PUBLIC PUMI_HAS_ZOLTAN)
 endif()
 
 scorec_export_library(apf_zoltan)

From 41d0466a96d58f116f2ffff93e5ba8ddc2867568 Mon Sep 17 00:00:00 2001
From: Morteza <mortezah@users.noreply.github.com>
Date: Wed, 30 Jun 2021 13:18:25 -0400
Subject: [PATCH 366/555] Update ma/maBalance.cc

adds comment explaining the reason for having a condition on #parts when calling Zoltan

Co-authored-by: Cameron Smith <cwsmith@users.noreply.github.com>
---
 ma/maBalance.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ma/maBalance.cc b/ma/maBalance.cc
index 8077531fb..7b37bd017 100644
--- a/ma/maBalance.cc
+++ b/ma/maBalance.cc
@@ -160,7 +160,7 @@ void preBalance(Adapt* a)
       (!in->shouldRunPreParma) &&
       (estimateWeightedImbalance(a) > in->maximumImbalance)) {
 #ifdef PUMI_HAS_ZOLTAN
-    if (PCU_Comm_Peers() < 16000) {
+    if (PCU_Comm_Peers() < 16000) {  // the parmetis multi-level graph partitioner memory usage grows significantly with process count beyond 16K processes
       runZoltan(a);
       return;
     }

From 111804ed3c1206b72f974f93348dca774369caad Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Wed, 30 Jun 2021 13:31:27 -0400
Subject: [PATCH 367/555] Adds MAX_ZOLTAN_GRAPH_RANKS preprocessor

---
 ma/maBalance.cc | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/ma/maBalance.cc b/ma/maBalance.cc
index 7b37bd017..edeefa96c 100644
--- a/ma/maBalance.cc
+++ b/ma/maBalance.cc
@@ -4,6 +4,8 @@
 #include <parma.h>
 #include <apfZoltan.h>
 
+#define MAX_ZOLTAN_GRAPH_RANKS 16*1024
+
 namespace ma {
 
 static double clamp(double x, double max, double min)
@@ -160,7 +162,9 @@ void preBalance(Adapt* a)
       (!in->shouldRunPreParma) &&
       (estimateWeightedImbalance(a) > in->maximumImbalance)) {
 #ifdef PUMI_HAS_ZOLTAN
-    if (PCU_Comm_Peers() < 16000) {  // the parmetis multi-level graph partitioner memory usage grows significantly with process count beyond 16K processes
+    // The parmetis multi-level graph partitioner memory usage grows
+    // significantly with process count beyond 16K processes
+    if (PCU_Comm_Peers() < MAX_ZOLTAN_GRAPH_RANKS) {
       runZoltan(a);
       return;
     }
@@ -198,7 +202,9 @@ void midBalance(Adapt* a)
       (!in->shouldRunMidParma) &&
       (estimateWeightedImbalance(a) > in->maximumImbalance)) {
 #ifdef PUMI_HAS_ZOLTAN
-    if (PCU_Comm_Peers() < 16000) {
+    // The parmetis multi-level graph partitioner memory usage grows
+    // significantly with process count beyond 16K processes
+    if (PCU_Comm_Peers() < MAX_ZOLTAN_GRAPH_RANKS) {
       runZoltan(a);
       return;
     }
@@ -244,7 +250,9 @@ void postBalance(Adapt* a)
       (!in->shouldRunPostParma) &&
       (estimateWeightedImbalance(a) > in->maximumImbalance)) {
 #ifdef PUMI_HAS_ZOLTAN
-    if (PCU_Comm_Peers() < 16000) {
+    // The parmetis multi-level graph partitioner memory usage grows
+    // significantly with process count beyond 16K processes
+    if (PCU_Comm_Peers() < MAX_ZOLTAN_GRAPH_RANKS) {
       runZoltan(a);
       printEntityImbalance(a->mesh);
       return;

From 5d167bd0bfac2a9edd1e8700c7226eb84d846944 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Wed, 30 Jun 2021 13:34:26 -0400
Subject: [PATCH 368/555] Updates documentation in maInput.h for doxygen

---
 ma/maInput.h | 52 ++++++++++++++++++++++++++++++++++++----------------
 1 file changed, 36 insertions(+), 16 deletions(-)

diff --git a/ma/maInput.h b/ma/maInput.h
index b154f16e3..b2da33478 100644
--- a/ma/maInput.h
+++ b/ma/maInput.h
@@ -41,7 +41,11 @@ class Input
     SolutionTransfer* solutionTransfer;
     bool ownsSolutionTransfer;
     ShapeHandlerFunction shapeHandler;
-/** \brief number of refine/coarsen iterations to run (default 3) */
+/** \brief number of refine/coarsen iterations to run (default 3)
+    \details this value will be set to the minimum required iterations
+    inside the call to ma::configure in cases where there is a size information.
+    Users can override this by setting in->maximumIterations after the
+    call to ma::configure and before the call to ma::adapt routine.*/
     int maximumIterations;
 /** \brief whether to perform the collapse step */
     bool shouldCoarsen;
@@ -49,10 +53,10 @@ class Input
     \details requires modeler support, see gmi_can_eval */
     bool shouldSnap;
 /** \brief whether to transfer parametric coordinates
-  \details requires modeler support, see gmi_reparam */
+    \details requires modeler support, see gmi_reparam */
     bool shouldTransferParametric;
 /** \brief whether to transfer to the parametric coords of the closest point
-  \details requires modeler support, see gmi_closest_point */
+    \details requires modeler support, see gmi_closest_point */
     bool shouldTransferToClosestPoint;
 /** \brief whether to update matched entity info (limited support) */
     bool shouldHandleMatching;
@@ -63,35 +67,51 @@ class Input
 /** \brief whether to print the worst shape quality */
     bool shouldPrintQuality;
 /** \brief minimum desired mean ratio cubed for simplex elements
-   \details a different measure is used for curved elements */
+    \details a different measure is used for curved elements */
     double goodQuality;
 /** \brief whether to check the quality of split elems in DoubleSplitsCollapse
-   (default false) */
+    (default false) */
     double shouldCheckQualityForDoubleSplits;
 /** \brief minimum valid mean ratio cubed for simplex elements (default 1e-10)
-   \details used to define inside-out tetrahedra.
-   a different measure is used for curved elements */
+    \details used to define inside-out tetrahedra.
+    a different measure is used for curved elements */
     double validQuality;
 /** \brief imbalance target for all load balancing tools (default 1.10) */
     double maximumImbalance;
-/** \brief whether to run zoltan predictive load balancing (default false) */
+/** \brief whether to run zoltan predictive load balancing (default false)
+    \details if this and all the other PreBalance options are false, pre-balancing
+    occurs only if the imbalance is greater than in->maximumImbalance */
     bool shouldRunPreZoltan;
-/** \brief whether to run zoltan predictive load balancing using RIB (default false) */
+/** \brief whether to run zoltan predictive load balancing using RIB (default false)
+    \details if this and all the other PreBalance options are false, pre-balancing
+    occurs only if the imbalance is greater than in->maximumImbalance */
     bool shouldRunPreZoltanRib;
-/** \brief whether to run parma predictive load balancing (default false) */
+/** \brief whether to run parma predictive load balancing (default false)
+    \details if this and all the other PreBalance options are false, pre-balancing
+    occurs only if the imbalance is greater than in->maximumImbalance */
     bool shouldRunPreParma;
-/** \brief whether to run zoltan during adaptation (default false) */
+/** \brief whether to run zoltan during adaptation (default false)
+    \details if this and all the other MidBalance options are false, mid-balancing
+    occurs only if the imbalance is greater than in->maximumImbalance */
     bool shouldRunMidZoltan;
-/** \brief whether to run parma during adaptation (default false)*/
+/** \brief whether to run parma during adaptation (default false)
+    \details if this and all the other MidBalance options are false, mid-balancing
+    occurs only if the imbalance is greater than in->maximumImbalance */
     bool shouldRunMidParma;
-/** \brief whether to run zoltan after adapting (default false) */
+/** \brief whether to run zoltan after adapting (default false)
+    \details if this and all the other PostBalance options are false, post-balancing
+    occurs only if the imbalance is greater than in->maximumImbalance */
     bool shouldRunPostZoltan;
-/** \brief whether to run zoltan RIB after adapting (default false) */
+/** \brief whether to run zoltan RIB after adapting (default false)
+    \details if this and all the other PostBalance options are false, post-balancing
+    occurs only if the imbalance is greater than in->maximumImbalance */
     bool shouldRunPostZoltanRib;
-/** \brief whether to run parma after adapting (default false) */
+/** \brief whether to run parma after adapting (default false)
+    \details if this and all the other PostBalance options are false, post-balancing
+    occurs only if the imbalance is greater than in->maximumImbalance */
     bool shouldRunPostParma;
 /** \brief the ratio between longest and shortest edges that differentiates a
-   "short edge" element from a "large angle" element. */
+    "short edge" element from a "large angle" element. */
     double maximumEdgeRatio;
 /** \brief whether to tetrahedronize the boundary layer (default false)  */
     bool shouldTurnLayerToTets;

From 3cf7a27f17b16d3db11f9b81ef20e1389fc29e25 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Sun, 4 Jul 2021 17:25:03 -0400
Subject: [PATCH 369/555] Adds makes output of configure APIs const by default

This way users cannot modify the input options. There is a new API
ma::makeAdvanced that can be called on a const Input* to remove the
constant-ness so (advanced) users can modify adapt options if they
choose to do so.
---
 crv/crvAdapt.cc                   |  2 +-
 ma/ma.cc                          | 10 +++++++++
 ma/ma.h                           | 12 ++++++++--
 ma/maDBG.cc                       |  2 +-
 ma/maInput.cc                     | 37 ++++++++++++++++++-------------
 ma/maInput.h                      | 18 +++++++++------
 ma/maLayerCoarsen.cc              |  2 +-
 phasta/phAdapt.cc                 |  8 +++----
 phasta/ph_convert.cc              |  2 +-
 test/aniso_ma_test.cc             |  2 +-
 test/bezierRefine.cc              |  6 ++---
 test/classifyThenAdapt.cc         |  2 +-
 test/convert.cc                   |  2 +-
 test/curvetest.cc                 |  4 ++--
 test/degenerateSurfs.cc           |  2 +-
 test/dg_ma_test.cc                |  2 +-
 test/fixlayer.cc                  |  2 +-
 test/fixshape.cc                  |  2 +-
 test/fusion.cc                    |  2 +-
 test/fusion3.cc                   |  2 +-
 test/generate.cc                  |  2 +-
 test/highOrderSizeFields.cc       | 12 +++++-----
 test/highOrderSolutionTransfer.cc |  2 +-
 test/ma_test.cc                   |  2 +-
 test/ma_test_analytic_model.cc    |  2 +-
 test/ph_adapt.cc                  |  2 +-
 test/refine2x.cc                  |  2 +-
 test/snap.cc                      |  2 +-
 test/tetrahedronize.cc            |  2 +-
 test/torus_ma_test.cc             |  2 +-
 test/uniform.cc                   |  2 +-
 31 files changed, 90 insertions(+), 63 deletions(-)

diff --git a/crv/crvAdapt.cc b/crv/crvAdapt.cc
index 3b6baec15..7e33c9729 100644
--- a/crv/crvAdapt.cc
+++ b/crv/crvAdapt.cc
@@ -154,7 +154,7 @@ ma::Input* configureShapeCorrection(
     ma::Mesh* m, ma::SizeField* f,
     ma::SolutionTransfer* s)
 {
-  ma::Input* in = ma::configureIdentity(m,f,s);
+  ma::Input* in = ma::makeAdvanced(ma::configureIdentity(m,f,s));
   in->shouldFixShape = true;
   in->shouldSnap = in->mesh->canSnap();
   in->shouldTransferParametric = in->mesh->canSnap();
diff --git a/ma/ma.cc b/ma/ma.cc
index b5c2d3ca6..2b0ef6c21 100644
--- a/ma/ma.cc
+++ b/ma/ma.cc
@@ -51,6 +51,11 @@ void adapt(Input* in)
   apf::printStats(m);
 }
 
+void adapt(const Input* in)
+{
+  adapt(makeAdvanced(in));
+}
+
 void adaptVerbose(Input* in, bool verbose)
 {
   print("version 2.0 - dev !");
@@ -115,6 +120,11 @@ void adaptVerbose(Input* in, bool verbose)
   apf::printStats(m);
 }
 
+void adaptVerbose(const Input* in, bool verbose)
+{
+  adaptVerbose(makeAdvanced(in), verbose);
+}
+
 
 void adapt(Mesh* m, AnisotropicFunction* f, SolutionTransfer* s)
 {
diff --git a/ma/ma.h b/ma/ma.h
index e142acd4d..261948e01 100644
--- a/ma/ma.h
+++ b/ma/ma.h
@@ -45,14 +45,22 @@ namespace ma {
 void adapt(Mesh* m, IsotropicFunction* f, SolutionTransfer* s=0);
 /** \brief adapt based on an anisotropic function */
 void adapt(Mesh* m, AnisotropicFunction* f, SolutionTransfer* s=0);
-/** \brief adapt with custom configuration
+/** \brief adapt with custom mutable configuration Input
   \details see maInput.h for details.
   note that this function will delete the Input object. */
 void adapt(Input* in);
-/** \brief adapt verbose for debugging
+/** \brief adapt with un-mutable configuration Input
+  \details see maInput.h for details.
+  note that this function will delete the Input object. */
+void adapt(const Input* in);
+/** \brief adapt verbose for debugging with mutable configuration Input
   \details see maInput.h for details. The mesh will be
   written (vtk-format) at each operation stage */
 void adaptVerbose(Input* in, bool verbosef = false);
+/** \brief adapt verbose for debugging with unmutable configuration Input
+  \details see maInput.h for details. The mesh will be
+  written (vtk-format) at each operation stage */
+void adaptVerbose(const Input* in, bool verbosef = false);
 /** \brief run uniform refinement, plus snapping and shape correction */
 void runUniformRefinement(Mesh* m, int n=1, SolutionTransfer* s=0);
 /** \brief run uniform refinement with matched entity support
diff --git a/ma/maDBG.cc b/ma/maDBG.cc
index 71d4459ea..ccbd48218 100644
--- a/ma/maDBG.cc
+++ b/ma/maDBG.cc
@@ -410,7 +410,7 @@ void uniformAdaptByModelTag(
     int mtag,
     int level)
 {
-  ma::Input* in = ma::configureUniformRefine(m, 0);
+  ma::Input* in = ma::makeAdvanced(ma::configureUniformRefine(m, 0));
   ma::validateInput(in);
   ma::Adapt* a = new ma::Adapt(in);
   for(int i = 0; i < level; i++) {
diff --git a/ma/maInput.cc b/ma/maInput.cc
index 9d2eae040..078fdd560 100644
--- a/ma/maInput.cc
+++ b/ma/maInput.cc
@@ -188,7 +188,7 @@ void setSolutionTransfer(Input* in, SolutionTransfer* s)
   }
 }
 
-Input* configure(
+static Input* configure(
     Mesh* m,
     SolutionTransfer* s)
 {
@@ -199,7 +199,7 @@ Input* configure(
   return in;
 }
 
-Input* configure(
+const Input* configure(
     Mesh* m,
     AnisotropicFunction* f,
     SolutionTransfer* s,
@@ -213,10 +213,10 @@ Input* configure(
   Input* in = configure(m,s);
   in->sizeField = makeSizeField(m, f, logInterpolation);
   updateMaxIterBasedOnSize(m, in);
-  return in;
+  return (const Input*)in;
 }
 
-Input* configure(
+const Input* configure(
     Mesh* m,
     IsotropicFunction* f,
     SolutionTransfer* s)
@@ -224,10 +224,10 @@ Input* configure(
   Input* in = configure(m,s);
   in->sizeField = makeSizeField(m, f);
   updateMaxIterBasedOnSize(m, in);
-  return in;
+  return (const Input*)in;
 }
 
-Input* configure(
+const Input* configure(
     Mesh* m,
     apf::Field* f,
     SolutionTransfer* s)
@@ -235,10 +235,10 @@ Input* configure(
   Input* in = configure(m,s);
   in->sizeField = makeSizeField(m, f);
   updateMaxIterBasedOnSize(m, in);
-  return in;
+  return (const Input*)in;
 }
 
-Input* configure(
+const Input* configure(
     Mesh* m,
     apf::Field* sizes,
     apf::Field* frames,
@@ -248,28 +248,28 @@ Input* configure(
   Input* in = configure(m,s);
   in->sizeField = makeSizeField(m, sizes, frames, logInterpolation);
   updateMaxIterBasedOnSize(m, in);
-  return in;
+  return (const Input*)in;
 }
 
-Input* configureUniformRefine(Mesh* m, int n, SolutionTransfer* s)
+const Input* configureUniformRefine(Mesh* m, int n, SolutionTransfer* s)
 {
   Input* in = configure(m,s);
   in->sizeField = new UniformRefiner(m);
   in->maximumIterations = n;
   in->shouldRefineLayer = true;
   in->splitAllLayerEdges = true;
-  return in;
+  return (const Input*)in;
 }
 
-Input* configureMatching(Mesh* m, int n, SolutionTransfer* s)
+const Input* configureMatching(Mesh* m, int n, SolutionTransfer* s)
 {
-  Input* in = configureUniformRefine(m,n,s);
+  Input* in = makeAdvanced(configureUniformRefine(m,n,s));
   in->shouldHandleMatching = true;
   in->shouldFixShape = false;
-  return in;
+  return (const Input*)in;
 }
 
-Input* configureIdentity(Mesh* m, SizeField* f, SolutionTransfer* s)
+const Input* configureIdentity(Mesh* m, SizeField* f, SolutionTransfer* s)
 {
   Input* in = configure(m,s);
   if (f)
@@ -285,7 +285,12 @@ Input* configureIdentity(Mesh* m, SizeField* f, SolutionTransfer* s)
   in->maximumIterations = 0;
   in->shouldFixShape = false;
   in->shouldSnap = false;
-  return in;
+  return (const Input*)in;
+}
+
+Input* makeAdvanced(const Input* in)
+{
+  return const_cast<Input*>(in);
 }
 
 }
diff --git a/ma/maInput.h b/ma/maInput.h
index b2da33478..51f12941a 100644
--- a/ma/maInput.h
+++ b/ma/maInput.h
@@ -135,7 +135,7 @@ class Input
  \param s if non-zero, use that to transfer all fields. otherwise,
           transfer any associated fields with default algorithms
  \param logInterpolation if true uses logarithmic interpolation */
-Input* configure(
+const Input* configure(
     Mesh* m,
     AnisotropicFunction* f,
     SolutionTransfer* s=0,
@@ -143,7 +143,7 @@ Input* configure(
 /** \brief generate a configuration based on an isotropic function
  \param s if non-zero, use that to transfer all fields. otherwise,
           transfer any associated fields with default algorithms */
-Input* configure(
+const Input* configure(
     Mesh* m,
     IsotropicFunction* f,
     SolutionTransfer* s=0);
@@ -155,7 +155,7 @@ Input* configure(
  \param s if non-zero, use that to transfer all fields. otherwise,
           transfer any associated fields with default algorithms
  \param logInterpolation if true uses logarithmic interpolation */
-Input* configure(
+const Input* configure(
     Mesh* m,
     apf::Field* sizes,
     apf::Field* frames,
@@ -165,21 +165,25 @@ Input* configure(
  \param size a scalar field of desired element size
  \param s if non-zero, use that to transfer all fields. otherwise,
           transfer any associated fields with default algorithms */
-Input* configure(
+const Input* configure(
     Mesh* m,
     apf::Field* size,
     SolutionTransfer* s=0);
 
 /** \brief generate a uniform refinement configuration */
-Input* configureUniformRefine(Mesh* m, int n=1, SolutionTransfer* s=0);
+const Input* configureUniformRefine(Mesh* m, int n=1, SolutionTransfer* s=0);
 /** \brief generate a matched uniform refinement configuration */
-Input* configureMatching(Mesh* m, int n=1, SolutionTransfer* s=0);
+const Input* configureMatching(Mesh* m, int n=1, SolutionTransfer* s=0);
 /** \brief generate a no-op configuration */
-Input* configureIdentity(Mesh* m, SizeField* f=0, SolutionTransfer* s=0);
+const Input* configureIdentity(Mesh* m, SizeField* f=0, SolutionTransfer* s=0);
 
 /** \brief used internally, but users can call this if they want */
 void validateInput(Input* in);
 
+/** \brief removes constant-ness from a constant Input pointer
+           so users can modify it */
+Input* makeAdvanced(const Input* in);
+
 }
 
 #endif
diff --git a/ma/maLayerCoarsen.cc b/ma/maLayerCoarsen.cc
index b1a8acad0..38ace0f05 100644
--- a/ma/maLayerCoarsen.cc
+++ b/ma/maLayerCoarsen.cc
@@ -190,7 +190,7 @@ static void migrateForLayerCollapse(Adapt* a, int d, int round)
 }
 
 void localizeLayerStacks(Mesh* m) {
-  Input* in = configureIdentity(m);
+  Input* in = makeAdvanced(configureIdentity(m));
   Adapt* a = new Adapt(in);
   //mark layer bases
   findLayerBase(a);
diff --git a/phasta/phAdapt.cc b/phasta/phAdapt.cc
index 805cfdbe9..9ae8f783a 100644
--- a/phasta/phAdapt.cc
+++ b/phasta/phAdapt.cc
@@ -57,7 +57,7 @@ struct AdaptCallback : public Parma_GroupCode
   AdaptCallback(apf::Mesh2* m, apf::Field* szfld, ph::Input* inp)
     : mesh(m), field(szfld), in(inp) { }
   void run(int) {
-    ma::Input* ma_in = ma::configure(mesh, field);
+    ma::Input* ma_in = ma::makeAdvanced(ma::configure(mesh, field));
     if( in ) {
       //chef defaults
       ma_in->shouldRunPreZoltan = true;
@@ -156,7 +156,7 @@ static void runFromGivenSize(Input& in, apf::Mesh2* m)
 
 void tetrahedronize(Input& in, apf::Mesh2* m)
 {
-  ma::Input* ma_in = ma::configureIdentity(m);
+  ma::Input* ma_in = ma::makeAdvanced(ma::configureIdentity(m));
   ph::setupBalance("preAdaptBalanceMethod", in.preAdaptBalanceMethod,
       ma_in->shouldRunPreParma, ma_in->shouldRunPreZoltan,
       ma_in->shouldRunPreZoltanRib);
@@ -212,7 +212,7 @@ namespace chef {
 
   void adaptLevelSet(ph::Input& in, apf::Mesh2* m)
   {
-    ma::Input* ma_in = ma::configureMatching(m, in.recursiveUR);
+    ma::Input* ma_in = ma::makeAdvanced(ma::configureMatching(m, in.recursiveUR));
     ph::setupBalance("preAdaptBalanceMethod", in.preAdaptBalanceMethod,
         ma_in->shouldRunPreParma, ma_in->shouldRunPreZoltan,
         ma_in->shouldRunPreZoltanRib);
@@ -244,7 +244,7 @@ namespace chef {
 
   void uniformRefinement(ph::Input& in, apf::Mesh2* m)
   {
-    ma::Input* ma_in = ma::configureMatching(m, in.recursiveUR);
+    ma::Input* ma_in = ma::makeAdvanced(ma::configureMatching(m, in.recursiveUR));
     ph::setupBalance("preAdaptBalanceMethod", in.preAdaptBalanceMethod,
         ma_in->shouldRunPreParma, ma_in->shouldRunPreZoltan,
         ma_in->shouldRunPreZoltanRib);
diff --git a/phasta/ph_convert.cc b/phasta/ph_convert.cc
index 65df57f3b..52e22a6af 100644
--- a/phasta/ph_convert.cc
+++ b/phasta/ph_convert.cc
@@ -55,7 +55,7 @@ static void fixPyramids(apf::Mesh2* m)
     return; /* no pyramids exist in 2D */
   if (apf::countEntitiesOfType(m, apf::Mesh::HEX))
     return; /* meshadapt can't even look at hexes */
-  ma::Input* in = ma::configureIdentity(m);
+  ma::Input* in = ma::makeAdvanced(ma::configureIdentity(m));
   in->shouldCleanupLayer = true;
   ma::adapt(in);
 }
diff --git a/test/aniso_ma_test.cc b/test/aniso_ma_test.cc
index b93dc500e..4eb9a0278 100644
--- a/test/aniso_ma_test.cc
+++ b/test/aniso_ma_test.cc
@@ -53,7 +53,7 @@ int main(int argc, char** argv)
   m->verify();
   apf::writeVtkFiles("aniso_before",m);
   AnIso sf(m);
-  ma::Input* in = ma::configure(m, &sf, 0, logInterpolation);
+  ma::Input* in = ma::makeAdvanced(ma::configure(m, &sf, 0, logInterpolation));
   in->shouldRunPreZoltan = true;
   in->shouldRunMidParma = true;
   in->shouldRunPostParma = true;
diff --git a/test/bezierRefine.cc b/test/bezierRefine.cc
index 1900202bb..681930071 100644
--- a/test/bezierRefine.cc
+++ b/test/bezierRefine.cc
@@ -223,7 +223,7 @@ void test2D()
       }
       }
       double v0 = measureMesh(m);
-      ma::Input* inRefine = ma::configureUniformRefine(m,1);
+      ma::Input* inRefine = ma::makeAdvanced(ma::configureUniformRefine(m,1));
       inRefine->shouldSnap = true;
       inRefine->shouldTransferParametric = true;
       if(order > 1)
@@ -270,7 +270,7 @@ void test3D()
     bc.run();
 
     double v0 = measureMesh(m);
-    ma::Input* inRefine = ma::configureUniformRefine(m,1);
+    ma::Input* inRefine = ma::makeAdvanced(ma::configureUniformRefine(m,1));
     inRefine->shouldSnap = false;
     inRefine->shouldTransferParametric = false;
     if(order > 1)
@@ -290,7 +290,7 @@ void test3D()
     bc.run();
 
     double v0 = measureMesh(m);
-    ma::Input* inRefine = ma::configureUniformRefine(m,1);
+    ma::Input* inRefine = ma::makeAdvanced(ma::configureUniformRefine(m,1));
     inRefine->shouldSnap = false;
     inRefine->shouldTransferParametric = false;
     if(order > 1)
diff --git a/test/classifyThenAdapt.cc b/test/classifyThenAdapt.cc
index 2cc2cccd6..e0ef96ac1 100644
--- a/test/classifyThenAdapt.cc
+++ b/test/classifyThenAdapt.cc
@@ -139,7 +139,7 @@ int main(int argc, char** argv)
 
   printClassCounts(m);
 
-  ma::Input* in = ma::configureUniformRefine(m, 1);
+  ma::Input* in = ma::makeAdvanced(ma::configureUniformRefine(m, 1));
   if (in->shouldSnap) {
     in->shouldSnap = false;
     PCU_ALWAYS_ASSERT(in->shouldTransferParametric);
diff --git a/test/convert.cc b/test/convert.cc
index 18f6fd5c1..bd27e6266 100644
--- a/test/convert.cc
+++ b/test/convert.cc
@@ -42,7 +42,7 @@ static void fixPyramids(apf::Mesh2* m)
     return; /* no pyramids exist in 2D */
   if (apf::countEntitiesOfType(m, apf::Mesh::HEX))
     return; /* meshadapt can't even look at hexes */
-  ma::Input* in = ma::configureIdentity(m);
+  ma::Input* in = ma::makeAdvanced(ma::configureIdentity(m));
   in->shouldCleanupLayer = true;
   ma::adapt(in);
 }
diff --git a/test/curvetest.cc b/test/curvetest.cc
index 70e90dbc2..3bf962ec2 100644
--- a/test/curvetest.cc
+++ b/test/curvetest.cc
@@ -142,7 +142,7 @@ static void testBezier(const char* modelFile, const char* meshFile,
     crv::BezierCurver bc(m,order,0);
     bc.run();
     Linear sf(m);
-    ma::Input* in = ma::configure(m,&sf);
+    ma::Input* in = ma::makeAdvanced(ma::configure(m,&sf));
     in->shouldSnap = true;
     in->shouldTransferParametric = true;
     in->maximumIterations = 1;
@@ -157,7 +157,7 @@ static void testBezier(const char* modelFile, const char* meshFile,
     crv::BezierCurver bc(m,order,1);
     bc.run();
     Linear sf(m);
-    ma::Input* in = ma::configure(m,&sf);
+    ma::Input* in = ma::makeAdvanced(ma::configure(m,&sf));
     in->shouldSnap = true;
     in->shouldTransferParametric = true;
     in->maximumIterations = 1;
diff --git a/test/degenerateSurfs.cc b/test/degenerateSurfs.cc
index 583adbf16..730e8cbf7 100644
--- a/test/degenerateSurfs.cc
+++ b/test/degenerateSurfs.cc
@@ -55,7 +55,7 @@ int main(int argc, char** argv)
   crv::BezierCurver bc(m, order, 0);
   bc.run();
 
-  ma::Input* in = ma::configureUniformRefine(m, level);
+  ma::Input* in = ma::makeAdvanced(ma::configureUniformRefine(m, level));
   if (in->shouldSnap) {
     PCU_ALWAYS_ASSERT(in->shouldTransferParametric);
   }
diff --git a/test/dg_ma_test.cc b/test/dg_ma_test.cc
index c46f04c8d..188592c6a 100644
--- a/test/dg_ma_test.cc
+++ b/test/dg_ma_test.cc
@@ -54,7 +54,7 @@ int main(int argc, char** argv)
   ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile);
   m->verify();
   Linear sf(m);
-  ma::Input* in = ma::configure(m, &sf);
+  ma::Input* in = ma::makeAdvanced(ma::configure(m, &sf));
   if (!PCU_Comm_Self())
     printf("Matched mesh: disabling"
            " snapping, and shape correction,\n");
diff --git a/test/fixlayer.cc b/test/fixlayer.cc
index 20b26ec70..0827bc025 100644
--- a/test/fixlayer.cc
+++ b/test/fixlayer.cc
@@ -27,7 +27,7 @@ int main(int argc, char** argv)
 #endif
   gmi_register_mesh();
   ma::Mesh* m = apf::loadMdsMesh(argv[1], argv[2]);
-  ma::Input* in = ma::configureIdentity(m);
+  ma::Input* in = ma::makeAdvanced(ma::configureIdentity(m));
   in->shouldCleanupLayer = true;
   ma::adapt(in);
   m->writeNative(argv[3]);
diff --git a/test/fixshape.cc b/test/fixshape.cc
index 543ee0df6..899e1c0e4 100644
--- a/test/fixshape.cc
+++ b/test/fixshape.cc
@@ -27,7 +27,7 @@ int main(int argc, char** argv)
 #endif
   gmi_register_mesh();
   ma::Mesh* m = apf::loadMdsMesh(argv[1],argv[2]);
-  ma::Input* in = ma::configureIdentity(m);
+  ma::Input* in = ma::makeAdvanced(ma::configureIdentity(m));
   in->shouldFixShape = true;
   ma::adapt(in);
   m->writeNative(argv[3]);
diff --git a/test/fusion.cc b/test/fusion.cc
index 2e83bbb6b..4107570dc 100644
--- a/test/fusion.cc
+++ b/test/fusion.cc
@@ -129,7 +129,7 @@ static void testIndexing(apf::Mesh2* m)
 static void fusionAdapt(apf::Mesh2* m)
 {
   Vortex sf(m);
-  ma::Input* in = ma::configure(m, &sf);
+  ma::Input* in = ma::makeAdvanced(ma::configure(m, &sf));
   in->maximumIterations = 9;
   in->shouldRunPreZoltan = true;
   in->shouldRunMidParma = true;
diff --git a/test/fusion3.cc b/test/fusion3.cc
index 99dd00234..4fa55788c 100644
--- a/test/fusion3.cc
+++ b/test/fusion3.cc
@@ -290,7 +290,7 @@ int main(int argc, char * argv[])
   apf::Mesh2* mesh=apf::loadMdsMesh(model, argv[1]);
   mesh->verify();
   Vortex sfv(mesh, center, modelLen);
-  ma::Input* in = ma::configure(mesh,&sfv);
+  ma::Input* in = ma::makeAdvanced(ma::configure(mesh,&sfv));
   in->maximumIterations = 9;
   ma::adapt(in);
   mesh->verify();
diff --git a/test/generate.cc b/test/generate.cc
index cfff3b502..a5c94f15e 100644
--- a/test/generate.cc
+++ b/test/generate.cc
@@ -227,7 +227,7 @@ void fixMatches(apf::Mesh2* m)
 
 void fixPyramids(apf::Mesh2* m)
 {
-  ma::Input* in = ma::configureIdentity(m);
+  ma::Input* in = ma::makeAdvanced(ma::configureIdentity(m));
   in->shouldCleanupLayer = true;
   ma::adapt(in);
 }
diff --git a/test/highOrderSizeFields.cc b/test/highOrderSizeFields.cc
index 8e6fa9b4c..7cbbe34fc 100644
--- a/test/highOrderSizeFields.cc
+++ b/test/highOrderSizeFields.cc
@@ -119,7 +119,7 @@ void testAdapt(
   apf::Mesh2* m = apf::loadMdsMesh(g,mesh);
   m->verify();
 
-  ma::Input* in = ma::configureUniformRefine(m, 1, 0);
+  const ma::Input* in = ma::configureUniformRefine(m, 1, 0);
   ma::adapt(in);
 
   apf::FieldShape* fs = apf::getH1Shape(order);
@@ -153,16 +153,16 @@ void testAdapt(
     m->end(it);
   }
 
-  in = ma::configure(m, sizes, frames, 0, true);
-  in->shouldFixShape = true;
-  in->maximumIterations = 10;
-  in->shouldForceAdaptation = true;
+  ma::Input* inAdv = ma::makeAdvanced(ma::configure(m, sizes, frames, 0, true));
+  inAdv->shouldFixShape = true;
+  inAdv->maximumIterations = 10;
+  inAdv->shouldForceAdaptation = true;
 
   std::stringstream ss;
   ss << "before_adapt_with_ho_sizefield_order_" << order;
   apf::writeVtkFiles(ss.str().c_str(), m);
   ss.str("");
-  ma::adaptVerbose(in);
+  ma::adaptVerbose(inAdv);
   ss << "after_adapt_with_ho_sizefield_order_" << order;
   apf::writeVtkFiles(ss.str().c_str(), m);
 
diff --git a/test/highOrderSolutionTransfer.cc b/test/highOrderSolutionTransfer.cc
index 8fbebcf2e..47e0484a4 100644
--- a/test/highOrderSolutionTransfer.cc
+++ b/test/highOrderSolutionTransfer.cc
@@ -209,7 +209,7 @@ void testCurveAdapt(
   apf::Field* f = addH1Field(m, field_order, exact_order);
   double l2ErrorBefore = testH1Field(m, f, apf::Vector3(1./3., 1./4., 1./5.), exact_order);
 
-  ma::Input* in = ma::configureUniformRefine(m,1);
+  ma::Input* in = ma::makeAdvanced(ma::configureUniformRefine(m,1));
   // Snap is off for solutions transfer testing.
   in->shouldSnap = false;
   in->goodQuality = 0.3*0.3*0.3;
diff --git a/test/ma_test.cc b/test/ma_test.cc
index 7b83cd19c..c9e3284c3 100644
--- a/test/ma_test.cc
+++ b/test/ma_test.cc
@@ -54,7 +54,7 @@ int main(int argc, char** argv)
   ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile);
   m->verify();
   Linear sf(m);
-  ma::Input* in = ma::configure(m, &sf);
+  ma::Input* in = ma::makeAdvanced(ma::configure(m, &sf));
   in->shouldRunPreZoltan = true;
   in->shouldRunMidParma = true;
   in->shouldRunPostParma = true;
diff --git a/test/ma_test_analytic_model.cc b/test/ma_test_analytic_model.cc
index 8c210618a..8422ca6b3 100644
--- a/test/ma_test_analytic_model.cc
+++ b/test/ma_test_analytic_model.cc
@@ -179,7 +179,7 @@ int main(int argc, char** argv)
 
   apf::writeVtkFiles("initial_mesh_on_analytic_model", m);
 
-  ma::Input* in = ma::configureUniformRefine(m, 2);
+  ma::Input* in = ma::makeAdvanced(ma::configureUniformRefine(m, 2));
   in->shouldSnap = true;
   in->shouldTransferParametric = true;
   in->shouldFixShape = true;
diff --git a/test/ph_adapt.cc b/test/ph_adapt.cc
index d8f0b7d1d..927ca1aca 100644
--- a/test/ph_adapt.cc
+++ b/test/ph_adapt.cc
@@ -75,7 +75,7 @@ int main(int argc, char** argv)
   sam::multiplySF(m, szFld, 1.0);
   apf::writeVtkFiles("before",m);
   /* mesh adaptation */
-  ma::Input* ma_in = ma::configure(m, szFld);
+  ma::Input* ma_in = ma::makeAdvanced(ma::configure(m, szFld));
   ma_in->shouldRunPreZoltan = true;
   ma_in->shouldRunMidParma = true;
   ma_in->shouldRunPostParma = true;
diff --git a/test/refine2x.cc b/test/refine2x.cc
index 96f138004..d09c090d5 100644
--- a/test/refine2x.cc
+++ b/test/refine2x.cc
@@ -103,7 +103,7 @@ int main(int argc, char** argv)
   gmi_register_mesh();
   ma::Mesh* m = apf::loadMdsMesh(argv[1],argv[2]);
   AnisotropicX* ansx = new AnisotropicX(m, atoi(argv[3]));
-  ma::Input* in = ma::configure(m, ansx);
+  ma::Input* in = ma::makeAdvanced(ma::configure(m, ansx));
 #ifdef PUMI_HAS_ZOLTAN
   in->shouldRunPreZoltanRib = true;
 #else
diff --git a/test/snap.cc b/test/snap.cc
index c5b63379a..7d633354b 100644
--- a/test/snap.cc
+++ b/test/snap.cc
@@ -21,7 +21,7 @@ int main(int argc, char** argv)
   gmi_sim_start();
   gmi_register_sim();
   ma::Mesh* m = apf::loadMdsMesh(argv[1],argv[2]);
-  ma::Input* in = ma::configureIdentity(m);
+  ma::Input* in = ma::makeAdvanced(ma::configureIdentity(m));
   in->shouldSnap = true;
   in->shouldTransferParametric = true;
   ma::adapt(in);
diff --git a/test/tetrahedronize.cc b/test/tetrahedronize.cc
index a6192cc4c..edd035d3f 100644
--- a/test/tetrahedronize.cc
+++ b/test/tetrahedronize.cc
@@ -27,7 +27,7 @@ int main(int argc, char** argv)
 #endif
   gmi_register_mesh();
   ma::Mesh* m = apf::loadMdsMesh(argv[1], argv[2]);
-  ma::Input* in = ma::configureIdentity(m);
+  ma::Input* in = ma::makeAdvanced(ma::configureIdentity(m));
   in->shouldTurnLayerToTets = true;
   ma::adapt(in);
   m->writeNative(argv[3]);
diff --git a/test/torus_ma_test.cc b/test/torus_ma_test.cc
index 190ad63f7..d7d6bcc06 100644
--- a/test/torus_ma_test.cc
+++ b/test/torus_ma_test.cc
@@ -55,7 +55,7 @@ int main(int argc, char** argv)
   m->verify();
   apf::writeVtkFiles("torus_before",m);
   CylindricalShock sf(m);
-  ma::Input* in = ma::configure(m, &sf);
+  ma::Input* in = ma::makeAdvanced(ma::configure(m, &sf));
   // Note that the optimal number of iterations will be set
   // inside the call to ma::configure above. However for this
   // test we override this value to 3 to reduce the run time of
diff --git a/test/uniform.cc b/test/uniform.cc
index dc43109e5..427598a06 100644
--- a/test/uniform.cc
+++ b/test/uniform.cc
@@ -46,7 +46,7 @@ int main(int argc, char** argv)
   gmi_register_mesh();
   getConfig(argc,argv);
   ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile);
-  ma::Input* in = ma::configureUniformRefine(m, 1);
+  ma::Input* in = ma::makeAdvanced(ma::configureUniformRefine(m, 1));
   if (in->shouldSnap) {
     in->shouldSnap = false;
     PCU_ALWAYS_ASSERT(in->shouldTransferParametric);

From 05af6559daa95a3a49f985adc1516799b92a9502 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Sun, 4 Jul 2021 17:38:19 -0400
Subject: [PATCH 370/555] Removes advance adapt options from
 classifyThenAdapt.cc

---
 test/classifyThenAdapt.cc | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/test/classifyThenAdapt.cc b/test/classifyThenAdapt.cc
index e0ef96ac1..a2d01f3d0 100644
--- a/test/classifyThenAdapt.cc
+++ b/test/classifyThenAdapt.cc
@@ -139,12 +139,7 @@ int main(int argc, char** argv)
 
   printClassCounts(m);
 
-  ma::Input* in = ma::makeAdvanced(ma::configureUniformRefine(m, 1));
-  if (in->shouldSnap) {
-    in->shouldSnap = false;
-    PCU_ALWAYS_ASSERT(in->shouldTransferParametric);
-  }
-  in->shouldFixShape = false;
+  const ma::Input* in = ma::configureUniformRefine(m, 1);
   ma::adapt(in);
 
   printClassCounts(m);

From c744ce7b91c5073a8008b41e8128af79b74118c3 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Sun, 4 Jul 2021 17:51:49 -0400
Subject: [PATCH 371/555] Removes advance adapt options from fusion.cc

---
 test/fusion.cc | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/test/fusion.cc b/test/fusion.cc
index 4107570dc..7da4c72f2 100644
--- a/test/fusion.cc
+++ b/test/fusion.cc
@@ -129,11 +129,7 @@ static void testIndexing(apf::Mesh2* m)
 static void fusionAdapt(apf::Mesh2* m)
 {
   Vortex sf(m);
-  ma::Input* in = ma::makeAdvanced(ma::configure(m, &sf));
-  in->maximumIterations = 9;
-  in->shouldRunPreZoltan = true;
-  in->shouldRunMidParma = true;
-  in->shouldRunPostParma = true;
+  const ma::Input* in = ma::configure(m, &sf);
   ma::adapt(in);
   m->verify();
 }

From 5586be6b4043adb08ce2b553d51e62dd01e1da9d Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Sun, 4 Jul 2021 17:53:27 -0400
Subject: [PATCH 372/555] emoves advance adapt options from fusion3cc

---
 test/fusion3.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/test/fusion3.cc b/test/fusion3.cc
index 4fa55788c..a8df775e4 100644
--- a/test/fusion3.cc
+++ b/test/fusion3.cc
@@ -290,8 +290,7 @@ int main(int argc, char * argv[])
   apf::Mesh2* mesh=apf::loadMdsMesh(model, argv[1]);
   mesh->verify();
   Vortex sfv(mesh, center, modelLen);
-  ma::Input* in = ma::makeAdvanced(ma::configure(mesh,&sfv));
-  in->maximumIterations = 9;
+  const ma::Input* in = ma::configure(mesh,&sfv);
   ma::adapt(in);
   mesh->verify();
   apf::writeVtkFiles("adapted",mesh);

From 20fa0ba1584b566101ee84e25ae80c07048a76f6 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Sun, 4 Jul 2021 18:01:44 -0400
Subject: [PATCH 373/555] Removes advanced options from 2 more tests

---
 test/ma_test_analytic_model.cc | 5 +----
 test/snap.cc                   | 4 +---
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/test/ma_test_analytic_model.cc b/test/ma_test_analytic_model.cc
index 8422ca6b3..6d5d45635 100644
--- a/test/ma_test_analytic_model.cc
+++ b/test/ma_test_analytic_model.cc
@@ -179,10 +179,7 @@ int main(int argc, char** argv)
 
   apf::writeVtkFiles("initial_mesh_on_analytic_model", m);
 
-  ma::Input* in = ma::makeAdvanced(ma::configureUniformRefine(m, 2));
-  in->shouldSnap = true;
-  in->shouldTransferParametric = true;
-  in->shouldFixShape = true;
+  const ma::Input* in = ma::configureUniformRefine(m, 2);
   ma::adapt(in);
 
   apf::writeVtkFiles("adapted_mesh_on_analytic_model", m);
diff --git a/test/snap.cc b/test/snap.cc
index 7d633354b..6134d245d 100644
--- a/test/snap.cc
+++ b/test/snap.cc
@@ -21,9 +21,7 @@ int main(int argc, char** argv)
   gmi_sim_start();
   gmi_register_sim();
   ma::Mesh* m = apf::loadMdsMesh(argv[1],argv[2]);
-  ma::Input* in = ma::makeAdvanced(ma::configureIdentity(m));
-  in->shouldSnap = true;
-  in->shouldTransferParametric = true;
+  const ma::Input* in = ma::configureIdentity(m);
   ma::adapt(in);
   m->writeNative(argv[3]);
   m->destroyNative();

From 1db66d6e154a6b9e1304681838b368b7aa5eefdc Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Tue, 6 Jul 2021 19:21:06 +0000
Subject: [PATCH 374/555] support simmodsuite 16

---
 cmake/FindSimModSuite.cmake | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmake/FindSimModSuite.cmake b/cmake/FindSimModSuite.cmake
index 244ea4ca1..3785ead8a 100644
--- a/cmake/FindSimModSuite.cmake
+++ b/cmake/FindSimModSuite.cmake
@@ -84,7 +84,7 @@ string(REGEX REPLACE
   "${SIM_VERSION}")
 
 set(MIN_VALID_SIM_VERSION 12.0.190225)
-set(MAX_VALID_SIM_VERSION 15.0.210501)
+set(MAX_VALID_SIM_VERSION 16.0-210623)
 if( ${SKIP_SIMMETRIX_VERSION_CHECK} )
   message(STATUS "Skipping Simmetrix SimModSuite version check."
     " This may result in undefined behavior")
@@ -113,7 +113,7 @@ message(STATUS "SIM_ARCHOS ${SIM_ARCHOS}")
 option(SIM_PARASOLID "Use Parasolid through Simmetrix" OFF)
 if (SIM_PARASOLID)
   set(MIN_SIM_PARASOLID_VERSION 290)
-  set(MAX_SIM_PARASOLID_VERSION 310)
+  set(MAX_SIM_PARASOLID_VERSION 330)
   foreach(version RANGE
       ${MAX_SIM_PARASOLID_VERSION}
       ${MIN_SIM_PARASOLID_VERSION} -10)

From f59f065e42b4f8572b805be842d2fd26a5d887f1 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 7 Jul 2021 16:38:15 +0000
Subject: [PATCH 375/555] fix the simmodsuite version regex

the period is a literal not a wildcard
---
 cmake/FindSimModSuite.cmake | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/cmake/FindSimModSuite.cmake b/cmake/FindSimModSuite.cmake
index 3785ead8a..8be42efb2 100644
--- a/cmake/FindSimModSuite.cmake
+++ b/cmake/FindSimModSuite.cmake
@@ -65,21 +65,21 @@ string(REGEX REPLACE
   "${SIMMODSUITE_INCLUDE_DIR}")
 
 string(REGEX MATCH
-  "[0-9]+.[0-9]+-[0-9]+"
+  "[0-9]+[.][0-9]+-[0-9]+"
   SIM_VERSION
   "${SIMMODSUITE_INCLUDE_DIR}")
 
 #VERSION_LESS and VERSION_GREATER need '.' delimited version strings.
 string(REGEX REPLACE
-  "([0-9]+.[0-9]+)-([0-9]+)"
+  "([0-9]+[.][0-9]+)-([0-9]+)"
   "\\1.\\2" SIM_DOT_VERSION
   "${SIM_VERSION}")
 string(REGEX REPLACE
-  "([0-9]+).([0-9]+)-([0-9]+)"
+  "([0-9]+)[.]([0-9]+)-([0-9]+)"
   "\\1" SIMMODSUITE_MAJOR_VERSION
   "${SIM_VERSION}")
 string(REGEX REPLACE
-  "([0-9]+).([0-9]+)-([0-9]+)"
+  "([0-9]+)[.]([0-9]+)-([0-9]+)"
   "\\3" SIMMODSUITE_MINOR_VERSION
   "${SIM_VERSION}")
 

From 7a192c8fc4c3a9edfa4dc1105142dede846dc2ea Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 7 Jul 2021 19:19:55 +0000
Subject: [PATCH 376/555] simmodsuite 16 api change

EN_getMatchingEnts takes two args
---
 apf_sim/apfSIM.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/apf_sim/apfSIM.cc b/apf_sim/apfSIM.cc
index 5a8489959..a5b14a0d3 100644
--- a/apf_sim/apfSIM.cc
+++ b/apf_sim/apfSIM.cc
@@ -1042,7 +1042,8 @@ static bool findMatches(Mesh* m)
     while ((e = m->iterate(it)))
     {
       pEntity ent = reinterpret_cast<pEntity>(e);
-#if SIMMODSUITE_MAJOR_VERSION <= 14 && SIMMODSUITE_MINOR_VERSION < 190921
+#if (SIMMODSUITE_MAJOR_VERSION <= 14 && SIMMODSUITE_MINOR_VERSION < 190921) || \
+    (SIMMODSUITE_MAJOR_VERSION >= 16 && SIMMODSUITE_MINOR_VERSION >= 210623)
       pPList l = EN_getMatchingEnts(ent, NULL);
 #else
       pPList l = EN_getMatchingEnts(ent, NULL, 0);

From 11f56878f1264252c0cc7f7053dc306165fc180b Mon Sep 17 00:00:00 2001
From: Morteza HS <hakimi1364@gmail.com>
Date: Wed, 7 Jul 2021 16:28:14 -0400
Subject: [PATCH 377/555] Removes unnecessary casts + implements copy c-tor

---
 ma/maInput.cc | 55 +++++++++++++++++++++++++++++++++++++++++++--------
 ma/maInput.h  |  2 ++
 2 files changed, 49 insertions(+), 8 deletions(-)

diff --git a/ma/maInput.cc b/ma/maInput.cc
index 078fdd560..641b5f6f4 100644
--- a/ma/maInput.cc
+++ b/ma/maInput.cc
@@ -26,6 +26,45 @@ Input::~Input()
     delete solutionTransfer;
 }
 
+Input::Input(const Input& in)
+{
+  mesh = in.mesh;
+  sizeField = in.sizeField ;
+  ownsSizeField = in.ownsSizeField;
+  solutionTransfer = in.solutionTransfer;
+  ownsSolutionTransfer = in.ownsSolutionTransfer;
+  shapeHandler = in.shapeHandler;
+  maximumIterations = in.maximumIterations;
+  shouldCoarsen = in.shouldCoarsen;
+  shouldSnap = in.shouldSnap;
+  shouldTransferParametric = in.shouldTransferParametric;
+  shouldTransferToClosestPoint = in.shouldTransferToClosestPoint;
+  shouldHandleMatching = in.shouldHandleMatching;
+  shouldFixShape = in.shouldFixShape;
+  shouldForceAdaptation = in.shouldForceAdaptation;
+  shouldPrintQuality = in.shouldPrintQuality;
+  goodQuality = in.goodQuality;
+  shouldCheckQualityForDoubleSplits = in.shouldCheckQualityForDoubleSplits;
+  validQuality = in.validQuality;
+  maximumImbalance = in.maximumImbalance;
+  shouldRunPreZoltan = in.shouldRunPreZoltan;
+  shouldRunPreZoltanRib = in.shouldRunPreZoltanRib;
+  shouldRunPreParma = in.shouldRunPreParma;
+  shouldRunMidZoltan = in.shouldRunMidZoltan;
+  shouldRunMidParma = in.shouldRunMidParma;
+  shouldRunPostZoltan = in.shouldRunPostZoltan;
+  shouldRunPostZoltanRib = in.shouldRunPostZoltanRib;
+  shouldRunPostParma = in.shouldRunPostParma;
+  maximumEdgeRatio = in.maximumEdgeRatio;
+  shouldTurnLayerToTets = in.shouldTurnLayerToTets;
+  shouldCleanupLayer = in.shouldCleanupLayer;
+  shouldRefineLayer = in.shouldRefineLayer;
+  shouldCoarsenLayer = in.shouldCoarsenLayer;
+  splitAllLayerEdges = in.splitAllLayerEdges;
+  userDefinedLayerTagName = in.userDefinedLayerTagName;
+  debugFolder = in.debugFolder;
+}
+
 void setDefaultValues(Input* in)
 {
   in->ownsSizeField = true;
@@ -213,7 +252,7 @@ const Input* configure(
   Input* in = configure(m,s);
   in->sizeField = makeSizeField(m, f, logInterpolation);
   updateMaxIterBasedOnSize(m, in);
-  return (const Input*)in;
+  return in;
 }
 
 const Input* configure(
@@ -224,7 +263,7 @@ const Input* configure(
   Input* in = configure(m,s);
   in->sizeField = makeSizeField(m, f);
   updateMaxIterBasedOnSize(m, in);
-  return (const Input*)in;
+  return in;
 }
 
 const Input* configure(
@@ -235,7 +274,7 @@ const Input* configure(
   Input* in = configure(m,s);
   in->sizeField = makeSizeField(m, f);
   updateMaxIterBasedOnSize(m, in);
-  return (const Input*)in;
+  return in;
 }
 
 const Input* configure(
@@ -248,7 +287,7 @@ const Input* configure(
   Input* in = configure(m,s);
   in->sizeField = makeSizeField(m, sizes, frames, logInterpolation);
   updateMaxIterBasedOnSize(m, in);
-  return (const Input*)in;
+  return in;
 }
 
 const Input* configureUniformRefine(Mesh* m, int n, SolutionTransfer* s)
@@ -258,7 +297,7 @@ const Input* configureUniformRefine(Mesh* m, int n, SolutionTransfer* s)
   in->maximumIterations = n;
   in->shouldRefineLayer = true;
   in->splitAllLayerEdges = true;
-  return (const Input*)in;
+  return in;
 }
 
 const Input* configureMatching(Mesh* m, int n, SolutionTransfer* s)
@@ -266,7 +305,7 @@ const Input* configureMatching(Mesh* m, int n, SolutionTransfer* s)
   Input* in = makeAdvanced(configureUniformRefine(m,n,s));
   in->shouldHandleMatching = true;
   in->shouldFixShape = false;
-  return (const Input*)in;
+  return in;
 }
 
 const Input* configureIdentity(Mesh* m, SizeField* f, SolutionTransfer* s)
@@ -285,12 +324,12 @@ const Input* configureIdentity(Mesh* m, SizeField* f, SolutionTransfer* s)
   in->maximumIterations = 0;
   in->shouldFixShape = false;
   in->shouldSnap = false;
-  return (const Input*)in;
+  return in;
 }
 
 Input* makeAdvanced(const Input* in)
 {
-  return const_cast<Input*>(in);
+  return new Input(*in);
 }
 
 }
diff --git a/ma/maInput.h b/ma/maInput.h
index 51f12941a..9aa3a40e0 100644
--- a/ma/maInput.h
+++ b/ma/maInput.h
@@ -35,6 +35,8 @@ class Input
 {
   public:
     ~Input();
+    Input() {} // default empty c-tor
+    Input(const Input& in); // copy c-tor
     Mesh* mesh;
     SizeField* sizeField;
     bool ownsSizeField;

From 67749bb5f4c826c1203ec7f0fbe96103347cf043 Mon Sep 17 00:00:00 2001
From: Morteza HS <hakimi1364@gmail.com>
Date: Wed, 7 Jul 2021 19:28:05 -0400
Subject: [PATCH 378/555] Default copy c-tor, default-tor in Input

Takes care of clean up of sizefield and solutiontransfer objects at
the end of adapt instead of inside d-tor of ma::Input.
---
 ma/ma.cc      |  8 ++++++++
 ma/maInput.cc | 51 +++------------------------------------------------
 ma/maInput.h  |  4 ++--
 3 files changed, 13 insertions(+), 50 deletions(-)

diff --git a/ma/ma.cc b/ma/ma.cc
index 2b0ef6c21..8ba200588 100644
--- a/ma/ma.cc
+++ b/ma/ma.cc
@@ -45,6 +45,10 @@ void adapt(Input* in)
   postBalance(a);
   Mesh* m = a->mesh;
   delete a;
+  if (in->ownsSizeField)
+    delete in->sizeField;
+  if (in->ownsSolutionTransfer)
+    delete in->solutionTransfer;
   delete in;
   double t1 = PCU_Time();
   print("mesh adapted in %f seconds",t1-t0);
@@ -114,6 +118,10 @@ void adaptVerbose(Input* in, bool verbose)
   postBalance(a);
   Mesh* m = a->mesh;
   delete a;
+  if (in->ownsSizeField)
+    delete in->sizeField;
+  if (in->ownsSolutionTransfer)
+    delete in->solutionTransfer;
   delete in;
   double t1 = PCU_Time();
   print("mesh adapted in %f seconds",t1-t0);
diff --git a/ma/maInput.cc b/ma/maInput.cc
index 641b5f6f4..89d7401d8 100644
--- a/ma/maInput.cc
+++ b/ma/maInput.cc
@@ -18,53 +18,6 @@
 
 namespace ma {
 
-Input::~Input()
-{
-  if (ownsSizeField)
-    delete sizeField;
-  if (ownsSolutionTransfer)
-    delete solutionTransfer;
-}
-
-Input::Input(const Input& in)
-{
-  mesh = in.mesh;
-  sizeField = in.sizeField ;
-  ownsSizeField = in.ownsSizeField;
-  solutionTransfer = in.solutionTransfer;
-  ownsSolutionTransfer = in.ownsSolutionTransfer;
-  shapeHandler = in.shapeHandler;
-  maximumIterations = in.maximumIterations;
-  shouldCoarsen = in.shouldCoarsen;
-  shouldSnap = in.shouldSnap;
-  shouldTransferParametric = in.shouldTransferParametric;
-  shouldTransferToClosestPoint = in.shouldTransferToClosestPoint;
-  shouldHandleMatching = in.shouldHandleMatching;
-  shouldFixShape = in.shouldFixShape;
-  shouldForceAdaptation = in.shouldForceAdaptation;
-  shouldPrintQuality = in.shouldPrintQuality;
-  goodQuality = in.goodQuality;
-  shouldCheckQualityForDoubleSplits = in.shouldCheckQualityForDoubleSplits;
-  validQuality = in.validQuality;
-  maximumImbalance = in.maximumImbalance;
-  shouldRunPreZoltan = in.shouldRunPreZoltan;
-  shouldRunPreZoltanRib = in.shouldRunPreZoltanRib;
-  shouldRunPreParma = in.shouldRunPreParma;
-  shouldRunMidZoltan = in.shouldRunMidZoltan;
-  shouldRunMidParma = in.shouldRunMidParma;
-  shouldRunPostZoltan = in.shouldRunPostZoltan;
-  shouldRunPostZoltanRib = in.shouldRunPostZoltanRib;
-  shouldRunPostParma = in.shouldRunPostParma;
-  maximumEdgeRatio = in.maximumEdgeRatio;
-  shouldTurnLayerToTets = in.shouldTurnLayerToTets;
-  shouldCleanupLayer = in.shouldCleanupLayer;
-  shouldRefineLayer = in.shouldRefineLayer;
-  shouldCoarsenLayer = in.shouldCoarsenLayer;
-  splitAllLayerEdges = in.splitAllLayerEdges;
-  userDefinedLayerTagName = in.userDefinedLayerTagName;
-  debugFolder = in.debugFolder;
-}
-
 void setDefaultValues(Input* in)
 {
   in->ownsSizeField = true;
@@ -329,7 +282,9 @@ const Input* configureIdentity(Mesh* m, SizeField* f, SolutionTransfer* s)
 
 Input* makeAdvanced(const Input* in)
 {
-  return new Input(*in);
+  Input* in2 = new Input(*in);
+  delete in;
+  return in2;
 }
 
 }
diff --git a/ma/maInput.h b/ma/maInput.h
index 9aa3a40e0..6cf717109 100644
--- a/ma/maInput.h
+++ b/ma/maInput.h
@@ -34,9 +34,9 @@ typedef ShapeHandler* (*ShapeHandlerFunction)(Adapt* a);
 class Input
 {
   public:
-    ~Input();
+    ~Input() {}
     Input() {} // default empty c-tor
-    Input(const Input& in); // copy c-tor
+    Input(const Input& in) = default; // copy c-tor
     Mesh* mesh;
     SizeField* sizeField;
     bool ownsSizeField;

From 68bb848adc8179075b954ddf52418737dd1c9968 Mon Sep 17 00:00:00 2001
From: Morteza HS <hakimi1364@gmail.com>
Date: Wed, 7 Jul 2021 19:29:44 -0400
Subject: [PATCH 379/555] Updates crv adapt routines to follow the same logic
 as ma adapt

---
 crv/crv.h       |  5 +++++
 crv/crvAdapt.cc | 15 +++++++++++++++
 2 files changed, 20 insertions(+)

diff --git a/crv/crv.h b/crv/crv.h
index e31e8cf43..3a0e7f9e8 100644
--- a/crv/crv.h
+++ b/crv/crv.h
@@ -122,6 +122,11 @@ ma::Input* configureShapeCorrection(
   note that this function will delete the Input object */
 void adapt(ma::Input* in);
 
+/** \brief crv adapt with custom configuration
+  \details see maInput.h for details.
+  note that this function will delete the Input object */
+void adapt(const ma::Input* in);
+
 /** \brief crv stats to get statistic information about the mesh
   \details statistic considered are (1)final/desired edge-lengths
   (2) linear quality (3) curved quality (minJ/maxJ)
diff --git a/crv/crvAdapt.cc b/crv/crvAdapt.cc
index 7e33c9729..fb129a534 100644
--- a/crv/crvAdapt.cc
+++ b/crv/crvAdapt.cc
@@ -155,6 +155,11 @@ ma::Input* configureShapeCorrection(
     ma::SolutionTransfer* s)
 {
   ma::Input* in = ma::makeAdvanced(ma::configureIdentity(m,f,s));
+  if (!in)
+    printf("in is null\n");
+  else
+    printf("in is OK\n");
+
   in->shouldFixShape = true;
   in->shouldSnap = in->mesh->canSnap();
   in->shouldTransferParametric = in->mesh->canSnap();
@@ -235,9 +240,19 @@ void adapt(ma::Input* in)
   apf::printStats(a->mesh);
   crv::clearTags(a);
   delete a;
+  if (in->ownsSizeField)
+    delete in->sizeField;
+  if (in->ownsSolutionTransfer)
+    delete in->solutionTransfer;
   delete in;
 }
 
+
+void adapt(const ma::Input* in)
+{
+  crv::adapt(ma::makeAdvanced(in));
+}
+
 /** \brief Measures entity related quantities for a given mesh
   \details  quantities include normalized edge length, linear quality
   and curved quality. The values can be computed in both metric (if

From 009d351a4b5e64133e812d1a30a8eae87946df7f Mon Sep 17 00:00:00 2001
From: Morteza HS <hakimi1364@gmail.com>
Date: Wed, 7 Jul 2021 19:32:55 -0400
Subject: [PATCH 380/555] Removes debug print statements

---
 crv/crvAdapt.cc | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/crv/crvAdapt.cc b/crv/crvAdapt.cc
index fb129a534..de18b96a5 100644
--- a/crv/crvAdapt.cc
+++ b/crv/crvAdapt.cc
@@ -155,11 +155,6 @@ ma::Input* configureShapeCorrection(
     ma::SolutionTransfer* s)
 {
   ma::Input* in = ma::makeAdvanced(ma::configureIdentity(m,f,s));
-  if (!in)
-    printf("in is null\n");
-  else
-    printf("in is OK\n");
-
   in->shouldFixShape = true;
   in->shouldSnap = in->mesh->canSnap();
   in->shouldTransferParametric = in->mesh->canSnap();

From 4b72b46c0f550821393dc7956e662de5c27420be Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Sat, 10 Jul 2021 03:18:04 +0000
Subject: [PATCH 381/555] require cxx11

---
 CMakeLists.txt | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index efa6f89a0..77b3be616 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -11,10 +11,13 @@ project(SCOREC VERSION 2.2.6 LANGUAGES CXX C)
 include(cmake/bob.cmake)
 include(cmake/xsdk.cmake)
 
-option(SCOREC_ENABLE_CXX11 "enable compilation with c++11 support" NO)
-
 option(USE_XSDK_DEFAULTS "enable the XDSK v0.3.0 default configuration" NO)
 
+#requre c++11 without extensions
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSION "NO")
+set(CMAKE_CXX_STANDARD 11)
+
 xsdk_begin_package()
 bob_begin_package()
 
@@ -28,9 +31,6 @@ if(NOT USE_XSDK_DEFAULTS)
   bob_begin_cxx_flags()
   bob_end_cxx_flags()
   set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS}")
-  if(SCOREC_ENABLE_CXX11)
-    bob_cxx11_flags()
-  endif()
 endif()
 message(STATUS "CMAKE_CXX_FLAGS = ${CMAKE_CXX_FLAGS}")
 

From d7ddb0f72218ff25a0262fc76a0d5de4a829e8db Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Sat, 10 Jul 2021 03:29:35 +0000
Subject: [PATCH 382/555] use off instead of no

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 77b3be616..106787abb 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -15,7 +15,7 @@ option(USE_XSDK_DEFAULTS "enable the XDSK v0.3.0 default configuration" NO)
 
 #requre c++11 without extensions
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
-set(CMAKE_CXX_EXTENSION "NO")
+set(CMAKE_CXX_EXTENSION OFF)
 set(CMAKE_CXX_STANDARD 11)
 
 xsdk_begin_package()

From f38d5b65e7c613d1061313ea5f478888526dd07b Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Sat, 10 Jul 2021 04:58:05 +0000
Subject: [PATCH 383/555] export cxx11 requirement

core-target.cmake contains: INTERFACE_COMPILE_FEATURES cxx_std_11
---
 CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 106787abb..90ea84a72 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -147,6 +147,7 @@ add_subdirectory(omega_h)
 # this INTERFACE target bundles all the enabled libraries together
 add_library(core INTERFACE)
 target_link_libraries(core INTERFACE ${SCOREC_EXPORTED_TARGETS})
+target_compile_features(core INTERFACE cxx_std_11)
 scorec_export_library(core)
 
 #check for mallinfo2

From a18d47cef55d9a9d0f13f20d7c6a5b8e24a78daf Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Mon, 12 Jul 2021 16:21:56 -0400
Subject: [PATCH 384/555] github ci yaml

---
 .github/workflows/cmake.yml | 45 +++++++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)
 create mode 100644 .github/workflows/cmake.yml

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
new file mode 100644
index 000000000..b4051556d
--- /dev/null
+++ b/.github/workflows/cmake.yml
@@ -0,0 +1,45 @@
+name: CMake test matrix
+on:
+  push:
+    branches: [ develop ]
+  pull_request:
+    branches: [ develop ]
+jobs:
+  build:
+    # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        compiler: [g++, clang++]
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Create Build Environment
+      run: cmake -E make_directory ${{runner.workspace}}/build
+
+    - name: download meshes
+      working-directory: $GITHUB_WORKSPACE
+      run: git submodule init
+           git submodule update
+
+    - name: Configure CMake
+      shell: bash
+      working-directory: ${{runner.workspace}}/build
+      run: export MPICXX=${{ matrix.compiler }}
+           cmake $GITHUB_WORKSPACE
+           -DCMAKE_CXX_COMPILER=mpicxx
+           -DCMAKE_C_COMPILER=mpicc
+           -DCMAKE_VERBOSE_MAKEFILE=on
+           -DMESHES=${GITHUB_WORKSPACE}/pumi-meshes
+           -DBUILD_TESTING=ON
+
+    - name: Build
+      working-directory: ${{runner.workspace}}/build
+      shell: bash
+      run: cmake --build .
+
+    - name: Test
+      working-directory: ${{runner.workspace}}/build
+      shell: bash
+      run: ctest --output-on-failure

From 2891720151072dc8143927c3e38e2e196bcb058b Mon Sep 17 00:00:00 2001
From: Cameron Smith <cwsmith@users.noreply.github.com>
Date: Mon, 12 Jul 2021 16:31:51 -0400
Subject: [PATCH 385/555] attempt to download meshes

---
 .github/workflows/cmake.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index b4051556d..36fe3c5e0 100644
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -20,8 +20,7 @@ jobs:
 
     - name: download meshes
       working-directory: $GITHUB_WORKSPACE
-      run: git submodule init
-           git submodule update
+      run: git submodule init && git submodule update
 
     - name: Configure CMake
       shell: bash

From c674e7a5476abd46823fb18fc5d0d683240efecc Mon Sep 17 00:00:00 2001
From: Cameron Smith <cwsmith@users.noreply.github.com>
Date: Tue, 13 Jul 2021 00:23:25 -0400
Subject: [PATCH 386/555] check for pumi meshes

---
 .github/workflows/cmake.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index 36fe3c5e0..b59861611 100644
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -20,7 +20,7 @@ jobs:
 
     - name: download meshes
       working-directory: $GITHUB_WORKSPACE
-      run: git submodule init && git submodule update
+      run: ls pumi-meshes
 
     - name: Configure CMake
       shell: bash

From fe524ad1e95b97adf4a54a43e97dae6ef25630dd Mon Sep 17 00:00:00 2001
From: Cameron Smith <cwsmith@users.noreply.github.com>
Date: Tue, 13 Jul 2021 00:26:30 -0400
Subject: [PATCH 387/555] meshes...

---
 .github/workflows/cmake.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index b59861611..984331932 100644
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -19,7 +19,6 @@ jobs:
       run: cmake -E make_directory ${{runner.workspace}}/build
 
     - name: download meshes
-      working-directory: $GITHUB_WORKSPACE
       run: ls pumi-meshes
 
     - name: Configure CMake

From e964462e703818e13cc2fb7ca78be51f5edb6f9d Mon Sep 17 00:00:00 2001
From: Cameron Smith <cwsmith@users.noreply.github.com>
Date: Tue, 13 Jul 2021 00:29:42 -0400
Subject: [PATCH 388/555] submodule commands

---
 .github/workflows/cmake.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index 984331932..7fe693bf7 100644
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -19,7 +19,7 @@ jobs:
       run: cmake -E make_directory ${{runner.workspace}}/build
 
     - name: download meshes
-      run: ls pumi-meshes
+      run: git submodule init && git submodule update
 
     - name: Configure CMake
       shell: bash

From cda5f69ba888bb8fe2d697abf4f122bbce9d4031 Mon Sep 17 00:00:00 2001
From: Cameron Smith <cwsmith@users.noreply.github.com>
Date: Tue, 13 Jul 2021 00:34:03 -0400
Subject: [PATCH 389/555] fix export

---
 .github/workflows/cmake.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index 7fe693bf7..6f53cf410 100644
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -24,7 +24,7 @@ jobs:
     - name: Configure CMake
       shell: bash
       working-directory: ${{runner.workspace}}/build
-      run: export MPICXX=${{ matrix.compiler }}
+      run: export MPICXX=${{ matrix.compiler }} && 
            cmake $GITHUB_WORKSPACE
            -DCMAKE_CXX_COMPILER=mpicxx
            -DCMAKE_C_COMPILER=mpicc

From 3582e9ca1154d1ece66a7d8f7125354c8d426b20 Mon Sep 17 00:00:00 2001
From: Cameron Smith <cwsmith@users.noreply.github.com>
Date: Tue, 13 Jul 2021 00:41:41 -0400
Subject: [PATCH 390/555] install mpich

---
 .github/workflows/cmake.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index 6f53cf410..134262ce2 100644
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -14,6 +14,9 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
+    
+    - name: install mpich
+      run: sudo apt install mpich
 
     - name: Create Build Environment
       run: cmake -E make_directory ${{runner.workspace}}/build

From 7981c301646a8eed46ecd9a4e990920cc8ed0dc2 Mon Sep 17 00:00:00 2001
From: Cameron Smith <cwsmith@users.noreply.github.com>
Date: Tue, 13 Jul 2021 00:57:48 -0400
Subject: [PATCH 391/555] build tests

---
 .github/workflows/cmake.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index 134262ce2..ec3ced6d9 100644
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -33,7 +33,7 @@ jobs:
            -DCMAKE_C_COMPILER=mpicc
            -DCMAKE_VERBOSE_MAKEFILE=on
            -DMESHES=${GITHUB_WORKSPACE}/pumi-meshes
-           -DBUILD_TESTING=ON
+           -DIS_TESTING=ON
 
     - name: Build
       working-directory: ${{runner.workspace}}/build

From 844b7ec578fac394586ea4870c2197c56a1c16a1 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Tue, 13 Jul 2021 10:07:30 -0400
Subject: [PATCH 392/555] these tests require zoltan

---
 test/testing.cmake | 40 +++++++++++++++++++++-------------------
 1 file changed, 21 insertions(+), 19 deletions(-)

diff --git a/test/testing.cmake b/test/testing.cmake
index ca97fea61..d262ba71d 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -273,21 +273,21 @@ if(ENABLE_SIMMETRIX)
     "pipe_unif.smb"
     "pipe.smb")
 endif()
-mpi_test(ma_serial 1
-  ./ma_test
-  "${MDIR}/pipe.${GXT}"
-  "pipe.smb")
-mpi_test(aniso_ma_serial 1
-  ./aniso_ma_test
-  "${MESHES}/cube/cube.dmg"
-  "${MESHES}/cube/pumi670/cube.smb"
-  "0")
-mpi_test(aniso_ma_serial_log_interpolation 1
-  ./aniso_ma_test
-  "${MESHES}/cube/cube.dmg"
-  "${MESHES}/cube/pumi670/cube.smb"
-  "1")
 if(ENABLE_ZOLTAN)
+  mpi_test(ma_serial 1
+    ./ma_test
+    "${MDIR}/pipe.${GXT}"
+    "pipe.smb")
+  mpi_test(aniso_ma_serial 1
+    ./aniso_ma_test
+    "${MESHES}/cube/cube.dmg"
+    "${MESHES}/cube/pumi670/cube.smb"
+    "0")
+  mpi_test(aniso_ma_serial_log_interpolation 1
+    ./aniso_ma_test
+    "${MESHES}/cube/cube.dmg"
+    "${MESHES}/cube/pumi670/cube.smb"
+    "1")
   mpi_test(torus_ma_parallel 4
     ./torus_ma_test
     "${MESHES}/torus/torus.dmg"
@@ -623,11 +623,13 @@ if(ENABLE_SIMMETRIX)
       "${MDIR}/upright.smd"
       "67k")
     endif()
-    # adapt_meshgen uses the output of parallel_meshgen
-    mpi_test(adapt_meshgen 4
-      ./ma_test
-      "${MDIR}/upright.smd"
-      "67k/")
+    if(ENABLE_ZOLTAN)
+      # adapt_meshgen uses the output of parallel_meshgen
+      mpi_test(adapt_meshgen 4
+        ./ma_test
+        "${MDIR}/upright.smd"
+        "67k/")
+    endif()
   endif()
   if(SIM_PARASOLID)
     mpi_test(convert_para 1

From 01ca5ac18f08494a2220656a7b9eb850c8d30620 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Tue, 13 Jul 2021 11:08:48 -0400
Subject: [PATCH 393/555] install mpich and meshes once

---
 .github/workflows/cmake.yml | 31 ++++++++++++++++++-------------
 1 file changed, 18 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index ec3ced6d9..675326b5b 100644
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -5,29 +5,34 @@ on:
   pull_request:
     branches: [ develop ]
 jobs:
+  setup:
+    runs-on: ubuntu-18.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: download meshes
+      working-directory: $GITHUB_WORKSPACE
+      run: git submodule init
+           git submodule update
+    - name: install mpich
+      run: sudo apt install mpich
+
   build:
+    needs: setup
     # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix
     runs-on: ubuntu-18.04
     strategy:
+      fail-fast: false
       matrix:
         compiler: [g++, clang++]
 
     steps:
-    - uses: actions/checkout@v2
-    
-    - name: install mpich
-      run: sudo apt install mpich
-
     - name: Create Build Environment
-      run: cmake -E make_directory ${{runner.workspace}}/build
-
-    - name: download meshes
-      run: git submodule init && git submodule update
+      run: cmake -E make_directory ${{runner.workspace}}/build-${{ matrix.compiler }}
 
     - name: Configure CMake
       shell: bash
-      working-directory: ${{runner.workspace}}/build
-      run: export MPICXX=${{ matrix.compiler }} && 
+      working-directory: ${{runner.workspace}}/build-${{ matrix.compiler }}
+      run: export MPICXX=${{ matrix.compiler }} &&
            cmake $GITHUB_WORKSPACE
            -DCMAKE_CXX_COMPILER=mpicxx
            -DCMAKE_C_COMPILER=mpicc
@@ -36,11 +41,11 @@ jobs:
            -DIS_TESTING=ON
 
     - name: Build
-      working-directory: ${{runner.workspace}}/build
+      working-directory: ${{runner.workspace}}/build-${{ matrix.compiler }}
       shell: bash
       run: cmake --build .
 
     - name: Test
-      working-directory: ${{runner.workspace}}/build
+      working-directory: ${{runner.workspace}}/build-${{ matrix.compiler }}
       shell: bash
       run: ctest --output-on-failure

From d5d640d39437025fb4744a1ee8cff09d47e242c8 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Tue, 13 Jul 2021 11:11:52 -0400
Subject: [PATCH 394/555] fix mesh download

---
 .github/workflows/cmake.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index 675326b5b..95dec6ec6 100644
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -10,9 +10,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: download meshes
-      working-directory: $GITHUB_WORKSPACE
-      run: git submodule init
-           git submodule update
+      run: git submodule init && git submodule update
     - name: install mpich
       run: sudo apt install mpich
 

From d3af32693aa2fc25a3698f08b9fb6576911a456d Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Tue, 13 Jul 2021 11:15:59 -0400
Subject: [PATCH 395/555] debug

---
 .github/workflows/cmake.yml | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index 95dec6ec6..c47d42cb2 100644
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -10,7 +10,10 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: download meshes
-      run: git submodule init && git submodule update
+      run: git submodule init
+           git submodule update
+           ls -altr
+           pwd
     - name: install mpich
       run: sudo apt install mpich
 
@@ -24,8 +27,11 @@ jobs:
         compiler: [g++, clang++]
 
     steps:
+    - uses: actions/checkout@v2
     - name: Create Build Environment
       run: cmake -E make_directory ${{runner.workspace}}/build-${{ matrix.compiler }}
+           ls -altr
+           pwd
 
     - name: Configure CMake
       shell: bash

From dcdeee864a5e318625f074f51c1becbacb3794ef Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Tue, 13 Jul 2021 11:18:07 -0400
Subject: [PATCH 396/555] use pipe

---
 .github/workflows/cmake.yml | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index c47d42cb2..3a8a9d5dd 100644
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -10,10 +10,11 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: download meshes
-      run: git submodule init
-           git submodule update
-           ls -altr
-           pwd
+      run: |
+        git submodule init
+        git submodule update
+        ls -altr
+        pwd
     - name: install mpich
       run: sudo apt install mpich
 
@@ -29,9 +30,10 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: Create Build Environment
-      run: cmake -E make_directory ${{runner.workspace}}/build-${{ matrix.compiler }}
-           ls -altr
-           pwd
+      run: |
+        cmake -E make_directory ${{runner.workspace}}/build-${{ matrix.compiler }}
+        ls -altr
+        pwd
 
     - name: Configure CMake
       shell: bash

From ab317d8f85cd0769737b42f1eb652b8d0dcdfe08 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Tue, 13 Jul 2021 11:24:24 -0400
Subject: [PATCH 397/555] Revert "use pipe"

This reverts commit dcdeee864a5e318625f074f51c1becbacb3794ef.
---
 .github/workflows/cmake.yml | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index 3a8a9d5dd..c47d42cb2 100644
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -10,11 +10,10 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: download meshes
-      run: |
-        git submodule init
-        git submodule update
-        ls -altr
-        pwd
+      run: git submodule init
+           git submodule update
+           ls -altr
+           pwd
     - name: install mpich
       run: sudo apt install mpich
 
@@ -30,10 +29,9 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: Create Build Environment
-      run: |
-        cmake -E make_directory ${{runner.workspace}}/build-${{ matrix.compiler }}
-        ls -altr
-        pwd
+      run: cmake -E make_directory ${{runner.workspace}}/build-${{ matrix.compiler }}
+           ls -altr
+           pwd
 
     - name: Configure CMake
       shell: bash

From d3105bf9feb3e08983552610b329364e8b41a17d Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Tue, 13 Jul 2021 11:24:27 -0400
Subject: [PATCH 398/555] Revert "debug"

This reverts commit d3af32693aa2fc25a3698f08b9fb6576911a456d.
---
 .github/workflows/cmake.yml | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index c47d42cb2..95dec6ec6 100644
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -10,10 +10,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: download meshes
-      run: git submodule init
-           git submodule update
-           ls -altr
-           pwd
+      run: git submodule init && git submodule update
     - name: install mpich
       run: sudo apt install mpich
 
@@ -27,11 +24,8 @@ jobs:
         compiler: [g++, clang++]
 
     steps:
-    - uses: actions/checkout@v2
     - name: Create Build Environment
       run: cmake -E make_directory ${{runner.workspace}}/build-${{ matrix.compiler }}
-           ls -altr
-           pwd
 
     - name: Configure CMake
       shell: bash

From 189ad0fd519f38c94e9e2aa47de55af3c63b8583 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Tue, 13 Jul 2021 11:24:27 -0400
Subject: [PATCH 399/555] Revert "fix mesh download"

This reverts commit d5d640d39437025fb4744a1ee8cff09d47e242c8.
---
 .github/workflows/cmake.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index 95dec6ec6..675326b5b 100644
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -10,7 +10,9 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: download meshes
-      run: git submodule init && git submodule update
+      working-directory: $GITHUB_WORKSPACE
+      run: git submodule init
+           git submodule update
     - name: install mpich
       run: sudo apt install mpich
 

From 35f3d48bd9ebb4b27b71bb74a74f78ad782dd80c Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Tue, 13 Jul 2021 11:24:28 -0400
Subject: [PATCH 400/555] Revert "install mpich and meshes once"

This reverts commit 01ca5ac18f08494a2220656a7b9eb850c8d30620.
---
 .github/workflows/cmake.yml | 31 +++++++++++++------------------
 1 file changed, 13 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index 675326b5b..ec3ced6d9 100644
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -5,34 +5,29 @@ on:
   pull_request:
     branches: [ develop ]
 jobs:
-  setup:
-    runs-on: ubuntu-18.04
-    steps:
-    - uses: actions/checkout@v2
-    - name: download meshes
-      working-directory: $GITHUB_WORKSPACE
-      run: git submodule init
-           git submodule update
-    - name: install mpich
-      run: sudo apt install mpich
-
   build:
-    needs: setup
     # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix
     runs-on: ubuntu-18.04
     strategy:
-      fail-fast: false
       matrix:
         compiler: [g++, clang++]
 
     steps:
+    - uses: actions/checkout@v2
+    
+    - name: install mpich
+      run: sudo apt install mpich
+
     - name: Create Build Environment
-      run: cmake -E make_directory ${{runner.workspace}}/build-${{ matrix.compiler }}
+      run: cmake -E make_directory ${{runner.workspace}}/build
+
+    - name: download meshes
+      run: git submodule init && git submodule update
 
     - name: Configure CMake
       shell: bash
-      working-directory: ${{runner.workspace}}/build-${{ matrix.compiler }}
-      run: export MPICXX=${{ matrix.compiler }} &&
+      working-directory: ${{runner.workspace}}/build
+      run: export MPICXX=${{ matrix.compiler }} && 
            cmake $GITHUB_WORKSPACE
            -DCMAKE_CXX_COMPILER=mpicxx
            -DCMAKE_C_COMPILER=mpicc
@@ -41,11 +36,11 @@ jobs:
            -DIS_TESTING=ON
 
     - name: Build
-      working-directory: ${{runner.workspace}}/build-${{ matrix.compiler }}
+      working-directory: ${{runner.workspace}}/build
       shell: bash
       run: cmake --build .
 
     - name: Test
-      working-directory: ${{runner.workspace}}/build-${{ matrix.compiler }}
+      working-directory: ${{runner.workspace}}/build
       shell: bash
       run: ctest --output-on-failure

From 517153d96d3da3318235101afdb74f8a78104909 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Tue, 13 Jul 2021 11:27:31 -0400
Subject: [PATCH 401/555] remove travis yaml

---
 .travis.yml | 50 --------------------------------------------------
 1 file changed, 50 deletions(-)
 delete mode 100644 .travis.yml

diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 3a874a611..000000000
--- a/.travis.yml
+++ /dev/null
@@ -1,50 +0,0 @@
-dist: trusty
-language: cpp
-compiler: gcc
-sudo: true
-
-branches:
-  only:
-   - develop
-
-addons:
-  apt:
-    update: true
-    sources:
-      - sourceline: 'ppa:ubuntu-toolchain-r/test'
-    packages:
-      - gcc-7
-      - g++-7
-      - mpich
-      - libmpich-dev
-      - clang
-
-before_install:
-  - export DEVROOT=/home/travis/build
-  - test -n $CC  && unset CC
-  - test -n $CXX && unset CXX
-  - pwd
-  - export NP=`grep -c ^processor /proc/cpuinfo`
-  - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 60 --slave /usr/bin/g++ g++ /usr/bin/g++-7
-  - sudo update-alternatives --config gcc
-  - cd ${DEVROOT}
-  - wget https://cmake.org/files/v3.12/cmake-3.12.1.tar.gz
-  - tar xzf cmake-3.12.1.tar.gz
-  - cd ${DEVROOT}/cmake-3.12.1
-  - ./bootstrap
-  - make -j ${NP} && sudo make install
-  - which cmake
-  - cmake --version
-  - export PATH=/usr/local/bin:${PATH}
-
-script:
-  - cd ${DEVROOT}/SCOREC/core
-  - mkdir -p ${DEVROOT}/SCOREC/core/build
-  - cd build
-  - cmake .. -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_INSTALL_PREFIX=${DEVROOT}/install/core -DIS_TESTING=ON -DBUILD_EXES=ON -DCMAKE_BUILD_TYPE=Release -DMESHES=${DEVROOT}/SCOREC/core/pumi-meshes
-  - make -j ${NP}
-  - ctest --output-on-failure
-  - cmake .. -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_INSTALL_PREFIX=${DEVROOT}/install/core -DIS_TESTING=ON -DBUILD_EXES=ON -DCMAKE_BUILD_TYPE=Debug -DMESHES=${DEVROOT}/SCOREC/core/pumi-meshes
-  - make -j ${NP}
-  - ctest --output-on-failure
-

From 9790ed824cbbee939f18db7013a416691ba816e1 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Tue, 13 Jul 2021 12:52:48 -0400
Subject: [PATCH 402/555] stk: get_buckets api change

see #348
---
 stk/apfSTK.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stk/apfSTK.cc b/stk/apfSTK.cc
index d59b84788..d0dc5d853 100644
--- a/stk/apfSTK.cc
+++ b/stk/apfSTK.cc
@@ -514,7 +514,7 @@ class StkBridge
       stk::mesh::EntityRank rank = stk::topology::NODE_RANK;
       if (isQP)
         rank = stk::topology::ELEMENT_RANK;
-      bulkData->get_buckets(rank, overlapSelector, buckets);
+      buckets = bulkData->get_buckets(rank, overlapSelector);
       GlobalMap* globalIdsToEnts = &globalIdsToVerts;
       if (isQP)
         globalIdsToEnts = &globalIdsToElems;

From 38b421563c6b736543fb15b1e4af0d7397a914a2 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Wed, 14 Jul 2021 16:36:40 -0700
Subject: [PATCH 403/555] bisected the code to find where globalToVert get
 corrupted.  This code will perform VERY badly   and write a ton of stuff

---
 apf/apfConstruct.cc          | 60 +++++++++++++++++++++++++++++++++++-
 test/matchedNodeElmReader.cc |  1 +
 2 files changed, 60 insertions(+), 1 deletion(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index 7c4352809..7afe794e1 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -15,9 +15,13 @@ static void constructVerts(
 {
   ModelEntity* interior = m->findModelEntity(m->getDimension(), 0);
   int end = nelem * apf::Mesh::adjacentCount[etype][0];
+  int self2 = PCU_Comm_Self();
   for (int i = 0; i < end; ++i)
-    if ( ! result.count(conn[i]))
+    if ( ! result.count(conn[i])) {
       result[conn[i]] = m->createVert_(interior);
+      if(conn[i] < 0 )      
+        lion_eprint(1, "constructVerts building globalToVert: self=%d,gid=%ld,i=%d,nelem=%ld  \n",self2,conn[i],i,nelem);
+    }
 }
 
 static void constructElements(
@@ -71,6 +75,8 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
   int self2 = PCU_Comm_Self();
   APF_ITERATE(GlobalToVert, globalToVert, it) {
     Gid gid = it->first;  
+    if(gid < 0 )      
+        lion_eprint(1, "constructResidence cgTV1: self=%d,gid=%ld,ifirst=%ld  \n",self2,gid,ifirst);
     if(ifirst==0 || ifirst==13437400 ) {
         lion_eprint(1, "constructResidence: self=%d,gid=%ld,ifirst=%ld  \n",self2,gid,ifirst);
     }
@@ -82,6 +88,8 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
   ifirst=0;
   APF_ITERATE(GlobalToVert, globalToVert, it) {
     Gid gid = it->first;  
+    if(gid < 0 )      
+        lion_eprint(1, "constructResidence cgTV2: self=%d,gid=%ld,ifirst=%ld  \n",self2,gid,ifirst);
     if(ifirst==0 || ifirst==13437400 ) {
         lion_eprint(1, "constructResidence: self=%d,gid=%ld,ifirst=%ld,max=%ld  \n",self2,gid,ifirst,max);
     }
@@ -102,10 +110,17 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
   PCU_Comm_Begin();
   APF_ITERATE(GlobalToVert, globalToVert, it) {
     Gid gid = it->first;  
+    if(gid < 0 )      
+        lion_eprint(1, "constructResidence cgTV3: self=%d,gid=%ld \n",self2,gid);
     int tmpI=gid / quotient;
     int to = std::min(peers - 1,tmpI); 
     PCU_COMM_PACK(to, gid);
   }
+  APF_ITERATE(GlobalToVert, globalToVert, it) {
+    Gid gid = it->first;  
+    if(gid < 0 )      
+        lion_eprint(1, "constructResidence cgTV3.1: self=%d,gid=%ld \n",self2,gid);
+  }
   PCU_Comm_Send();
   Gid myOffset = self * quotient;
   /* brokers store all the part ids that sent messages
@@ -118,6 +133,11 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
     int tmpI=tmpL;
     tmpParts.at(tmpI).push_back(from);
   }
+  APF_ITERATE(GlobalToVert, globalToVert, it) {
+    Gid gid = it->first;  
+    if(gid < 0 )      
+        lion_eprint(1, "constructResidence cgTV3.2: self=%d,gid=%ld \n",self2,gid);
+  }
   /* for each global id, send all associated part ids
      to all associated parts */
   PCU_Comm_Begin();
@@ -137,6 +157,11 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
   /* receiving a global id and associated parts,
      lookup the vertex and classify it on the partition
      model entity for that set of parts */
+  APF_ITERATE(GlobalToVert, globalToVert, it) {
+    Gid gid = it->first;  
+    if(gid < 0 )      
+        lion_eprint(1, "constructResidence cgTV3.4: self=%d,gid=%ld \n",self2,gid);
+  }
   while (PCU_Comm_Receive()) {
     Gid gid;
     PCU_COMM_UNPACK(gid);
@@ -148,9 +173,19 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
       PCU_COMM_UNPACK(part);
       residence.insert(part);
     }
+    APF_ITERATE(GlobalToVert, globalToVert, it) {
+      Gid gid = it->first;  
+      if(gid < 0 )      
+          lion_eprint(1, "constructResidence cgTV3.45: self=%d,gid=%ld \n",self2,gid);
+    }
     MeshEntity* vert = globalToVert[gid];
     m->setResidence(vert, residence);
   }
+  APF_ITERATE(GlobalToVert, globalToVert, it) {
+    Gid gid = it->first;  
+    if(gid < 0 )      
+        lion_eprint(1, "constructResidence cgTV3.5: self=%d,gid=%ld \n",self2,gid);
+  }
 }
 
 /* given correct residence from the above algorithm,
@@ -159,9 +194,16 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
 static void constructRemotes(Mesh2* m, GlobalToVert& globalToVert)
 {
   int self = PCU_Comm_Self();
+  APF_ITERATE(GlobalToVert, globalToVert, it) {
+    Gid gid = it->first;  
+    if(gid < 0 )      
+        lion_eprint(1, "constructRemotes cgTV3.7: self=%d,gid=%ld \n",self,gid);
+  }
   PCU_Comm_Begin();
   APF_ITERATE(GlobalToVert, globalToVert, it) {
     Gid gid = it->first;
+    if(gid < 0 )      
+        lion_eprint(1, "constructRemotes cgTV4: self=%d,gid=%ld \n",self,gid);
     MeshEntity* vert = it->second;
     Parts residence;
     m->getResidence(vert, residence);
@@ -169,6 +211,7 @@ static void constructRemotes(Mesh2* m, GlobalToVert& globalToVert)
       if (*rit != self) {
         PCU_COMM_PACK(*rit, gid);
         PCU_COMM_PACK(*rit, vert);
+        PCU_Debug_Print("sending remotes: gid %ld to %d\n", gid, *rit);
       }
   }
   PCU_Comm_Send();
@@ -180,7 +223,10 @@ static void constructRemotes(Mesh2* m, GlobalToVert& globalToVert)
     int from = PCU_Comm_Sender();
     MeshEntity* vert = globalToVert[gid];
     m->addRemote(vert, from, remote);
+    PCU_Debug_Print("receive remotes: from %d gid %ld\n", from, gid);
   }
+  // who is not stuck?
+  lion_eprint(1, "%d done inside remotes \n",PCU_Comm_Self());
 }
 
 void construct(Mesh2* m, const Gid* conn, int nelem, int etype,
@@ -189,8 +235,20 @@ void construct(Mesh2* m, const Gid* conn, int nelem, int etype,
   constructVerts(m, conn, nelem, etype, globalToVert);
   constructElements(m, conn, nelem, etype, globalToVert);
   constructResidence(m, globalToVert);
+  int self = PCU_Comm_Self();
+  APF_ITERATE(GlobalToVert, globalToVert, it) {
+    Gid gid = it->first;  
+    if(gid < 0 )      
+        lion_eprint(1, "constructRemotes cgTV3.6: self=%d,gid=%ld \n",self,gid);
+  }
+  PCU_Barrier();
+  lion_eprint(1, "%d after residence \n",PCU_Comm_Self());
   constructRemotes(m, globalToVert);
+  PCU_Barrier();
+  lion_eprint(1, "%d after Remotes \n",PCU_Comm_Self());
   stitchMesh(m);
+  PCU_Barrier();
+  lion_eprint(1, "%d after stitch \n",PCU_Comm_Self());
   m->acceptChanges();
 }
 
diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 1356c7ddc..19f212d7d 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -802,6 +802,7 @@ int main(int argc, char** argv)
   gmi_model* model = apf::makeMdsBox(2,2,2,1,1,1,0);
   apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, m.dim, isMatched);
   apf::GlobalToVert outMap;
+  PCU_Debug_Open();
   apf::construct(mesh, m.elements, m.localNumElms, m.elementType, outMap);
   delete [] m.elements;
   apf::alignMdsRemotes(mesh);

From c77f9d9a19d38ee5f6486fc46f8a183871ba6063 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Wed, 14 Jul 2021 17:12:31 -0700
Subject: [PATCH 404/555] I guess I don't understand c++ scoping of variables
 because I would have thought that I was using gid in a nested scope but this
 fixes that apparently non-bug as the results are unchanged

---
 apf/apfConstruct.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index 7afe794e1..f718b1300 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -174,9 +174,9 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
       residence.insert(part);
     }
     APF_ITERATE(GlobalToVert, globalToVert, it) {
-      Gid gid = it->first;  
-      if(gid < 0 )      
-          lion_eprint(1, "constructResidence cgTV3.45: self=%d,gid=%ld \n",self2,gid);
+      Gid gid2 = it->first;  
+      if(gid2 < 0 )      
+          lion_eprint(1, "constructResidence cgTV3.45: self=%d,gid=%ld \n",self2,gid2);
     }
     MeshEntity* vert = globalToVert[gid];
     m->setResidence(vert, residence);

From 807ff6ac5cbfc20553d1e2db50a81127d2cd1911 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Thu, 15 Jul 2021 00:19:21 -0700
Subject: [PATCH 405/555] Sanitize only found an out of bounds access that I
 put in for multi-topology which I think I fixed and made far simpler here.

---
 apf/apfConstruct.cc | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index f718b1300..7cee6aa35 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -36,18 +36,18 @@ static void constructElements(
     Downward verts;
     int offset = i * nev;
     Gid vCur=conn[offset];
-    Gid vNext=-1;
     int uniqueVerts=1;
-    for (int j = 0; j < nev; ++j) {
-      if(irep ==0){
-        verts[j] = globalToVert[vCur]; // conn[j + offset]];
-        vNext=conn[j+1+offset];
-        if(vNext == vCur) irep=1; // this was last one
-        else vCur=vNext; // Keep going but set this to what it needs next
-        uniqueVerts=j;
+    verts[0]=globalToVert[vCur];
+    Gid vPrev=vCur;
+    for (int j = 1; j < nev; ++j) {
+      vCur=conn[j+offset];
+      if (vCur != vPrev) { // multi-topology determined by repeating last used vtx
+         verts[j] = globalToVert[vCur]; 
+         uniqueVerts++;
+         vPrev=vCur; // for next check
       } 
     }
-    uniqueVerts++;
+
     if(uniqueVerts==4) etypeL=apf::Mesh::TET;
     if(uniqueVerts==5) etypeL=apf::Mesh::PYRAMID;
     if(uniqueVerts==6) etypeL=apf::Mesh::PRISM;

From c35b099ea7eec8c87775bf2db082934e6f8f852e Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Thu, 15 Jul 2021 22:22:16 -0700
Subject: [PATCH 406/555] so many print statements you can't find the code

---
 apf/apfConstruct.cc | 68 +++++++++++++++++++++++++++++++++++----------
 1 file changed, 54 insertions(+), 14 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index 7cee6aa35..4e20e35ff 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -19,8 +19,9 @@ static void constructVerts(
   for (int i = 0; i < end; ++i)
     if ( ! result.count(conn[i])) {
       result[conn[i]] = m->createVert_(interior);
-      if(conn[i] < 0 )      
+      if(conn[i] < 0 || conn[i] > 4305368187 ) { 
         lion_eprint(1, "constructVerts building globalToVert: self=%d,gid=%ld,i=%d,nelem=%ld  \n",self2,conn[i],i,nelem);
+      }
     }
 }
 
@@ -75,8 +76,9 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
   int self2 = PCU_Comm_Self();
   APF_ITERATE(GlobalToVert, globalToVert, it) {
     Gid gid = it->first;  
-    if(gid < 0 )      
+    if(gid < 0 || gid > 4305368187 ) { 
         lion_eprint(1, "constructResidence cgTV1: self=%d,gid=%ld,ifirst=%ld  \n",self2,gid,ifirst);
+    }
     if(ifirst==0 || ifirst==13437400 ) {
         lion_eprint(1, "constructResidence: self=%d,gid=%ld,ifirst=%ld  \n",self2,gid,ifirst);
     }
@@ -88,8 +90,9 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
   ifirst=0;
   APF_ITERATE(GlobalToVert, globalToVert, it) {
     Gid gid = it->first;  
-    if(gid < 0 )      
+    if(gid < 0 || gid > max ){ 
         lion_eprint(1, "constructResidence cgTV2: self=%d,gid=%ld,ifirst=%ld  \n",self2,gid,ifirst);
+    }
     if(ifirst==0 || ifirst==13437400 ) {
         lion_eprint(1, "constructResidence: self=%d,gid=%ld,ifirst=%ld,max=%ld  \n",self2,gid,ifirst,max);
     }
@@ -97,12 +100,15 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
   }
   Gid total = max + 1;
   int peers = PCU_Comm_Peers();
-  int quotient = total / peers; // this seems to work as C++ is doing the math in 64 bit and the result is assumed not to overflow 32
-  int remainder = total % peers; // same
+  Gid quotientL = total / peers; 
+  int quotient = quotientL;
+  Gid remainderL = total % peers; 
+  int remainder = remainderL;
   int mySize = quotient;
   int self = PCU_Comm_Self();
   if (self == (peers - 1))
     mySize += remainder;
+  if (self == (peers - 1)) lion_eprint(1, "CR1 mysize=%d \n",mySize);
   typedef std::vector< std::vector<int> > TmpParts;
   TmpParts tmpParts(mySize);
   /* if we have a vertex, send its global id to the
@@ -110,19 +116,27 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
   PCU_Comm_Begin();
   APF_ITERATE(GlobalToVert, globalToVert, it) {
     Gid gid = it->first;  
-    if(gid < 0 )      
+    if(gid < 0 || gid > max ){ 
         lion_eprint(1, "constructResidence cgTV3: self=%d,gid=%ld \n",self2,gid);
-    int tmpI=gid / quotient;
+    }
+    if (self == (peers - 1)) 
+      lion_eprint(1, "CR2: quotient=%d,mySize=%d,gid=%ld \n",quotient,mySize,gid);
+    Gid tmpL=gid / quotient;
+    int tmpI=tmpL;
+    if (self == (peers - 1)) lion_eprint(1, "CR3: tmpI=%d,peers=%d,quotient=%d \n",tmpI,peers,quotient);
     int to = std::min(peers - 1,tmpI); 
+    if (self == (peers - 1)) lion_eprint(1, "CR4: to=%d,peers=%d,gid=%ld \n",to,peers,gid);
     PCU_COMM_PACK(to, gid);
   }
   APF_ITERATE(GlobalToVert, globalToVert, it) {
     Gid gid = it->first;  
-    if(gid < 0 )      
+    if(gid < 0 || gid > max ){ 
         lion_eprint(1, "constructResidence cgTV3.1: self=%d,gid=%ld \n",self2,gid);
+    }
   }
   PCU_Comm_Send();
   Gid myOffset = self * quotient;
+  if (self == (peers - 1)) lion_eprint(1, "CR5: self=%d,myOffset=%ld,quotient=%d \n",self,myOffset,quotient);
   /* brokers store all the part ids that sent messages
      for each global id */
   while (PCU_Comm_Receive()) {
@@ -130,22 +144,31 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
     PCU_COMM_UNPACK(gid);
     int from = PCU_Comm_Sender();
     Gid tmpL=gid - myOffset; // forcing 64 bit difference until we know it is safe
+    if (self == (peers - 1)) lion_eprint(1, "CR6: tmpL=%ld,myOffset=%ld,gid=%ld \n",tmpL,myOffset,gid);
     int tmpI=tmpL;
+    if (self == (peers - 1)) lion_eprint(1, "CR7: tmpI=%d \n",tmpI);
     tmpParts.at(tmpI).push_back(from);
   }
   APF_ITERATE(GlobalToVert, globalToVert, it) {
     Gid gid = it->first;  
-    if(gid < 0 )      
+    if(gid < 0 || gid > max ){ 
         lion_eprint(1, "constructResidence cgTV3.2: self=%d,gid=%ld \n",self2,gid);
+    }
   }
   /* for each global id, send all associated part ids
      to all associated parts */
   PCU_Comm_Begin();
+  if (self == (peers - 1)) lion_eprint(1, "CR8: mySize=%d \n", mySize);
   for (int i = 0; i < mySize; ++i) {
     std::vector<int>& parts = tmpParts[i];
     for (size_t j = 0; j < parts.size(); ++j) {
       int to = parts[j];
       Gid gid = i + myOffset;
+      if(gid < 0 || gid > max ) {
+        lion_eprint(1, "constructResidence cgTV3.3: self=%d,gid=%ld \n",self2,gid);
+        if (self == (peers - 1)) lion_eprint(1, "CR9: myOffset=%ld,i=%d \n", myOffset,i);
+
+  }
       int nparts = parts.size();
       PCU_COMM_PACK(to, gid);
       PCU_COMM_PACK(to, nparts);
@@ -159,10 +182,13 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
      model entity for that set of parts */
   APF_ITERATE(GlobalToVert, globalToVert, it) {
     Gid gid = it->first;  
-    if(gid < 0 )      
+    if(gid < 0 || gid > max ){      
         lion_eprint(1, "constructResidence cgTV3.4: self=%d,gid=%ld \n",self2,gid);
+    }
   }
+  int whichWhile=0;
   while (PCU_Comm_Receive()) {
+    whichWhile++;
     Gid gid;
     PCU_COMM_UNPACK(gid);
     int nparts;
@@ -171,20 +197,34 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
     for (int i = 0; i < nparts; ++i) {
       int part;
       PCU_COMM_UNPACK(part);
-      residence.insert(part);
+      //residence.insert(part); //one less suspect
     }
     APF_ITERATE(GlobalToVert, globalToVert, it) {
       Gid gid2 = it->first;  
-      if(gid2 < 0 )      
-          lion_eprint(1, "constructResidence cgTV3.45: self=%d,gid=%ld \n",self2,gid2);
+      if(gid2 < 0 || gid2 > max ){      
+          lion_eprint(1, "constructResidence cgTV3.45: self=%d,gid=%ld,whichWhile=%d \n",self2,gid2,whichWhile);
+      }
     }
     MeshEntity* vert = globalToVert[gid];
+    APF_ITERATE(GlobalToVert, globalToVert, it) {
+      Gid gid2 = it->first;  
+      if(gid2 < 0 || gid2 > max ){      
+          lion_eprint(1, "constructResidence cgTV3.46: self=%d,gid=%ld,whichWhile=%d \n",self2,gid2,whichWhile);
+      }
+    }
     m->setResidence(vert, residence);
+    APF_ITERATE(GlobalToVert, globalToVert, it) {
+      Gid gid2 = it->first;  
+      if(gid2 < 0 || gid2 > max ){      
+          lion_eprint(1, "constructResidence cgTV3.47: self=%d,gid=%ld,whichWhile=%d \n",self2,gid2,whichWhile);
+      }
+    }
   }
   APF_ITERATE(GlobalToVert, globalToVert, it) {
     Gid gid = it->first;  
-    if(gid < 0 )      
+    if(gid < 0 || gid > max ){ 
         lion_eprint(1, "constructResidence cgTV3.5: self=%d,gid=%ld \n",self2,gid);
+    }
   }
 }
 

From a63c95c4ff595812df31f5f47d365fd8ff121b76 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Fri, 16 Jul 2021 07:16:58 -0700
Subject: [PATCH 407/555] one more lurking overflow in myOffset

---
 apf/apfConstruct.cc  | 22 +++++++++++++++-------
 apf/apfConvertTags.h |  2 +-
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index 4e20e35ff..c9dc5f73a 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -114,28 +114,36 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
   /* if we have a vertex, send its global id to the
      broker for that global id */
   PCU_Comm_Begin();
+  int sometimes=0;
   APF_ITERATE(GlobalToVert, globalToVert, it) {
     Gid gid = it->first;  
     if(gid < 0 || gid > max ){ 
         lion_eprint(1, "constructResidence cgTV3: self=%d,gid=%ld \n",self2,gid);
     }
-    if (self == (peers - 1)) 
-      lion_eprint(1, "CR2: quotient=%d,mySize=%d,gid=%ld \n",quotient,mySize,gid);
+    sometimes++;
+    if (self == (peers - 1)&& (sometimes%10000==1)) 
+      lion_eprint(1, "CR2: quotient=%d,mySize=%d,gid=%ld,itcount=%d \n",quotient,mySize,gid,sometimes);
     Gid tmpL=gid / quotient;
     int tmpI=tmpL;
-    if (self == (peers - 1)) lion_eprint(1, "CR3: tmpI=%d,peers=%d,quotient=%d \n",tmpI,peers,quotient);
+    if (self == (peers - 1)&& (sometimes%10000==1)) lion_eprint(1, "CR3: tmpI=%d,peers=%d,quotient=%d,itcount=%d \n",tmpI,peers,quotient,sometimes);
     int to = std::min(peers - 1,tmpI); 
-    if (self == (peers - 1)) lion_eprint(1, "CR4: to=%d,peers=%d,gid=%ld \n",to,peers,gid);
+    if (self == (peers - 1)&& (quotient<0)) lion_eprint(1, "CR3.5: tmpI=%d,peers=%d,quotient=%d,itcount=%d \n",tmpI,peers,quotient,sometimes);
+    if (self == (peers - 1)&& (sometimes%10000==1)) lion_eprint(1, "CR4: to=%d,peers=%d,gid=%ld,itCount=%d \n",to,peers,gid,sometimes);
     PCU_COMM_PACK(to, gid);
+    if (self == (peers - 1)&& (quotient<0)) lion_eprint(1, "CR4.125: tmpI=%d,peers=%d,quotient=%d,itcount=%d \n",tmpI,peers,quotient,sometimes);
   }
+  Gid myOffset3 = self * quotientL;
+  if (self == (peers - 1)) lion_eprint(1, "CR4.25: self=%d,myOffset=%ld,quotient=%ld \n",self,myOffset3,quotientL);
   APF_ITERATE(GlobalToVert, globalToVert, it) {
     Gid gid = it->first;  
     if(gid < 0 || gid > max ){ 
         lion_eprint(1, "constructResidence cgTV3.1: self=%d,gid=%ld \n",self2,gid);
     }
   }
+  Gid myOffset2 = self * quotientL;
+  if (self == (peers - 1)) lion_eprint(1, "CR4.5: self=%d,myOffset=%ld,quotient=%ld \n",self,myOffset2,quotientL);
   PCU_Comm_Send();
-  Gid myOffset = self * quotient;
+  Gid myOffset = (long)self * quotient;
   if (self == (peers - 1)) lion_eprint(1, "CR5: self=%d,myOffset=%ld,quotient=%d \n",self,myOffset,quotient);
   /* brokers store all the part ids that sent messages
      for each global id */
@@ -304,7 +312,7 @@ void setCoords(Mesh2* m, const double* coords, int nverts,
   int self = PCU_Comm_Self();
   if (self == (peers - 1))
     mySize += remainder;
-  Gid myOffset = self * quotient;
+  Gid myOffset = (long)self * quotient;
 
   /* Force each peer to have exactly mySize verts.
      This means we might need to send and recv some coords */
@@ -405,7 +413,7 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts,
   int self = PCU_Comm_Self();
   if (self == (peers - 1))
     mySize += remainder;
-  Gid myOffset = self * quotient;
+  Gid myOffset = (long)self * quotient;
 
   /* Force each peer to have exactly mySize verts.
      This means we might need to send and recv some matches */
diff --git a/apf/apfConvertTags.h b/apf/apfConvertTags.h
index 020916fa5..3966d3f4d 100644
--- a/apf/apfConvertTags.h
+++ b/apf/apfConvertTags.h
@@ -81,7 +81,7 @@ apf::MeshTag* setMappedTag(Mesh2* m, const char* tagName,
   int self = PCU_Comm_Self();
   if (self == (peers - 1))
     mySize += remainder;
-  apf::Gid myOffset = self * quotient;
+  apf::Gid myOffset = (long)self * quotient;
 
   /* Force each peer to have exactly mySize verts.
      This means we might need to send and recv some coords */

From 7be9789e238bc76d6737e22e40106a4b858ba55a Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Fri, 16 Jul 2021 08:46:23 -0700
Subject: [PATCH 408/555] innocent suspect released

---
 apf/apfConstruct.cc | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index c9dc5f73a..bdacd0919 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -152,9 +152,7 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
     PCU_COMM_UNPACK(gid);
     int from = PCU_Comm_Sender();
     Gid tmpL=gid - myOffset; // forcing 64 bit difference until we know it is safe
-    if (self == (peers - 1)) lion_eprint(1, "CR6: tmpL=%ld,myOffset=%ld,gid=%ld \n",tmpL,myOffset,gid);
     int tmpI=tmpL;
-    if (self == (peers - 1)) lion_eprint(1, "CR7: tmpI=%d \n",tmpI);
     tmpParts.at(tmpI).push_back(from);
   }
   APF_ITERATE(GlobalToVert, globalToVert, it) {
@@ -205,7 +203,7 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
     for (int i = 0; i < nparts; ++i) {
       int part;
       PCU_COMM_UNPACK(part);
-      //residence.insert(part); //one less suspect
+      residence.insert(part); //one less suspect
     }
     APF_ITERATE(GlobalToVert, globalToVert, it) {
       Gid gid2 = it->first;  

From 584318821d2826406d4f83a94bad5575ad1c672e Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Fri, 16 Jul 2021 10:01:55 -0600
Subject: [PATCH 409/555] cleared out the performance impacting debug
 statements

---
 apf/apfConstruct.cc | 93 +++------------------------------------------
 1 file changed, 5 insertions(+), 88 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index bdacd0919..a7375f585 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -19,7 +19,8 @@ static void constructVerts(
   for (int i = 0; i < end; ++i)
     if ( ! result.count(conn[i])) {
       result[conn[i]] = m->createVert_(interior);
-      if(conn[i] < 0 || conn[i] > 4305368187 ) { 
+//      if(conn[i] < 0 || conn[i] > 4305368187 ) { 
+      if(conn[i] < 0 ) { 
         lion_eprint(1, "constructVerts building globalToVert: self=%d,gid=%ld,i=%d,nelem=%ld  \n",self2,conn[i],i,nelem);
       }
     }
@@ -76,9 +77,6 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
   int self2 = PCU_Comm_Self();
   APF_ITERATE(GlobalToVert, globalToVert, it) {
     Gid gid = it->first;  
-    if(gid < 0 || gid > 4305368187 ) { 
-        lion_eprint(1, "constructResidence cgTV1: self=%d,gid=%ld,ifirst=%ld  \n",self2,gid,ifirst);
-    }
     if(ifirst==0 || ifirst==13437400 ) {
         lion_eprint(1, "constructResidence: self=%d,gid=%ld,ifirst=%ld  \n",self2,gid,ifirst);
     }
@@ -114,34 +112,15 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
   /* if we have a vertex, send its global id to the
      broker for that global id */
   PCU_Comm_Begin();
-  int sometimes=0;
   APF_ITERATE(GlobalToVert, globalToVert, it) {
     Gid gid = it->first;  
     if(gid < 0 || gid > max ){ 
         lion_eprint(1, "constructResidence cgTV3: self=%d,gid=%ld \n",self2,gid);
-    }
-    sometimes++;
-    if (self == (peers - 1)&& (sometimes%10000==1)) 
-      lion_eprint(1, "CR2: quotient=%d,mySize=%d,gid=%ld,itcount=%d \n",quotient,mySize,gid,sometimes);
+    } 
     Gid tmpL=gid / quotient;
-    int tmpI=tmpL;
-    if (self == (peers - 1)&& (sometimes%10000==1)) lion_eprint(1, "CR3: tmpI=%d,peers=%d,quotient=%d,itcount=%d \n",tmpI,peers,quotient,sometimes);
-    int to = std::min(peers - 1,tmpI); 
-    if (self == (peers - 1)&& (quotient<0)) lion_eprint(1, "CR3.5: tmpI=%d,peers=%d,quotient=%d,itcount=%d \n",tmpI,peers,quotient,sometimes);
-    if (self == (peers - 1)&& (sometimes%10000==1)) lion_eprint(1, "CR4: to=%d,peers=%d,gid=%ld,itCount=%d \n",to,peers,gid,sometimes);
+    int tmpI=tmpL;    int to = std::min(peers - 1,tmpI); 
     PCU_COMM_PACK(to, gid);
-    if (self == (peers - 1)&& (quotient<0)) lion_eprint(1, "CR4.125: tmpI=%d,peers=%d,quotient=%d,itcount=%d \n",tmpI,peers,quotient,sometimes);
   }
-  Gid myOffset3 = self * quotientL;
-  if (self == (peers - 1)) lion_eprint(1, "CR4.25: self=%d,myOffset=%ld,quotient=%ld \n",self,myOffset3,quotientL);
-  APF_ITERATE(GlobalToVert, globalToVert, it) {
-    Gid gid = it->first;  
-    if(gid < 0 || gid > max ){ 
-        lion_eprint(1, "constructResidence cgTV3.1: self=%d,gid=%ld \n",self2,gid);
-    }
-  }
-  Gid myOffset2 = self * quotientL;
-  if (self == (peers - 1)) lion_eprint(1, "CR4.5: self=%d,myOffset=%ld,quotient=%ld \n",self,myOffset2,quotientL);
   PCU_Comm_Send();
   Gid myOffset = (long)self * quotient;
   if (self == (peers - 1)) lion_eprint(1, "CR5: self=%d,myOffset=%ld,quotient=%d \n",self,myOffset,quotient);
@@ -155,26 +134,14 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
     int tmpI=tmpL;
     tmpParts.at(tmpI).push_back(from);
   }
-  APF_ITERATE(GlobalToVert, globalToVert, it) {
-    Gid gid = it->first;  
-    if(gid < 0 || gid > max ){ 
-        lion_eprint(1, "constructResidence cgTV3.2: self=%d,gid=%ld \n",self2,gid);
-    }
-  }
   /* for each global id, send all associated part ids
      to all associated parts */
   PCU_Comm_Begin();
-  if (self == (peers - 1)) lion_eprint(1, "CR8: mySize=%d \n", mySize);
   for (int i = 0; i < mySize; ++i) {
     std::vector<int>& parts = tmpParts[i];
     for (size_t j = 0; j < parts.size(); ++j) {
       int to = parts[j];
       Gid gid = i + myOffset;
-      if(gid < 0 || gid > max ) {
-        lion_eprint(1, "constructResidence cgTV3.3: self=%d,gid=%ld \n",self2,gid);
-        if (self == (peers - 1)) lion_eprint(1, "CR9: myOffset=%ld,i=%d \n", myOffset,i);
-
-  }
       int nparts = parts.size();
       PCU_COMM_PACK(to, gid);
       PCU_COMM_PACK(to, nparts);
@@ -186,15 +153,7 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
   /* receiving a global id and associated parts,
      lookup the vertex and classify it on the partition
      model entity for that set of parts */
-  APF_ITERATE(GlobalToVert, globalToVert, it) {
-    Gid gid = it->first;  
-    if(gid < 0 || gid > max ){      
-        lion_eprint(1, "constructResidence cgTV3.4: self=%d,gid=%ld \n",self2,gid);
-    }
-  }
-  int whichWhile=0;
   while (PCU_Comm_Receive()) {
-    whichWhile++;
     Gid gid;
     PCU_COMM_UNPACK(gid);
     int nparts;
@@ -203,34 +162,10 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
     for (int i = 0; i < nparts; ++i) {
       int part;
       PCU_COMM_UNPACK(part);
-      residence.insert(part); //one less suspect
-    }
-    APF_ITERATE(GlobalToVert, globalToVert, it) {
-      Gid gid2 = it->first;  
-      if(gid2 < 0 || gid2 > max ){      
-          lion_eprint(1, "constructResidence cgTV3.45: self=%d,gid=%ld,whichWhile=%d \n",self2,gid2,whichWhile);
-      }
+      residence.insert(part);
     }
     MeshEntity* vert = globalToVert[gid];
-    APF_ITERATE(GlobalToVert, globalToVert, it) {
-      Gid gid2 = it->first;  
-      if(gid2 < 0 || gid2 > max ){      
-          lion_eprint(1, "constructResidence cgTV3.46: self=%d,gid=%ld,whichWhile=%d \n",self2,gid2,whichWhile);
-      }
-    }
     m->setResidence(vert, residence);
-    APF_ITERATE(GlobalToVert, globalToVert, it) {
-      Gid gid2 = it->first;  
-      if(gid2 < 0 || gid2 > max ){      
-          lion_eprint(1, "constructResidence cgTV3.47: self=%d,gid=%ld,whichWhile=%d \n",self2,gid2,whichWhile);
-      }
-    }
-  }
-  APF_ITERATE(GlobalToVert, globalToVert, it) {
-    Gid gid = it->first;  
-    if(gid < 0 || gid > max ){ 
-        lion_eprint(1, "constructResidence cgTV3.5: self=%d,gid=%ld \n",self2,gid);
-    }
   }
 }
 
@@ -240,11 +175,6 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
 static void constructRemotes(Mesh2* m, GlobalToVert& globalToVert)
 {
   int self = PCU_Comm_Self();
-  APF_ITERATE(GlobalToVert, globalToVert, it) {
-    Gid gid = it->first;  
-    if(gid < 0 )      
-        lion_eprint(1, "constructRemotes cgTV3.7: self=%d,gid=%ld \n",self,gid);
-  }
   PCU_Comm_Begin();
   APF_ITERATE(GlobalToVert, globalToVert, it) {
     Gid gid = it->first;
@@ -257,7 +187,6 @@ static void constructRemotes(Mesh2* m, GlobalToVert& globalToVert)
       if (*rit != self) {
         PCU_COMM_PACK(*rit, gid);
         PCU_COMM_PACK(*rit, vert);
-        PCU_Debug_Print("sending remotes: gid %ld to %d\n", gid, *rit);
       }
   }
   PCU_Comm_Send();
@@ -269,7 +198,6 @@ static void constructRemotes(Mesh2* m, GlobalToVert& globalToVert)
     int from = PCU_Comm_Sender();
     MeshEntity* vert = globalToVert[gid];
     m->addRemote(vert, from, remote);
-    PCU_Debug_Print("receive remotes: from %d gid %ld\n", from, gid);
   }
   // who is not stuck?
   lion_eprint(1, "%d done inside remotes \n",PCU_Comm_Self());
@@ -281,20 +209,9 @@ void construct(Mesh2* m, const Gid* conn, int nelem, int etype,
   constructVerts(m, conn, nelem, etype, globalToVert);
   constructElements(m, conn, nelem, etype, globalToVert);
   constructResidence(m, globalToVert);
-  int self = PCU_Comm_Self();
-  APF_ITERATE(GlobalToVert, globalToVert, it) {
-    Gid gid = it->first;  
-    if(gid < 0 )      
-        lion_eprint(1, "constructRemotes cgTV3.6: self=%d,gid=%ld \n",self,gid);
-  }
-  PCU_Barrier();
   lion_eprint(1, "%d after residence \n",PCU_Comm_Self());
   constructRemotes(m, globalToVert);
-  PCU_Barrier();
-  lion_eprint(1, "%d after Remotes \n",PCU_Comm_Self());
   stitchMesh(m);
-  PCU_Barrier();
-  lion_eprint(1, "%d after stitch \n",PCU_Comm_Self());
   m->acceptChanges();
 }
 

From 0fae2ca702b7fc0d6d4a960d1face1e6634bc4b9 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Fri, 16 Jul 2021 17:55:24 -0700
Subject: [PATCH 410/555] but wait...there's more

---
 apf/apfConstruct.cc  | 4 ++--
 apf/apfConvertTags.h | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index a7375f585..ddba3c164 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -239,7 +239,7 @@ void setCoords(Mesh2* m, const double* coords, int nverts,
   Gid tmpL=start / quotient; 
   int tmpInt=tmpL;
   int to = std::min(peers - 1, tmpInt);
-  tmpL=(to+1)*quotient-start; 
+  tmpL=(to+1)*(long)quotient-start; 
   tmpInt=tmpL;
   int n = std::min(tmpInt, nverts);
   if(n > 1000) {
@@ -342,7 +342,7 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts,
   Gid tmpL=start / quotient; 
   int tmpInt=tmpL;
   int to = std::min(peers - 1, tmpInt);
-  tmpL=(to+1)*quotient-start; 
+  tmpL=(to+1)*(long)quotient-start; 
   tmpInt=tmpL;
   int n = std::min(tmpInt, nverts);
   while (nverts > 0) {
diff --git a/apf/apfConvertTags.h b/apf/apfConvertTags.h
index 3966d3f4d..069caa43b 100644
--- a/apf/apfConvertTags.h
+++ b/apf/apfConvertTags.h
@@ -93,7 +93,7 @@ apf::MeshTag* setMappedTag(Mesh2* m, const char* tagName,
   apf::Gid tmpL=start / quotient;
   int tmpI=tmpL;
   int to = std::min(peers - 1, tmpI);
-  tmpL=(to+1)*quotient-start;
+  tmpL=(to+1)*(long)quotient-start;
   tmpI=tmpL;
   int n = std::min(tmpI, nverts);
   while (nverts > 0) {

From 3dac5e29c1b8eddde06c8a37932fa958926c9e5b Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Fri, 16 Jul 2021 19:01:32 -0600
Subject: [PATCH 411/555] bring back debug print since verify claims there are
 remotes that overflow

---
 apf/apfConstruct.cc | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index ddba3c164..75e5a20a7 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -19,7 +19,7 @@ static void constructVerts(
   for (int i = 0; i < end; ++i)
     if ( ! result.count(conn[i])) {
       result[conn[i]] = m->createVert_(interior);
-//      if(conn[i] < 0 || conn[i] > 4305368187 ) { 
+//      if(conn[i] < 0 || conn[i] > 4305368187 ) { // for whatever reason max is not stored but is found and checked later
       if(conn[i] < 0 ) { 
         lion_eprint(1, "constructVerts building globalToVert: self=%d,gid=%ld,i=%d,nelem=%ld  \n",self2,conn[i],i,nelem);
       }
@@ -187,6 +187,7 @@ static void constructRemotes(Mesh2* m, GlobalToVert& globalToVert)
       if (*rit != self) {
         PCU_COMM_PACK(*rit, gid);
         PCU_COMM_PACK(*rit, vert);
+        PCU_Debug_Print("sending remotes: gid %ld to %d\n", gid, *rit);
       }
   }
   PCU_Comm_Send();
@@ -198,6 +199,7 @@ static void constructRemotes(Mesh2* m, GlobalToVert& globalToVert)
     int from = PCU_Comm_Sender();
     MeshEntity* vert = globalToVert[gid];
     m->addRemote(vert, from, remote);
+    PCU_Debug_Print("receive remotes: from %d gid %ld\n", from, gid);
   }
   // who is not stuck?
   lion_eprint(1, "%d done inside remotes \n",PCU_Comm_Self());
@@ -211,7 +213,11 @@ void construct(Mesh2* m, const Gid* conn, int nelem, int etype,
   constructResidence(m, globalToVert);
   lion_eprint(1, "%d after residence \n",PCU_Comm_Self());
   constructRemotes(m, globalToVert);
+  PCU_Barrier();
+  lion_eprint(1, "%d after Remotes \n",PCU_Comm_Self());
   stitchMesh(m);
+  PCU_Barrier();
+  lion_eprint(1, "%d after stitch \n",PCU_Comm_Self());
   m->acceptChanges();
 }
 

From 79f384c3e2530006ebe3378b4c1adfcba8b2acf0 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Sat, 17 Jul 2021 13:28:39 -0700
Subject: [PATCH 412/555] sometimes the error message is telling the
 truth...since Verify (and align remotes) were packing edge and face data that
 was not only  shared with peers BUT ALSO to SELF, the more numerous edges and
 faces combined into a single loop DOES overflow INT_MAX.  This commit breaks
 those messages into two rounds of communication.  Also adds the option to
 shift verify  until AFTER the mesh is written  with a command line argument. 
 Also sets ability  to not have fathers (like not having matches) by typing
 NULL for the filename.

---
 apf/apfVerify.cc             |  8 ++++----
 mds/mds_apf.c                | 15 +++++++++------
 test/matchedNodeElmReader.cc | 30 ++++++++++++++++++++----------
 3 files changed, 33 insertions(+), 20 deletions(-)

diff --git a/apf/apfVerify.cc b/apf/apfVerify.cc
index b4fa5da3d..9bb785611 100644
--- a/apf/apfVerify.cc
+++ b/apf/apfVerify.cc
@@ -281,19 +281,19 @@ static void receiveAllCopies(Mesh* m)
 
 static void verifyRemoteCopies(Mesh* m)
 {
-  PCU_Comm_Begin();
   for (int d = 0; d <= m->getDimension(); ++d)
   {
+    PCU_Comm_Begin();
     MeshIterator* it = m->begin(d);
     MeshEntity* e;
     while ((e = m->iterate(it)))
       if (m->isShared(e) && !m->isGhost(e))
         sendAllCopies(m, e);
     m->end(it);
+    PCU_Comm_Send();
+    while (PCU_Comm_Receive())
+     receiveAllCopies(m);
   }
-  PCU_Comm_Send();
-  while (PCU_Comm_Receive())
-    receiveAllCopies(m);
 }
 
 // ghost verification
diff --git a/mds/mds_apf.c b/mds/mds_apf.c
index 0efe5ff35..19166c070 100644
--- a/mds/mds_apf.c
+++ b/mds/mds_apf.c
@@ -269,8 +269,10 @@ static int align_copies(struct mds_net* net, struct mds* m)
   mds_id e;
   struct mds_copies* c;
   int did_change = 0;
-  PCU_Comm_Begin();
-  for (d = 1; d < m->d; ++d)
+//  long countAll=0;
+//  long countDTC=0;
+  for (d = 1; d < m->d; ++d){
+    PCU_Comm_Begin();
     for (e = mds_begin(m, d); e != MDS_NONE; e = mds_next(m, e)) {
       c = mds_get_copies(net, e);
       if (!c)
@@ -278,10 +280,11 @@ static int align_copies(struct mds_net* net, struct mds* m)
       if (owns_copies(e, c))
         downs_to_copies(m, e, c);
     }
-  PCU_Comm_Send();
-  while (PCU_Comm_Receive())
-    if (recv_down_copies(net, m))
-      did_change = 1;
+    PCU_Comm_Send();
+    while (PCU_Comm_Receive())
+      if (recv_down_copies(net, m))
+        did_change = 1;
+  }
   return PCU_Or(did_change);
 }
 
diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 19f212d7d..dd66bfe0a 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -736,12 +736,14 @@ void readMesh(const char* meshfilename,
   readClassification(ff, mesh.localNumVerts, &(mesh.classification));
   fclose(ff);
 
+  if( strcmp(fathers2Dfilename, "NULL") ) {
   //add an argument to readMesh for the fathers2D
-  sprintf(filename, "%s.%d",fathers2Dfilename,self);
-  FILE* fff = fopen(filename, "r");
-  PCU_ALWAYS_ASSERT(fff);
-  readClassification(fff, mesh.localNumVerts, &(mesh.fathers2D)); // note we re-use classification reader
-  fclose(fff);
+    sprintf(filename, "%s.%d",fathers2Dfilename,self);
+    FILE* fff = fopen(filename, "r");
+    PCU_ALWAYS_ASSERT(fff);
+    readClassification(fff, mesh.localNumVerts, &(mesh.fathers2D)); // note we re-use classification reader
+    fclose(fff);
+  }
 
   if( strcmp(matchfilename, "NULL") ) {
     sprintf(filename, "%s.%d",matchfilename,self);
@@ -768,7 +770,8 @@ int main(int argc, char** argv)
   MPI_Init(&argc,&argv);
   PCU_Comm_Init();
   lion_set_verbosity(1);
-  if( argc != 10 ) {
+  int noVerify=0;    // maintain default of verifying if not explicitly requesting it off
+  if( argc < 10 ) {
     if( !PCU_Comm_Self() ) {
       printf("Usage: %s <ascii mesh connectivity .cnn> "
           "<ascii vertex coordinates .crd> "
@@ -777,7 +780,8 @@ int main(int argc, char** argv)
           "<ascii vertex fathers2D flag .fathers2D> "
           "<ascii solution flag .soln> "
           "<ascii conn header> "
-          "<output model .dmg> <output mesh .smb>\n",
+          "<output model .dmg> <output mesh .smb>",
+          "turn off verify mesh if equal 1 (on if you give nothing)\n",
           argv[0]);
     }
     return 0;
@@ -786,6 +790,7 @@ int main(int argc, char** argv)
   gmi_register_mesh();
   gmi_register_null();
 
+  if( argc == 11 ) noVerify=atoi(argv[10]);
 
   double t0 = PCU_Time();
   MeshInfo m;
@@ -823,9 +828,13 @@ int main(int argc, char** argv)
   apf::removeTagFromDimension(mesh, tc, 0);
   mesh->destroyTag(tc);
  
-  apf::MeshTag* tf = setMappedTag(mesh, "fathers2D", m.fathers2D, 1,
+  if( strcmp(argv[5], "NULL") ) {
+    apf::MeshTag* tf = setMappedTag(mesh, "fathers2D", m.fathers2D, 1,
       m.localNumVerts, outMap);
-  (void) tf;
+    (void) tf;
+  } else if(!PCU_Comm_Self())
+    fprintf(stderr, "fathers2D not requrested \n");
+
   //mesh->destroyTag(tf);
 
   if(0==1) {
@@ -851,11 +860,12 @@ int main(int argc, char** argv)
 
   if(!PCU_Comm_Self())
     fprintf(stderr, "seconds to create mesh %.3f\n", PCU_Time()-t0);
-  mesh->verify();
+  if(noVerify != 1) mesh->verify();
 
   outMap.clear();
   gmi_write_dmg(model, argv[8]);
   mesh->writeNative(argv[9]);
+  if(noVerify != 0) mesh->verify();
   apf::writeVtkFiles("rendered",mesh);
 
   mesh->destroyNative();

From 18fa46f1709811de1d588c9da717972f45ef4813 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Sat, 17 Jul 2021 14:21:40 -0700
Subject: [PATCH 413/555] and a couple more of the same multi-entity-dimenion
 loops packed into a single message that overflows INT_MAX

---
 apf/apfVerify.cc             | 27 ++++++++++++---------------
 test/matchedNodeElmReader.cc |  2 +-
 2 files changed, 13 insertions(+), 16 deletions(-)

diff --git a/apf/apfVerify.cc b/apf/apfVerify.cc
index 9bb785611..448b1ee41 100644
--- a/apf/apfVerify.cc
+++ b/apf/apfVerify.cc
@@ -521,19 +521,18 @@ static void receiveAlignment(Mesh* m)
 
 static void verifyAlignment(Mesh* m)
 {
-  PCU_Comm_Begin();
-  for (int d = 1; d <= m->getDimension(); ++d)
-  {
+  for (int d = 1; d <= m->getDimension(); ++d) {
+    PCU_Comm_Begin();
     MeshIterator* it = m->begin(d);
     MeshEntity* e;
     while ((e = m->iterate(it)))
       if (m->isShared(e))
         sendAlignment(m, e);
     m->end(it);
+    PCU_Comm_Send();
+    while (PCU_Comm_Receive())
+      receiveAlignment(m);
   }
-  PCU_Comm_Send();
-  while (PCU_Comm_Receive())
-    receiveAlignment(m);
 }
 
 void packFieldInfo(Field* f, int to)
@@ -773,31 +772,29 @@ static void verifyTags(Mesh* m)
   
   // verify tag data
 
-  PCU_Comm_Begin();
-  for (int d = 0; d <= m->getDimension(); ++d)
+  for (int d = 0; d <= m->getDimension(); ++d) 
   {
+    PCU_Comm_Begin();
     MeshIterator* it = m->begin(d);
     MeshEntity* e;
-    while ((e = m->iterate(it)))
+    while ((e = m->iterate(it))) 
     {
       if (m->getOwner(e)!=PCU_Comm_Self()) continue;
-      if (m->isShared(e))
-      {
+      if (m->isShared(e)) {
         Copies r;
         m->getRemotes(e, r);
         sendTagData(m, e, tags, r);
       }
-      if (m->isGhosted(e))
-      {
+      if (m->isGhosted(e)) {
         Copies g;
         m->getGhosts(e, g);
         sendTagData(m, e, tags, g, true);
       }
     } // while
     m->end(it);
+    PCU_Comm_Send();
+    receiveTagData(m, tags);
   } // for
-  PCU_Comm_Send();
-  receiveTagData(m, tags);
 }
 
 void verify(Mesh* m, bool abort_on_error)
diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index dd66bfe0a..5bd1f486c 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -833,7 +833,7 @@ int main(int argc, char** argv)
       m.localNumVerts, outMap);
     (void) tf;
   } else if(!PCU_Comm_Self())
-    fprintf(stderr, "fathers2D not requrested \n");
+    fprintf(stderr, "fathers2D not requested \n");
 
   //mesh->destroyTag(tf);
 

From 2d42969b53b450a1a3d0cea5649325d26a9d50c9 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Sat, 17 Jul 2021 15:14:26 -0700
Subject: [PATCH 414/555]  adding verify  to describe....probably would be
 better to make it a flag request as it can take a while

---
 test/describe.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/describe.cc b/test/describe.cc
index 73e79a56b..d881ba575 100644
--- a/test/describe.cc
+++ b/test/describe.cc
@@ -101,6 +101,7 @@ int main(int argc, char** argv)
   gmi_register_mesh();
   print_stats("malloc used before", get_chunks());
   apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]);
+  m->verify();
   print_stats("kernel heap", get_peak());
   print_stats("malloc used", get_chunks());
   Parma_PrintPtnStats(m, "");

From a9e12f92dff4e4f5cca7b14e5a9fe6e14216c056 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Mon, 19 Jul 2021 22:23:35 -0700
Subject: [PATCH 415/555]  we were running  out  of memory in reorder so this
 collected some stats around there

---
 phasta/phCook.cc | 42 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/phasta/phCook.cc b/phasta/phCook.cc
index 76f9ddf2e..36a93adf5 100644
--- a/phasta/phCook.cc
+++ b/phasta/phCook.cc
@@ -30,6 +30,39 @@
 #include <stdlib.h>
 #include <cstring>
 #include <iostream>
+#include <malloc.h>
+
+static void print_stats(const char* name, double value)
+{
+  double min, max, avg;
+  min = PCU_Min_Double(value);
+  max = PCU_Max_Double(value);
+  avg = PCU_Add_Double(value);
+  avg /= PCU_Comm_Peers();
+  double imb = max / avg;
+  if (!PCU_Comm_Self())
+    printf("%s: min %f max %f avg %f imb %f\n", name, min, max, avg, imb);
+}
+
+#if defined(__linux__)
+
+static double get_chunks()
+{
+  struct mallinfo m = mallinfo();
+  return m.uordblks + m.hblkhd;
+}
+
+#else
+static double get_chunks()
+{
+  cheese
+  if(!PCU_Comm_Self())
+    printf("%s:%d: OS Not supported\n", __FILE__, __LINE__);
+  return(-1.0);
+}
+#endif
+
+
 
 #define SIZET(a) static_cast<size_t>(a)
 
@@ -151,9 +184,18 @@ namespace ph {
         in.isReorder )
     {
       apf::MeshTag* order = NULL;
+
+      print_stats("malloc used before Bfs", get_chunks());
+
       if (in.isReorder && PCU_Comm_Peers() > 1)
         order = Parma_BfsReorder(m);
+
+      print_stats("malloc used before reorder", get_chunks());
+
       apf::reorderMdsMesh(m,order);
+
+      print_stats("malloc used after reorder", get_chunks());
+
     }
   }
 

From cd3171e091a294efff5457a4b15dc32b57ee2c18 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Wed, 21 Jul 2021 13:30:44 -0400
Subject: [PATCH 416/555] Updates the comment for makeAdvanced API

---
 ma/maInput.h | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/ma/maInput.h b/ma/maInput.h
index 6cf717109..b6829c0c6 100644
--- a/ma/maInput.h
+++ b/ma/maInput.h
@@ -182,8 +182,7 @@ const Input* configureIdentity(Mesh* m, SizeField* f=0, SolutionTransfer* s=0);
 /** \brief used internally, but users can call this if they want */
 void validateInput(Input* in);
 
-/** \brief removes constant-ness from a constant Input pointer
-           so users can modify it */
+/** \brief creates a new non-constant Input for advanced users */
 Input* makeAdvanced(const Input* in);
 
 }

From 5aa84665067a39cfd0ad17165d07168531870343 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimi1364@gmail.com>
Date: Wed, 21 Jul 2021 14:26:43 -0400
Subject: [PATCH 417/555] ph_adapt and chef9 only run with ENABLE_ZOLTAN ON

---
 test/testing.cmake | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/testing.cmake b/test/testing.cmake
index d262ba71d..606f7935a 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -136,7 +136,7 @@ if(ENABLE_SIMMETRIX AND SIM_PARASOLID AND SIMMODSUITE_SimAdvMeshing_FOUND)
 endif(ENABLE_SIMMETRIX AND SIM_PARASOLID AND SIMMODSUITE_SimAdvMeshing_FOUND)
 
 set(MDIR ${MESHES}/phasta/loopDriver)
-if(ENABLE_SIMMETRIX AND PCU_COMPRESS AND SIM_PARASOLID
+if(ENABLE_ZOLTAN AND ENABLE_SIMMETRIX AND PCU_COMPRESS AND SIM_PARASOLID
     AND SIMMODSUITE_SimAdvMeshing_FOUND)
   mpi_test(ph_adapt 1
     ${CMAKE_CURRENT_BINARY_DIR}/ph_adapt
@@ -724,7 +724,7 @@ if (PCU_COMPRESS)
   add_test(NAME chef8
     COMMAND diff -r out_mesh/ good_mesh/
     WORKING_DIRECTORY ${MDIR})
-  if(ENABLE_SIMMETRIX)
+  if(ENABLE_ZOLTAN AND ENABLE_SIMMETRIX)
     mpi_test(chef9 2 ${CMAKE_CURRENT_BINARY_DIR}/chef
       WORKING_DIRECTORY ${MESHES}/phasta/simModelAndAttributes)
   endif()

From 51b592f9710692d9bc98c0f0190f27efccda12bd Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Wed, 28 Jul 2021 21:33:28 -0600
Subject: [PATCH 418/555] add vtk output if requested and fix but

---
 apf/apfConstruct.cc | 48 ---------------------------------------------
 phasta/phCook.cc    |  2 ++
 phasta/phInput.cc   |  2 ++
 phasta/phInput.h    |  1 +
 4 files changed, 5 insertions(+), 48 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index 75e5a20a7..d5e7bb1cf 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -34,7 +34,6 @@ static void constructElements(
   int nev = apf::Mesh::adjacentCount[etype][0];
   unsigned etypeL=1;
   for (int i = 0; i < nelem; ++i) {
-    int irep=0;
     Downward verts;
     int offset = i * nev;
     Gid vCur=conn[offset];
@@ -187,7 +186,6 @@ static void constructRemotes(Mesh2* m, GlobalToVert& globalToVert)
       if (*rit != self) {
         PCU_COMM_PACK(*rit, gid);
         PCU_COMM_PACK(*rit, vert);
-        PCU_Debug_Print("sending remotes: gid %ld to %d\n", gid, *rit);
       }
   }
   PCU_Comm_Send();
@@ -199,7 +197,6 @@ static void constructRemotes(Mesh2* m, GlobalToVert& globalToVert)
     int from = PCU_Comm_Sender();
     MeshEntity* vert = globalToVert[gid];
     m->addRemote(vert, from, remote);
-    PCU_Debug_Print("receive remotes: from %d gid %ld\n", from, gid);
   }
   // who is not stuck?
   lion_eprint(1, "%d done inside remotes \n",PCU_Comm_Self());
@@ -213,11 +210,7 @@ void construct(Mesh2* m, const Gid* conn, int nelem, int etype,
   constructResidence(m, globalToVert);
   lion_eprint(1, "%d after residence \n",PCU_Comm_Self());
   constructRemotes(m, globalToVert);
-  PCU_Barrier();
-  lion_eprint(1, "%d after Remotes \n",PCU_Comm_Self());
   stitchMesh(m);
-  PCU_Barrier();
-  lion_eprint(1, "%d after stitch \n",PCU_Comm_Self());
   m->acceptChanges();
 }
 
@@ -324,7 +317,6 @@ void setCoords(Mesh2* m, const double* coords, int nverts,
 void setMatches(Mesh2* m, const Gid* matches, int nverts,
     GlobalToVert& globalToVert)
 {
-  PCU_Debug_Open();
   Gid max = getMax(globalToVert);
   Gid total = max + 1;
   int peers = PCU_Comm_Peers();
@@ -339,8 +331,6 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts,
   /* Force each peer to have exactly mySize verts.
      This means we might need to send and recv some matches */
   Gid* c = new Gid[mySize];
-  PCU_Debug_Print("%d mysize %d\n", self, mySize);
-
   Gid start = PCU_Exscan_Long(nverts);
 
   PCU_Comm_Begin();
@@ -355,8 +345,6 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts,
     PCU_COMM_PACK(to, start);
     PCU_COMM_PACK(to, n);
     PCU_Comm_Pack(to, matches, n*sizeof(Gid));
-    PCU_Debug_Print("%d sending start %ld n %d to %d\n",
-        self, start, n, to);
 
     nverts -= n;
     start += n;
@@ -369,16 +357,8 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts,
     PCU_COMM_UNPACK(start);
     PCU_COMM_UNPACK(n); /// in-place 64
     PCU_Comm_Unpack(&c[(start - myOffset)], n*sizeof(Gid));
-    PCU_Debug_Print("%d receiving start %ld n %d from %d\n",
-        self, start, n, PCU_Comm_Sender());
   }
 
-  for (int i = 0; i < mySize; ++i) {
-    Gid match = c[i];
-    if( match != -1 )
-      PCU_Debug_Print("%d found match %ld at gid %ld\n",
-          self, match, i+myOffset);
-  }
 
   /* Tell all the owners of the matches what we need */
   typedef std::vector< std::vector<int> > TmpParts;
@@ -389,8 +369,6 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts,
     int tmpI=gid / quotient;
     int to = std::min(peers - 1,tmpI); 
     PCU_COMM_PACK(to, gid);
-    PCU_Debug_Print("%d requesting matches of gid %ld isShared %d isOwned %d from %d\n",
-        self, gid, m->isShared(it->second), m->isOwned(it->second), to);
   }
   PCU_Comm_Send();
   while (PCU_Comm_Receive()) {
@@ -411,10 +389,6 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts,
       Gid matchGid = c[i];
       PCU_COMM_PACK(to, gid);
       PCU_COMM_PACK(to, matchGid);
-      if( matchGid != -1 ) {
-        PCU_Debug_Print("%d packing i %d gid %ld matchGid %ld to %d\n",
-            self, i, gid, matchGid, to);
-      }
     }
   }
   PCU_Comm_Send();
@@ -426,10 +400,6 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts,
     PCU_ALWAYS_ASSERT(gid != match);
     PCU_ALWAYS_ASSERT(globalToVert.count(gid));
     m->setLongTag(globalToVert[gid], matchGidTag, &match);
-    if( match != -1 ) {
-      PCU_Debug_Print("%d attaching match %ld to gid %ld\n",
-          self, match, gid);
-    }
   }
 
   /* Use the 1D partitioning of global ids to distribute the 
@@ -445,8 +415,6 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts,
     int to = std::min(peers - 1,tmpI); 
     PCU_COMM_PACK(to, gid);
     PCU_COMM_PACK(to, e);
-    PCU_Debug_Print("%d packing pointer to %d gid %ld vert %p\n",
-        self, to, gid, (void*)e);
   }
   PCU_Comm_Send();
   typedef std::pair< int, apf::MeshEntity* > EntOwnerPtrs;
@@ -459,8 +427,6 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts,
     PCU_COMM_UNPACK(vert);
     int owner = PCU_Comm_Sender();
     gidPtrs[gid-myOffset].push_back(EntOwnerPtrs(owner,vert));
-    PCU_Debug_Print("%d unpacking pointer from %d gid %ld vert %p\n",
-        self, owner, gid, (void*)vert);
   }
 
   /* Tell the brokers of the matches we need */
@@ -477,8 +443,6 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts,
       int to = std::min(peers - 1,tmpI);  //broker
       PCU_COMM_PACK(to, gid); // send the local vert gid
       PCU_COMM_PACK(to, matchGid); // and the match gid needed
-      PCU_Debug_Print("%d packing req ptr to %d gid %ld matchGid %ld\n",
-          self, to, gid, matchGid);
     }
   }
   PCU_Comm_Send();
@@ -490,8 +454,6 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts,
     MatchingPair mp(gid,matchGid);
     int from = PCU_Comm_Sender();
     matchParts[mp].push_back(from); // store a list of the proceses that need the pair (entity gid, match gid)
-    PCU_Debug_Print("%d unpacking ptr req from %d gid %ld matchGid %ld\n",
-        self, from, gid, matchGid);
   }
 
   /* Send the match pointer and owner process to everybody
@@ -514,8 +476,6 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts,
         PCU_COMM_PACK(to, owner);
         apf::MeshEntity* ent = eop.second;
         PCU_COMM_PACK(to, ent);
-        PCU_Debug_Print("%d packing match ptr to %d gid %ld matchGid %ld vert %p owner %d\n",
-            self, to, gid, matchGid, (void*)ent, owner);
       }
     }
   }
@@ -532,14 +492,8 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts,
       PCU_COMM_UNPACK(owner);
       MeshEntity* match;
       PCU_COMM_UNPACK(match);
-      PCU_Debug_Print("%d unpacked match ptr from %d gid %ld matchGid %ld matchPtr %p owner %d\n",
-          self, PCU_Comm_Sender(), gid, matchGid, (void*)match, owner);
       PCU_ALWAYS_ASSERT(globalToVert.count(gid));
       MeshEntity* partner = globalToVert[gid];
-      if(match == partner && owner == self) {
-        PCU_Debug_Print("%d match == partner owner == self match %p partner %p owner %d\n",
-            self, (void*)match, (void*)partner, owner);
-      }
       PCU_ALWAYS_ASSERT(! (match == partner && owner == self) );
       m->addMatch(partner, owner, match);
     }
@@ -556,8 +510,6 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts,
         int rightPart = cp->first;
         apf::MeshEntity* right = cp->second;
         m->addMatch(left, rightPart, right);
-        PCU_Debug_Print("%d add remote copy match ptr to %d gid %ld\n",
-          self, rightPart, it->first); // is this last argument correct
       }
     }
   }
diff --git a/phasta/phCook.cc b/phasta/phCook.cc
index 36a93adf5..f733ade58 100644
--- a/phasta/phCook.cc
+++ b/phasta/phCook.cc
@@ -295,6 +295,8 @@ namespace chef {
     ph::Output out;
     out.openfile_write = openfile_write;
     bake(g,m,in,out);
+    if ((in.writeVTK) == 1)  apf::writeVtkFiles("rendered",m);
+
   }
   void cook(gmi_model*& g, apf::Mesh2*& m,
       ph::Input& ctrl) {
diff --git a/phasta/phInput.cc b/phasta/phInput.cc
index 167de61a4..e7c184379 100644
--- a/phasta/phInput.cc
+++ b/phasta/phInput.cc
@@ -62,6 +62,7 @@ static void setDefaults(Input& in)
   in.parmaVerbosity = 1; //fairly quiet
   in.writeGeomBCFiles = 0;  // write additional geombc file for vis in streaming
   in.writeRestartFiles = 0;  // write additional restart file for vis in streaming
+  in.writeVTK = 0; 
   in.ramdisk = 0;
   in.meshqCrtn = 0.027; 
   in.elementImbalance = 1.03;
@@ -146,6 +147,7 @@ static void formMaps(Input& in, StringMap& stringMap, IntMap& intMap, DblMap& db
   intMap["axisymmetry"] = &in.axisymmetry;
   intMap["parmaLoops"] = &in.parmaLoops;
   intMap["parmaVerbosity"] = &in.parmaVerbosity;
+  intMap["writeVTK"] = &in.writeVTK;
   intMap["writeGeomBCFiles"] = &in.writeGeomBCFiles;
   intMap["writeRestartFiles"] = &in.writeRestartFiles;
   intMap["ramdisk"] = &in.ramdisk;
diff --git a/phasta/phInput.h b/phasta/phInput.h
index b4c1de98f..21e816a3b 100644
--- a/phasta/phInput.h
+++ b/phasta/phInput.h
@@ -142,6 +142,7 @@ class Input
     int formEdges;
     int parmaLoops;
     int parmaVerbosity;
+    int writeVTK;
     /** \brief write the geombc file during in-memory data transfer
        between phasta and chef. */
     int writeGeomBCFiles;

From 81a6999766f03c3def5bee96805abe5e67039ba6 Mon Sep 17 00:00:00 2001
From: Cameron Smith <cwsmith@users.noreply.github.com>
Date: Sun, 8 Aug 2021 18:58:08 -0400
Subject: [PATCH 419/555] citation.cff points at toms paper

---
 CITATION.cff | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 CITATION.cff

diff --git a/CITATION.cff b/CITATION.cff
new file mode 100644
index 000000000..f3c5fc944
--- /dev/null
+++ b/CITATION.cff
@@ -0,0 +1,19 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+authors:
+- family-names: "Ibanez"
+  given-names: "Daniel A."
+- family-names: "Seol"
+  given-names: "E. Seegyoung"
+- family-names: "Smith"
+  given-names: "Cameron W."
+  orcid: "https://orcid.org/0000-0001-9258-5226"  
+- family-names: "Shephard"
+  given-names: "Mark S."
+title: "PUMI: Parallel Unstructured Mesh Infrastructure"
+doi: https://dl.acm.org/doi/10.1145/2814935
+month: 6
+year: 2016
+issue: "42, 3"
+number: "Article 17"
+journal: "ACM Trans. Math. Softw"

From 597c78bc42cbf22de5650288d1d140beb431b778 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 11 Aug 2021 10:45:31 -0400
Subject: [PATCH 420/555] attempt to fix formatting

following https://github.com/citation-file-format/citation-file-format/blob/main/schema-guide.md
---
 CITATION.cff | 44 +++++++++++++++++++++++++++-----------------
 1 file changed, 27 insertions(+), 17 deletions(-)

diff --git a/CITATION.cff b/CITATION.cff
index f3c5fc944..7666a669a 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -1,19 +1,29 @@
 cff-version: 1.2.0
+title: "PUMI"
 message: "If you use this software, please cite it as below."
-authors:
-- family-names: "Ibanez"
-  given-names: "Daniel A."
-- family-names: "Seol"
-  given-names: "E. Seegyoung"
-- family-names: "Smith"
-  given-names: "Cameron W."
-  orcid: "https://orcid.org/0000-0001-9258-5226"  
-- family-names: "Shephard"
-  given-names: "Mark S."
-title: "PUMI: Parallel Unstructured Mesh Infrastructure"
-doi: https://dl.acm.org/doi/10.1145/2814935
-month: 6
-year: 2016
-issue: "42, 3"
-number: "Article 17"
-journal: "ACM Trans. Math. Softw"
+keywords:
+  - "unstructured meshes"
+  - "parallel"
+  - "mesh adaptation"
+  - "load balancing"
+  - "C++"
+  - "finite elements"
+references:
+  - authors:
+      - family-names: "Ibanez"
+        given-names: "Daniel A."
+      - family-names: "Seol"
+        given-names: "E. Seegyoung"
+      - family-names: "Smith"
+        given-names: "Cameron W."
+        orcid: "https://orcid.org/0000-0001-9258-5226"  
+      - family-names: "Shephard"
+        given-names: "Mark S."
+  type: article
+  title: "PUMI: Parallel Unstructured Mesh Infrastructure"
+  doi: "https://dl.acm.org/doi/10.1145/2814935"
+  month: 6
+  year: 2016
+  volume: 42
+  issue: 3
+  journal: "ACM Trans. Math. Softw"

From 7d0515eabe7ec9a615db2850fbbba8624cc7c4a5 Mon Sep 17 00:00:00 2001
From: Cameron Smith <cwsmith@users.noreply.github.com>
Date: Wed, 11 Aug 2021 10:48:22 -0400
Subject: [PATCH 421/555] Update CITATION.cff

---
 CITATION.cff | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/CITATION.cff b/CITATION.cff
index 7666a669a..0d38e7afd 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -9,7 +9,8 @@ keywords:
   - "C++"
   - "finite elements"
 references:
-  - authors:
+  - type: article
+    authors:
       - family-names: "Ibanez"
         given-names: "Daniel A."
       - family-names: "Seol"
@@ -19,11 +20,10 @@ references:
         orcid: "https://orcid.org/0000-0001-9258-5226"  
       - family-names: "Shephard"
         given-names: "Mark S."
-  type: article
-  title: "PUMI: Parallel Unstructured Mesh Infrastructure"
-  doi: "https://dl.acm.org/doi/10.1145/2814935"
-  month: 6
-  year: 2016
-  volume: 42
-  issue: 3
-  journal: "ACM Trans. Math. Softw"
+    title: "PUMI: Parallel Unstructured Mesh Infrastructure"
+    doi: "https://dl.acm.org/doi/10.1145/2814935"
+    month: 6
+    year: 2016
+    volume: 42
+    issue: 3
+    journal: "ACM Trans. Math. Softw"

From a2f0277f931fe163aa58cbd90b4d406802d95d10 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 11 Aug 2021 10:57:17 -0400
Subject: [PATCH 422/555] more fixes

use 'cffconvert --validate -if CITATION.cff' to find errors
---
 CITATION.cff | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/CITATION.cff b/CITATION.cff
index 0d38e7afd..b7e904134 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -1,6 +1,22 @@
-cff-version: 1.2.0
+cff-version: 1.1.0
 title: "PUMI"
 message: "If you use this software, please cite it as below."
+authors:
+  - family-names: "Ibanez"
+    given-names: "Daniel A."
+  - family-names: "Smith"
+    given-names: "Cameron W."
+    orcid: "https://orcid.org/0000-0001-9258-5226"  
+  - family-names: "Granzow"
+    given-names: "Brian"
+  - family-names: "Zaide"
+    given-names: "Daniel"
+  - family-names: "Hakimi"
+    given-names: "Morteza"
+  - family-names: "Seol"
+    given-names: "E. Seegyoung"
+date-released: 2021-06-25
+version: 2.2.6
 keywords:
   - "unstructured meshes"
   - "parallel"
@@ -21,9 +37,9 @@ references:
       - family-names: "Shephard"
         given-names: "Mark S."
     title: "PUMI: Parallel Unstructured Mesh Infrastructure"
-    doi: "https://dl.acm.org/doi/10.1145/2814935"
+    doi: "10.1145/2814935"
     month: 6
     year: 2016
     volume: 42
-    issue: 3
+    issue: "3"
     journal: "ACM Trans. Math. Softw"

From 86609e438880dd0f8549b5ecba842ce8aa3f88eb Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimm2@rpi.edu>
Date: Fri, 13 Aug 2021 15:27:41 -0400
Subject: [PATCH 423/555] fixes the leak associated with maLayerCoarsen

also adds a comment about cleaning up input and associated sizefield and
solutionstransfer objects
---
 crv/crvAdapt.cc      | 1 +
 ma/ma.cc             | 2 ++
 ma/maLayerCoarsen.cc | 5 +++++
 3 files changed, 8 insertions(+)

diff --git a/crv/crvAdapt.cc b/crv/crvAdapt.cc
index de18b96a5..e13ce5193 100644
--- a/crv/crvAdapt.cc
+++ b/crv/crvAdapt.cc
@@ -235,6 +235,7 @@ void adapt(ma::Input* in)
   apf::printStats(a->mesh);
   crv::clearTags(a);
   delete a;
+  // cleanup input object and associated sizefield and solutiontransfer objects
   if (in->ownsSizeField)
     delete in->sizeField;
   if (in->ownsSolutionTransfer)
diff --git a/ma/ma.cc b/ma/ma.cc
index 8ba200588..64d692e25 100644
--- a/ma/ma.cc
+++ b/ma/ma.cc
@@ -45,6 +45,7 @@ void adapt(Input* in)
   postBalance(a);
   Mesh* m = a->mesh;
   delete a;
+  // cleanup input object and associated sizefield and solutiontransfer objects
   if (in->ownsSizeField)
     delete in->sizeField;
   if (in->ownsSolutionTransfer)
@@ -118,6 +119,7 @@ void adaptVerbose(Input* in, bool verbose)
   postBalance(a);
   Mesh* m = a->mesh;
   delete a;
+  // cleanup input object and associated sizefield and solutiontransfer objects
   if (in->ownsSizeField)
     delete in->sizeField;
   if (in->ownsSolutionTransfer)
diff --git a/ma/maLayerCoarsen.cc b/ma/maLayerCoarsen.cc
index 38ace0f05..12feced3d 100644
--- a/ma/maLayerCoarsen.cc
+++ b/ma/maLayerCoarsen.cc
@@ -206,6 +206,11 @@ void localizeLayerStacks(Mesh* m) {
   for (int d = 1; d < m->getDimension(); ++d)
     migrateForLayerCollapse(a,d,round);
   delete a;
+  // cleanup input object and associated sizefield and solutiontransfer objects
+  if (in->ownsSizeField)
+    delete in->sizeField;
+  if (in->ownsSolutionTransfer)
+    delete in->solutionTransfer;
   delete in;
 }
 

From 106bfec30186733de126d96f925155b7a15481b7 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimm2@rpi.edu>
Date: Mon, 16 Aug 2021 15:02:15 -0400
Subject: [PATCH 424/555] Fixes some of memory leaks mentioned in #344

---
 crv/crvVtk.cc                        | 11 ++++-------
 mds/apfMDS.cc                        |  2 +-
 test/highOrderSolutionTransfer.cc    |  2 +-
 test/residualErrorEstimation_test.cc |  3 +++
 test/test_matrix_grad.cc             |  2 ++
 5 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/crv/crvVtk.cc b/crv/crvVtk.cc
index 489e1f3ae..5cdd0f4d9 100644
--- a/crv/crvVtk.cc
+++ b/crv/crvVtk.cc
@@ -949,7 +949,7 @@ static void makeDirectories(const char* prefix, int type, int n)
 
 void writeCurvedWireFrame(apf::Mesh* m, int n, const char* prefix)
 {
-  apf::Mesh2* wireMesh = apf::makeEmptyMdsMesh(0, 1, false);
+  apf::Mesh2* wireMesh = apf::makeEmptyMdsMesh(NULL, 1, false);
   apf::Field* f = m->getCoordinateField();
   apf::MeshEntity* ent;
   apf::MeshIterator* it;
@@ -991,16 +991,13 @@ void writeCurvedWireFrame(apf::Mesh* m, int n, const char* prefix)
     }
     count++;
   }
-  // wireMesh is not a valid mesh, so neither of the following will work!
-  /* apf::deriveMdsModel(wireMesh); */
-  /* wireMesh->acceptChanges(); */
-  /* wireMesh->verify(); */
+  m->end(it);
   apf::printStats(wireMesh);
   std::stringstream ss;
   ss << prefix << "_wire";
   apf::writeVtkFiles(ss.str().c_str(),wireMesh);
-  /* wireMesh->destroyNative(); */
-  /* apf::destroyMesh(wireMesh); */
+  wireMesh->destroyNative();
+  apf::destroyMesh(wireMesh);
 }
 
 
diff --git a/mds/apfMDS.cc b/mds/apfMDS.cc
index d1687164b..d612f7c08 100644
--- a/mds/apfMDS.cc
+++ b/mds/apfMDS.cc
@@ -600,7 +600,7 @@ class MeshMDS : public Mesh2
       apf::destroyField(coordinateField);
       coordinateField = 0;
       gmi_model* model = static_cast<gmi_model*>(mesh->user_model);
-      if (ownsModel)
+      if (ownsModel && model)
         gmi_destroy(model);
       mds_apf_destroy(mesh);
       mesh = 0;
diff --git a/test/highOrderSolutionTransfer.cc b/test/highOrderSolutionTransfer.cc
index 8fbebcf2e..8dd32a0e2 100644
--- a/test/highOrderSolutionTransfer.cc
+++ b/test/highOrderSolutionTransfer.cc
@@ -159,7 +159,7 @@ double testH1Field(
 
   int dim = m->getDimension();
   int count;
-  apf::MeshIterator* it = m->begin(0);
+  apf::MeshIterator* it;
   apf::MeshEntity* ent;
 
   // Verify that interpolated solution field agrees with exact field.
diff --git a/test/residualErrorEstimation_test.cc b/test/residualErrorEstimation_test.cc
index 9cbd63491..085966460 100644
--- a/test/residualErrorEstimation_test.cc
+++ b/test/residualErrorEstimation_test.cc
@@ -155,6 +155,9 @@ double computeElementExactError(apf::Mesh* mesh, apf::MeshEntity* e,
   if (error < 0.0)
     error = -error;
 
+  apf::destroyElement(el);
+  apf::destroyMeshElement(me);
+
   return sqrt(error);
 }
 
diff --git a/test/test_matrix_grad.cc b/test/test_matrix_grad.cc
index c479c464b..71eb13729 100644
--- a/test/test_matrix_grad.cc
+++ b/test/test_matrix_grad.cc
@@ -114,6 +114,8 @@ int main(int argc, char* argv[])
   check_matrix_deriv->process(mesh);
   delete check_matrix_deriv;
   std::cout<<"Done"<<std::endl;
+  mesh->destroyNative();
+  apf::destroyMesh(mesh);
   PCU_Comm_Free();
   MPI_Finalize();
   return 0;

From 0bc5a0593c82dbdfaed644b8e96adb40894ace42 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimm2@rpi.edu>
Date: Tue, 17 Aug 2021 13:35:54 -0400
Subject: [PATCH 425/555] Fixes an error that cause Simx Parasolid inquiry to
 fail

Related to the leaks mentioned in #344
---
 ma/maSnap.cc | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/ma/maSnap.cc b/ma/maSnap.cc
index fd9b1f47b..195dca005 100644
--- a/ma/maSnap.cc
+++ b/ma/maSnap.cc
@@ -220,7 +220,12 @@ static Vector getInvSphere(const Vector& x)
   double phi = atan2(x[1], x[0]);
   if (phi < 0)
     phi = phi + 2 * M_PI;
-  double the = acos(x[2]);
+  // the following 3 lines are to avoid errors
+  // caused by floating point operations
+  double x2 = x[2];
+  if (x2 > 1.0) x2 = 1.0;
+  if (x2 < -1.0) x2 = -1.0;
+  double the = acos(x2);
   Vector res(phi, the, 0.0);
   return res;
 }

From 41177584034790db2343e07130bfa444e33be85c Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimm2@rpi.edu>
Date: Tue, 17 Aug 2021 14:18:37 -0400
Subject: [PATCH 426/555] Fixes leaks in serialize, collapse, and rm_extrusion

---
 test/collapse.cc     | 2 ++
 test/rm_extrusion.cc | 1 +
 test/serialize.cc    | 2 ++
 3 files changed, 5 insertions(+)

diff --git a/test/collapse.cc b/test/collapse.cc
index 6615ebf97..da45c1fd4 100644
--- a/test/collapse.cc
+++ b/test/collapse.cc
@@ -60,6 +60,8 @@ int main(int argc, char** argv) {
   code.mesh = apf::loadMdsMesh(modelFile, meshFile);
   apf::Unmodulo outMap(PCU_Comm_Self(), PCU_Comm_Peers());
   Parma_ShrinkPartition(code.mesh, partitionFactor, code);
+  code.mesh->destroyNative();
+  apf::destroyMesh(code.mesh);
 #ifdef HAVE_SIMMETRIX
   gmi_sim_stop();
   Sim_unregisterAllKeys();
diff --git a/test/rm_extrusion.cc b/test/rm_extrusion.cc
index 4e36da2c4..a21133b8b 100644
--- a/test/rm_extrusion.cc
+++ b/test/rm_extrusion.cc
@@ -135,6 +135,7 @@ int main(int argc, char** argv)
 
   M_release(sim_mesh);
   Progress_delete(progress);
+  gmi_destroy(mdl);
   gmi_sim_stop();
   SimPartitionedMesh_stop();
   Sim_unregisterAllKeys();
diff --git a/test/serialize.cc b/test/serialize.cc
index b84a4baa0..f233770b4 100644
--- a/test/serialize.cc
+++ b/test/serialize.cc
@@ -50,6 +50,8 @@ int main( int argc, char* argv[])
   code.meshFile = argv[3];
   apf::Unmodulo outMap(PCU_Comm_Self(), PCU_Comm_Peers());
   Parma_ShrinkPartition(code.mesh, atoi(argv[4]), code);
+  code.mesh->destroyNative();
+  apf::destroyMesh(code.mesh);
 #ifdef HAVE_SIMMETRIX
   gmi_sim_stop();
   Sim_unregisterAllKeys();

From 01b61e6b20cf729d0d17c4c90d273f6f45ba77b3 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Wed, 18 Aug 2021 14:08:18 +0000
Subject: [PATCH 427/555] back to default of not partitioning if splitFactor 1
 in adapt.inp

---
 phasta/phPartition.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/phasta/phPartition.cc b/phasta/phPartition.cc
index a5b84732b..8546c4188 100644
--- a/phasta/phPartition.cc
+++ b/phasta/phPartition.cc
@@ -51,7 +51,7 @@ apf::Migration* getSplitPlan(Input& in, apf::Mesh2* m)
 {
   PCU_ALWAYS_ASSERT(in.splitFactor >= 1);
   apf::Migration* plan;
-  if (in.splitFactor != 0 ) { // 1) {
+  if (in.splitFactor != 1 ) { //0 ) { // 1) {  // 0 will force reparttition  even when split factor is 1
     apf::Splitter* splitter;
     if (in.partitionMethod == "rib") { //prefer SCOREC RIB over Zoltan RIB
       splitter = Parma_MakeRibSplitter(m);

From 24cfcb1766c485e5e6fd6c14eb27f29776760532 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Sun, 19 Sep 2021 17:07:14 -0600
Subject: [PATCH 428/555] typo

---
 test/matchedNodeElmReader.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 5bd1f486c..a0b5a236d 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -780,7 +780,7 @@ int main(int argc, char** argv)
           "<ascii vertex fathers2D flag .fathers2D> "
           "<ascii solution flag .soln> "
           "<ascii conn header> "
-          "<output model .dmg> <output mesh .smb>",
+          "<output model .dmg> <output mesh .smb>"
           "turn off verify mesh if equal 1 (on if you give nothing)\n",
           argv[0]);
     }

From 56e11a9b66db17ae1bcda9c07ae456d8f01a4d21 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Tue, 19 Oct 2021 12:38:28 -0400
Subject: [PATCH 429/555] support spack running the smoke tests

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 90ea84a72..36015fb2e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -69,7 +69,7 @@ set(VALGRIND_ARGS "" CACHE STRING
   "the command line arguments to VALGRIND")
 
 # smoke test target - a few tests are defined later with the 'SMOKE_TEST' label
-add_custom_target(test_install
+add_custom_target(check
   COMMAND ${CMAKE_CTEST_COMMAND} -L SMOKE_TEST
   COMMENT "running a smoke test on the installed binaries")
 

From 31fc488f40dbc69a1e17380c7cd85ef91fd393fa Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Tue, 19 Oct 2021 14:00:24 -0400
Subject: [PATCH 430/555] pumi needs the mallinfo preproc flag

---
 CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 90ea84a72..3b7dad544 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -156,6 +156,7 @@ if(${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU")
   check_cxx_symbol_exists(mallinfo2 "malloc.h" PUMI_HAS_MALLINFO2)
   if(PUMI_HAS_MALLINFO2)
     target_compile_definitions(core INTERFACE -DPUMI_HAS_MALLINFO2)
+    target_compile_definitions(pumi PRIVATE -DPUMI_HAS_MALLINFO2)
   endif()
 endif()
 

From 3acb6df5a00624974d7727c07c482934ec427118 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Tue, 19 Oct 2021 15:43:13 -0400
Subject: [PATCH 431/555] fix smoke test label

---
 test/testing.cmake | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/test/testing.cmake b/test/testing.cmake
index 606f7935a..1c1120051 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -12,9 +12,7 @@ function(smoke_test TESTNAME PROCS EXE)
   set(tname smoke_test_${TESTNAME})
   add_test(
     NAME ${tname}
-    COMMAND ${MPIRUN} ${MPIRUN_PROCFLAG} ${PROCS} ${VALGRIND} ${VALGRIND_ARGS} ${EXE} ${ARGN}
-    CONFIGURATIONS SMOKE_TEST_CONFIG
-    WORKING_DIRECTORY ${CMAKE_INSTALL_PREFIX}/bin)
+    COMMAND ${MPIRUN} ${MPIRUN_PROCFLAG} ${PROCS} ${VALGRIND} ${VALGRIND_ARGS} ${EXE} ${ARGN})
   SET_TESTS_PROPERTIES(${tname} PROPERTIES LABELS "SMOKE_TEST" )
 endfunction(smoke_test)
 

From 475c109634cf826a01f14bdb22a0c26df7666615 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Tue, 19 Oct 2021 15:57:06 -0400
Subject: [PATCH 432/555] smoke tests always defined

---
 test/CMakeLists.txt     |  1 +
 test/smokeTesting.cmake | 22 ++++++++++++++++++++++
 test/testing.cmake      | 19 -------------------
 3 files changed, 23 insertions(+), 19 deletions(-)
 create mode 100644 test/smokeTesting.cmake

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 4c13c0c06..835135020 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -208,6 +208,7 @@ endif()
 # in scorec-config.cmake
 set(TARGETS ${TARGETS} PARENT_SCOPE)
 
+include(smokeTesting.cmake)
 # Add the nightly testing script, if needed
 if(IS_TESTING)
   include(testing.cmake)
diff --git a/test/smokeTesting.cmake b/test/smokeTesting.cmake
new file mode 100644
index 000000000..ef0ed6024
--- /dev/null
+++ b/test/smokeTesting.cmake
@@ -0,0 +1,22 @@
+function(smoke_test TESTNAME PROCS EXE)
+  set(tname smoke_test_${TESTNAME})
+  add_test(
+    NAME ${tname}
+    COMMAND ${MPIRUN} ${MPIRUN_PROCFLAG} ${PROCS} ${VALGRIND} ${VALGRIND_ARGS} ${EXE} ${ARGN})
+  SET_TESTS_PROPERTIES(${tname} PROPERTIES LABELS "SMOKE_TEST" )
+endfunction(smoke_test)
+
+set(MDIR ${MESHES}/pipe)
+smoke_test(uniform_serial 1
+  ./uniform
+  "${MDIR}/pipe.dmg"
+  "${MDIR}/pipe.smb"
+  "pipe_unif.smb")
+
+smoke_test(split_2 2
+  ./split
+  "${MDIR}/pipe.dmg"
+  "${MDIR}/pipe.smb"
+  "pipe_2_.smb"
+  2)
+
diff --git a/test/testing.cmake b/test/testing.cmake
index 1c1120051..31443373c 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -8,14 +8,6 @@ function(mpi_test TESTNAME PROCS EXE)
   )
 endfunction(mpi_test)
 
-function(smoke_test TESTNAME PROCS EXE)
-  set(tname smoke_test_${TESTNAME})
-  add_test(
-    NAME ${tname}
-    COMMAND ${MPIRUN} ${MPIRUN_PROCFLAG} ${PROCS} ${VALGRIND} ${VALGRIND_ARGS} ${EXE} ${ARGN})
-  SET_TESTS_PROPERTIES(${tname} PROPERTIES LABELS "SMOKE_TEST" )
-endfunction(smoke_test)
-
 mpi_test(shapefun 1 ./shapefun)
 mpi_test(shapefun2 1 ./shapefun2)
 mpi_test(bezierElevation 1 ./bezierElevation)
@@ -259,11 +251,6 @@ mpi_test(uniform_serial 1
   "pipe.smb"
   "pipe_unif.smb")
 mpi_test(classifyThenAdapt 1 ./classifyThenAdapt)
-smoke_test(uniform_serial 1
-  ./uniform
-  "${MDIR}/pipe.${GXT}"
-  "${MDIR}/pipe.smb"
-  "pipe_unif.smb")
 if(ENABLE_SIMMETRIX)
   mpi_test(snap_serial 1
     ./snap
@@ -313,12 +300,6 @@ mpi_test(split_2 2
   "pipe.smb"
   ${MESHFILE}
   2)
-smoke_test(split_2 2
-  ./split
-  "${MDIR}/pipe.${GXT}"
-  "${MDIR}/pipe.smb"
-  ${MESHFILE}
-  2)
 mpi_test(collapse_2 2
   ./collapse
   "${MDIR}/pipe.${GXT}"

From fd6ccc83e17ad8df04dac8a11931061a0d155f95 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Thu, 21 Oct 2021 13:17:58 -0400
Subject: [PATCH 433/555] install files needed for spack stand-alone tests

---
 test/smokeTesting.cmake | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/test/smokeTesting.cmake b/test/smokeTesting.cmake
index ef0ed6024..fe4259436 100644
--- a/test/smokeTesting.cmake
+++ b/test/smokeTesting.cmake
@@ -20,3 +20,7 @@ smoke_test(split_2 2
   "pipe_2_.smb"
   2)
 
+# install the test input files for use in spack post-install tests
+install(FILES "${MDIR}/pipe.dmg" "${MDIR}/pipe0.smb"
+        DESTINATION ${CMAKE_INSTALL_PREFIX}/testdata)
+

From 64ad01e49fc01366eb5f2f9fddbd6499f34ba2f4 Mon Sep 17 00:00:00 2001
From: Thomas-Ulrich <ulrich@geophysik.uni-muenchen.de>
Date: Mon, 8 Nov 2021 10:27:00 +0100
Subject: [PATCH 434/555] use relative path in .gitmodules

---
 .gitmodules | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitmodules b/.gitmodules
index c0cc81b3e..5be9b0df6 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,3 +1,3 @@
 [submodule "pumi-meshes"]
 	path = pumi-meshes
-	url = https://github.com/SCOREC/pumi-meshes.git
+	url = ../../SCOREC/pumi-meshes.git

From 1b2be649c156a6c950fe8333543a217c6cce0459 Mon Sep 17 00:00:00 2001
From: Cameron Smith <cwsmith@users.noreply.github.com>
Date: Mon, 8 Nov 2021 08:42:38 -0500
Subject: [PATCH 435/555] add pull request template

---
 .github/pull_request_template.md | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 .github/pull_request_template.md

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
new file mode 100644
index 000000000..0938c8d80
--- /dev/null
+++ b/.github/pull_request_template.md
@@ -0,0 +1,15 @@
+## Title of Pull Request
+
+Pull requests should target the `develop` branch.
+
+In your PR description please include:
+
+- a brief description
+- references to existing issues this PR fixes (if applicable)
+- for new functionality, please describe the test cases added to exercise it
+
+More information on PUMI development is here: 
+
+https://github.com/SCOREC/core/wiki/Developer-Guide
+
+Thank you for your contribution.

From a1082e894c6c3712fd6ff7b63469a1edb3a61764 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Thu, 16 Dec 2021 15:03:00 -0500
Subject: [PATCH 436/555] create apf tag from physical type #355

only supports tags on linear faces and regions
---
 mds/mdsGmsh.cc | 30 ++++++++++++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/mds/mdsGmsh.cc b/mds/mdsGmsh.cc
index a2584a23f..a613f8e9c 100644
--- a/mds/mdsGmsh.cc
+++ b/mds/mdsGmsh.cc
@@ -55,6 +55,8 @@ struct Reader {
   bool isQuadratic;
   std::map<long, Node> nodeMap;
   std::map<long, apf::MeshEntity*> entMap[4];
+  std::vector<int> physicalTypeFace;
+  std::vector<int> physicalTypeRgn;
 };
 
 void initReader(Reader* r, apf::Mesh2* m, const char* filename)
@@ -158,7 +160,13 @@ void readElement(Reader* r)
   int dim = apf::Mesh::typeDimension[apfType];
   long ntags = getLong(r);
   PCU_ALWAYS_ASSERT(ntags >= 2);
-  getLong(r); /* discard physical type */
+  const int physType = static_cast<int>(getLong(r));
+  //tag only faces and regions
+  if(dim == 2) {
+    r->physicalTypeFace.push_back(physType);
+  } else if(dim == 3) {
+    r->physicalTypeRgn.push_back(physType);
+  }
   long gtag = getLong(r);
   for (long i = 2; i < ntags; ++i)
     getLong(r); /* discard all other element tags */
@@ -183,11 +191,28 @@ void readElements(Reader* r)
   seekMarker(r, "$Elements");
   long n = getLong(r);
   getLine(r);
+  r->physicalTypeFace.reserve(n);
+  r->physicalTypeRgn.reserve(n);
   for (long i = 0; i < n; ++i)
     readElement(r);
   checkMarker(r, "$EndElements");
 }
 
+void setElmPhysicalType(Reader* r, apf::Mesh2* m) {
+  apf::MeshEntity* e;
+  apf::MeshTag* tag = m->createIntTag("physical_type", 1);
+  for(int dim=2; dim<=m->getDimension(); dim++) {
+    int* tagPtr;
+    if(dim==2) tagPtr = r->physicalTypeFace.data();
+    if(dim==3) tagPtr = r->physicalTypeRgn.data();
+    int i = 0;
+    apf::MeshIterator* it = m->begin(dim);
+    while ((e = m->iterate(it)))
+      m->setIntTag(e, tag, &tagPtr[i++]);
+    m->end(it);
+  }
+}
+
 static const double gmshTet10EdgeIndices[6] = {0, 1, 2, 3, 5, 4};
 
 static int getQuadGmshIdx(const int apfIdx, const int apfType) {
@@ -254,8 +279,9 @@ void readGmsh(apf::Mesh2* m, const char* filename)
   initReader(&r, m, filename);
   readNodes(&r);
   readElements(&r);
-  freeReader(&r);
   m->acceptChanges();
+  setElmPhysicalType(&r,m);
+  freeReader(&r);
   if (r.isQuadratic)
     readQuadratic(&r, m, filename);
 }

From b8170659a21646fe3eddfcdc87ba032a0c6adc54 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Thu, 16 Dec 2021 16:13:48 -0500
Subject: [PATCH 437/555] latest pumi-meshes

---
 pumi-meshes | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pumi-meshes b/pumi-meshes
index d8d3f38b4..d2be1da48 160000
--- a/pumi-meshes
+++ b/pumi-meshes
@@ -1 +1 @@
-Subproject commit d8d3f38b4d2c64245dbcf432251eedd51a671036
+Subproject commit d2be1da48f4b0a5fdcd07e1f4ef388c2fba9ae34

From a331d85076ca125f64ef9c9a2dacbb5aadaed6cb Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Thu, 16 Dec 2021 16:16:08 -0500
Subject: [PATCH 438/555] test gmsh to smb conversion

---
 test/testing.cmake | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/test/testing.cmake b/test/testing.cmake
index 31443373c..2e012f2c1 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -189,6 +189,13 @@ mpi_test(create_misSquare 1
   ${MESHES}/square/square.smb
   mis_test)
 
+set(MDIR ${MESHES}/gmsh)
+mpi_test(twoQuads 1
+  ./from_gmsh
+  ".null"
+  "${MDIR}/twoQuads.msh"
+  "${MDIR}/twoQuads.smb")
+
 set(MDIR ${MESHES}/ugrid)
 mpi_test(naca_ugrid 2
   ./from_ugrid

From 57c5f932d0087ffe0fea25ab939691b4daf0b552 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Thu, 16 Dec 2021 15:12:41 -0500
Subject: [PATCH 439/555] render the tag

---
 test/gmsh.cc | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/test/gmsh.cc b/test/gmsh.cc
index 307fab07e..75e209fca 100644
--- a/test/gmsh.cc
+++ b/test/gmsh.cc
@@ -3,11 +3,29 @@
 #include <gmi_mesh.h>
 #include <apfMDS.h>
 #include <apfMesh2.h>
+#include <apfShape.h>
 #include <PCU.h>
 #include <lionPrint.h>
 #include <cstdlib>
 #include <string.h>
 
+apf::Field* convert_my_tag(apf::Mesh* m, std::string name) {
+  apf::MeshTag* t = m->findTag(name.c_str());
+  apf::MeshEntity* elm;
+  apf::MeshIterator* it = m->begin(m->getDimension());
+  apf::Field* f = apf::createField(m, name.c_str(), apf::SCALAR, 
+                                   apf::getConstant(m->getDimension()));
+  int vals[1];
+  double vals_d[1];
+  while ((elm = m->iterate(it))) {
+    m->getIntTag(elm, t, vals);
+    vals_d[0] = vals[0];
+    apf::setComponents(f, elm, 0, vals_d);
+  }
+  m->end(it);
+  return f;
+}
+
 int main(int argc, char** argv)
 {
   MPI_Init(&argc,&argv);
@@ -26,6 +44,8 @@ int main(int argc, char** argv)
   if (std::string(argv[1]).compare(".null") == 0)
     apf::deriveMdsModel(m);
   m->verify();
+  convert_my_tag(m,"physical_type");
+  apf::writeVtkFiles("foo.vtu", m);
   m->writeNative(argv[3]);
   m->destroyNative();
   apf::destroyMesh(m);

From 72a07f93535dc9e55a8ae0f5a7255ba9b1083970 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Tue, 21 Dec 2021 12:38:28 -0500
Subject: [PATCH 440/555] store physical ent for mesh edges, faces, and rgns

---
 mds/mdsGmsh.cc | 33 ++++++++++++++++++---------------
 1 file changed, 18 insertions(+), 15 deletions(-)

diff --git a/mds/mdsGmsh.cc b/mds/mdsGmsh.cc
index a613f8e9c..a6b522d0c 100644
--- a/mds/mdsGmsh.cc
+++ b/mds/mdsGmsh.cc
@@ -55,8 +55,9 @@ struct Reader {
   bool isQuadratic;
   std::map<long, Node> nodeMap;
   std::map<long, apf::MeshEntity*> entMap[4];
-  std::vector<int> physicalTypeFace;
-  std::vector<int> physicalTypeRgn;
+  //the 0th vector is not used as mesh vertices don't have a 'physical entity'
+  //association in the legacy 2.* gmsh format
+  std::vector<int> physicalType[4];
 };
 
 void initReader(Reader* r, apf::Mesh2* m, const char* filename)
@@ -159,14 +160,20 @@ void readElement(Reader* r)
   int nverts = apf::Mesh::adjacentCount[apfType][0];
   int dim = apf::Mesh::typeDimension[apfType];
   long ntags = getLong(r);
+  /* The Gmsh 4.9 documentation on the legacy 2.* format states:
+   * "By default, the first tag is the tag of the physical entity to which the
+   * element belongs; the second is the tag of the elementary model entity to
+   * which the element belongs; the third is the number of mesh partitions to
+   * which the element belongs, followed by the partition ids (negative
+   * partition ids indicate ghost cells). A zero tag is equivalent to no tag.
+   * Gmsh and most codes using the MSH 2 format require at least the first two
+   * tags (physical and elementary tags)."
+   * A physical entity is a user defined grouping of elementary model entities.
+   * An elementary model entity is a geometric model entity. */
   PCU_ALWAYS_ASSERT(ntags >= 2);
   const int physType = static_cast<int>(getLong(r));
-  //tag only faces and regions
-  if(dim == 2) {
-    r->physicalTypeFace.push_back(physType);
-  } else if(dim == 3) {
-    r->physicalTypeRgn.push_back(physType);
-  }
+  PCU_ALWAYS_ASSERT(dim>=0 && dim<4);
+  r->physicalType[dim].push_back(physType);
   long gtag = getLong(r);
   for (long i = 2; i < ntags; ++i)
     getLong(r); /* discard all other element tags */
@@ -191,8 +198,6 @@ void readElements(Reader* r)
   seekMarker(r, "$Elements");
   long n = getLong(r);
   getLine(r);
-  r->physicalTypeFace.reserve(n);
-  r->physicalTypeRgn.reserve(n);
   for (long i = 0; i < n; ++i)
     readElement(r);
   checkMarker(r, "$EndElements");
@@ -200,11 +205,9 @@ void readElements(Reader* r)
 
 void setElmPhysicalType(Reader* r, apf::Mesh2* m) {
   apf::MeshEntity* e;
-  apf::MeshTag* tag = m->createIntTag("physical_type", 1);
-  for(int dim=2; dim<=m->getDimension(); dim++) {
-    int* tagPtr;
-    if(dim==2) tagPtr = r->physicalTypeFace.data();
-    if(dim==3) tagPtr = r->physicalTypeRgn.data();
+  apf::MeshTag* tag = m->createIntTag("gmsh_physical_entity", 1);
+  for(int dim=1; dim<=m->getDimension(); dim++) { //vertices don't have a physical entity ?
+    int* tagPtr = r->physicalType[dim].data();
     int i = 0;
     apf::MeshIterator* it = m->begin(dim);
     while ((e = m->iterate(it)))

From e8f38615e5200a854a43eb48112b5d8375dc3738 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 22 Dec 2021 08:38:35 -0500
Subject: [PATCH 441/555] skip dimensions without tags

---
 mds/mdsGmsh.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mds/mdsGmsh.cc b/mds/mdsGmsh.cc
index a6b522d0c..63e5ce463 100644
--- a/mds/mdsGmsh.cc
+++ b/mds/mdsGmsh.cc
@@ -207,6 +207,7 @@ void setElmPhysicalType(Reader* r, apf::Mesh2* m) {
   apf::MeshEntity* e;
   apf::MeshTag* tag = m->createIntTag("gmsh_physical_entity", 1);
   for(int dim=1; dim<=m->getDimension(); dim++) { //vertices don't have a physical entity ?
+    if( ! r->physicalType[dim].size() ) continue;
     int* tagPtr = r->physicalType[dim].data();
     int i = 0;
     apf::MeshIterator* it = m->begin(dim);

From 79e7fb48a54c2ec39bfd986e3a4e037d45495ba7 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 22 Dec 2021 08:39:06 -0500
Subject: [PATCH 442/555] debug: tag name change

---
 test/gmsh.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/gmsh.cc b/test/gmsh.cc
index 75e209fca..7674ea8fd 100644
--- a/test/gmsh.cc
+++ b/test/gmsh.cc
@@ -44,7 +44,7 @@ int main(int argc, char** argv)
   if (std::string(argv[1]).compare(".null") == 0)
     apf::deriveMdsModel(m);
   m->verify();
-  convert_my_tag(m,"physical_type");
+  convert_my_tag(m,"gmsh_physical_entity");
   apf::writeVtkFiles("foo.vtu", m);
   m->writeNative(argv[3]);
   m->destroyNative();

From f1657e39117a2bda23b1d54cd960f44c4c21eb1c Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Thu, 20 Jan 2022 16:28:05 -0500
Subject: [PATCH 443/555] sanity check change

all tests pass
---
 gmi/gmi_base.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/gmi/gmi_base.c b/gmi/gmi_base.c
index cfd6d4b21..73c142d80 100644
--- a/gmi/gmi_base.c
+++ b/gmi/gmi_base.c
@@ -12,6 +12,8 @@
 #include "agm.h"
 #include <stdlib.h>
 #include <string.h>
+#include <stdint.h>
+#include <assert.h>
 
 struct gmi_model_ops gmi_base_ops = {
   .begin    = gmi_base_begin,
@@ -39,15 +41,20 @@ struct gmi_ent* gmi_from_agm(struct agm_ent e)
 struct agm_ent agm_from_gmi(struct gmi_ent* e)
 {
   char* p;
-  int uid;
+  intptr_t uid;
+  int iuid;
   struct agm_ent a;
   p = (char*)e;
-  uid = p - ((char*)0);
+  iuid = p - ((char*)0);
+  uid = (intptr_t)e;
+  assert(iuid == uid);
   if (uid == 0) {
     a.type = 0;
     a.id = -1;
   } else {
     uid -= 1;
+    iuid -= 1;
+    assert(uid == iuid);
     a.type = uid % AGM_ENT_TYPES;
     a.id = uid / AGM_ENT_TYPES;
   }

From 9aed42ef4fb407cca7a4db1eddc29bbfa5d4e7d7 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Thu, 20 Jan 2022 16:29:45 -0500
Subject: [PATCH 444/555] use intptr_t #356

---
 gmi/gmi_base.c | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/gmi/gmi_base.c b/gmi/gmi_base.c
index 73c142d80..ad44e7be3 100644
--- a/gmi/gmi_base.c
+++ b/gmi/gmi_base.c
@@ -40,21 +40,14 @@ struct gmi_ent* gmi_from_agm(struct agm_ent e)
 
 struct agm_ent agm_from_gmi(struct gmi_ent* e)
 {
-  char* p;
   intptr_t uid;
-  int iuid;
   struct agm_ent a;
-  p = (char*)e;
-  iuid = p - ((char*)0);
   uid = (intptr_t)e;
-  assert(iuid == uid);
   if (uid == 0) {
     a.type = 0;
     a.id = -1;
   } else {
     uid -= 1;
-    iuid -= 1;
-    assert(uid == iuid);
     a.type = uid % AGM_ENT_TYPES;
     a.id = uid / AGM_ENT_TYPES;
   }

From 6716090be0bf5bab8f0d5ba52c1d30debeef7094 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 21 Jan 2022 16:29:57 -0500
Subject: [PATCH 445/555] Revert "debug: tag name change"

This reverts commit 79e7fb48a54c2ec39bfd986e3a4e037d45495ba7.
---
 test/gmsh.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/gmsh.cc b/test/gmsh.cc
index 7674ea8fd..75e209fca 100644
--- a/test/gmsh.cc
+++ b/test/gmsh.cc
@@ -44,7 +44,7 @@ int main(int argc, char** argv)
   if (std::string(argv[1]).compare(".null") == 0)
     apf::deriveMdsModel(m);
   m->verify();
-  convert_my_tag(m,"gmsh_physical_entity");
+  convert_my_tag(m,"physical_type");
   apf::writeVtkFiles("foo.vtu", m);
   m->writeNative(argv[3]);
   m->destroyNative();

From e6de0e1049e6f555462c989e35c57cc1bea4d1cc Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 21 Jan 2022 16:30:00 -0500
Subject: [PATCH 446/555] Revert "render the tag"

This reverts commit 57c5f932d0087ffe0fea25ab939691b4daf0b552.
---
 test/gmsh.cc | 20 --------------------
 1 file changed, 20 deletions(-)

diff --git a/test/gmsh.cc b/test/gmsh.cc
index 75e209fca..307fab07e 100644
--- a/test/gmsh.cc
+++ b/test/gmsh.cc
@@ -3,29 +3,11 @@
 #include <gmi_mesh.h>
 #include <apfMDS.h>
 #include <apfMesh2.h>
-#include <apfShape.h>
 #include <PCU.h>
 #include <lionPrint.h>
 #include <cstdlib>
 #include <string.h>
 
-apf::Field* convert_my_tag(apf::Mesh* m, std::string name) {
-  apf::MeshTag* t = m->findTag(name.c_str());
-  apf::MeshEntity* elm;
-  apf::MeshIterator* it = m->begin(m->getDimension());
-  apf::Field* f = apf::createField(m, name.c_str(), apf::SCALAR, 
-                                   apf::getConstant(m->getDimension()));
-  int vals[1];
-  double vals_d[1];
-  while ((elm = m->iterate(it))) {
-    m->getIntTag(elm, t, vals);
-    vals_d[0] = vals[0];
-    apf::setComponents(f, elm, 0, vals_d);
-  }
-  m->end(it);
-  return f;
-}
-
 int main(int argc, char** argv)
 {
   MPI_Init(&argc,&argv);
@@ -44,8 +26,6 @@ int main(int argc, char** argv)
   if (std::string(argv[1]).compare(".null") == 0)
     apf::deriveMdsModel(m);
   m->verify();
-  convert_my_tag(m,"physical_type");
-  apf::writeVtkFiles("foo.vtu", m);
   m->writeNative(argv[3]);
   m->destroyNative();
   apf::destroyMesh(m);

From 80d75bf3549aefe9b2d5424701aaf8cc7d07eaa2 Mon Sep 17 00:00:00 2001
From: Jacob Merson <jacob@jacobmerson.com>
Date: Thu, 17 Feb 2022 02:20:41 -0500
Subject: [PATCH 447/555] Fix RPATH to not use CMAKE_INSTALL_PREFIX

The use of CMAKE_INSTALL_PREFIX causes issues when setting the prefix at
install time using cmake --install with --prefix
---
 cmake/bob.cmake         | 16 +++++++++++-----
 test/smokeTesting.cmake |  3 ++-
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/cmake/bob.cmake b/cmake/bob.cmake
index 90be9078b..fb68a8dc9 100644
--- a/cmake/bob.cmake
+++ b/cmake/bob.cmake
@@ -1,3 +1,4 @@
+include(GNUInstallDirs)
 function(bob_always_full_rpath)
   # CMake RPATH "always full" configuration, see:
   # https://cmake.org/Wiki/CMake_RPATH_handling#Always_full_RPATH
@@ -6,12 +7,17 @@ function(bob_always_full_rpath)
   # when building, don't use the install RPATH already
   # (but later on when installing)
   set(CMAKE_BUILD_WITH_INSTALL_RPATH False PARENT_SCOPE)
-  # the RPATH to be used when installing, but only if it's not a system directory
-  list(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES
-       "${CMAKE_INSTALL_PREFIX}/lib" isSystemDir)
-  if("${isSystemDir}" STREQUAL "-1")
-    set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib" PARENT_SCOPE)
+
+  if(APPLE)
+      set(base @loader_path)
+  else()
+      set(base $ORIGIN)
   endif()
+  file(RELATIVE_PATH relDir
+       ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}
+       ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}
+  )
+  set(CMAKE_INSTALL_RPATH ${base} ${base}/${relDir} PARENT_SCOPE)
   # add the automatically determined parts of the RPATH
   # which point to directories outside the build tree to the install RPATH
   set(CMAKE_INSTALL_RPATH_USE_LINK_PATH True PARENT_SCOPE)
diff --git a/test/smokeTesting.cmake b/test/smokeTesting.cmake
index fe4259436..49a460dec 100644
--- a/test/smokeTesting.cmake
+++ b/test/smokeTesting.cmake
@@ -20,7 +20,8 @@ smoke_test(split_2 2
   "pipe_2_.smb"
   2)
 
+include(GNUInstallDirs)
 # install the test input files for use in spack post-install tests
 install(FILES "${MDIR}/pipe.dmg" "${MDIR}/pipe0.smb"
-        DESTINATION ${CMAKE_INSTALL_PREFIX}/testdata)
+  DESTINATION ${CMAKE_INSTALL_DATADIR}/testdata)
 

From c6bfeef79db758d2096ddad4b3e9b77585394fe3 Mon Sep 17 00:00:00 2001
From: Jacob Merson <jacob@jacobmerson.com>
Date: Tue, 15 Feb 2022 01:53:05 -0500
Subject: [PATCH 448/555] Add release build to github workflow

This commit cleans up the github workflow and adds a release build to
the build matrix. In addition it fixes an error where the correct
compiler was not being selected during build. Additionally, the
operating system has been updated to ubuntu-latest and gcc-10 is used.
---
 .github/workflows/cmake.yml | 49 ++++++++++++++++++-------------------
 1 file changed, 24 insertions(+), 25 deletions(-)

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index ec3ced6d9..b3c5550fd 100644
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -7,40 +7,39 @@ on:
 jobs:
   build:
     # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
-        compiler: [g++, clang++]
+        compiler:
+          - { compiler: GNU, CC: gcc-10, CXX: g++-10 }
+          - { compiler: LLVM, CC: clang, CXX: clang++ }
+        build_type: [Debug, Release]
 
     steps:
     - uses: actions/checkout@v2
+      with:
+        submodules: recursive
     
-    - name: install mpich
-      run: sudo apt install mpich
-
-    - name: Create Build Environment
-      run: cmake -E make_directory ${{runner.workspace}}/build
-
-    - name: download meshes
-      run: git submodule init && git submodule update
+    - name: install mpich and gcc
+      run: |
+           sudo apt update
+           sudo apt install gcc-10 g++-10 mpich
 
     - name: Configure CMake
-      shell: bash
-      working-directory: ${{runner.workspace}}/build
-      run: export MPICXX=${{ matrix.compiler }} && 
-           cmake $GITHUB_WORKSPACE
-           -DCMAKE_CXX_COMPILER=mpicxx
-           -DCMAKE_C_COMPILER=mpicc
-           -DCMAKE_VERBOSE_MAKEFILE=on
-           -DMESHES=${GITHUB_WORKSPACE}/pumi-meshes
-           -DIS_TESTING=ON
+      env:
+        MPICH_CXX: ${{matrix.compiler.CXX}}
+        MPICH_CC: ${{matrix.compiler.CC}}
+      run: cmake -S ${{github.workspace}} -B ${{github.workspace}}/build -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_C_COMPILER=mpicc -DCMAKE_VERBOSE_MAKEFILE=ON -DMESHES=${{github.workspace}}/pumi-meshes -DIS_TESTING=ON -DCMAKE_BUILD_TYPE=${{matrix.build_type}}
 
     - name: Build
-      working-directory: ${{runner.workspace}}/build
-      shell: bash
-      run: cmake --build .
+      env:
+        MPICH_CXX: ${{matrix.compiler.CXX}}
+        MPICH_CC: ${{matrix.compiler.CC}}
+      run: cmake --build ${{github.workspace}}/build --config ${{matrix.build_type}} -j
 
     - name: Test
-      working-directory: ${{runner.workspace}}/build
-      shell: bash
-      run: ctest --output-on-failure
+      env:
+        MPICH_CXX: ${{matrix.compiler.CXX}}
+        MPICH_CC: ${{matrix.compiler.CC}}
+      working-directory: ${{github.workspace}}/build
+      run: ctest --output-on-failure -C ${{matrix.build_type}}

From babf73b8c8ee5ef46f49f2c9dd5ff7e5a3e907bd Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 2 Mar 2022 14:19:49 -0500
Subject: [PATCH 449/555] increase default omegah version to 10

we still support version 9 from the sandia repo
---
 CMakeLists.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 20fc39fe9..961506a67 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -93,6 +93,7 @@ option(SKIP_SIMMETRIX_VERSION_CHECK "enable at your own risk; it may result in u
 option(ENABLE_SIMMETRIX "Build with Simmetrix support" OFF)
 message(STATUS "ENABLE_SIMMETRIX: ${ENABLE_SIMMETRIX}")
 option(ENABLE_OMEGA_H "Enable the Omega_h interface" OFF)
+option(PUMI_USE_OMEGA_H_VERSION "Specify the Omega_h version PUMI should use" 10.0.0)
 message(STATUS "ENABLE_OMEGA_H: ${ENABLE_OMEGA_H}")
 if(ENABLE_SIMMETRIX)
   add_definitions(-DHAVE_SIMMETRIX)
@@ -116,7 +117,7 @@ endif()
 if(ENABLE_OMEGA_H)
   # find the omega_h library
   set(SCOREC_USE_Omega_h_DEFAULT ${ENABLE_OMEGA_H})
-  set(Omega_h_REQUIRED_VERSION 9.0.0)
+  set(Omega_h_REQUIRED_VERSION ${PUMI_USE_OMEGA_H_VERSION})
   bob_public_dep(Omega_h)
 endif()
 

From 3f2476f1ff76900ee08e325e1e8d3ba4348357e2 Mon Sep 17 00:00:00 2001
From: Cameron Smith <cwsmith@users.noreply.github.com>
Date: Thu, 3 Mar 2022 08:52:19 -0500
Subject: [PATCH 450/555] actions: don't cancel at first failure

---
 .github/workflows/cmake.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index b3c5550fd..6ed2bd420 100644
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -9,6 +9,7 @@ jobs:
     # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         compiler:
           - { compiler: GNU, CC: gcc-10, CXX: g++-10 }

From d0767f993ab0ee614a441c2867b118c8192598c0 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Thu, 3 Mar 2022 19:42:10 -0500
Subject: [PATCH 451/555] byte swap test for doubles #360

---
 test/CMakeLists.txt |  1 +
 test/swapDoubles.cc | 29 +++++++++++++++++++++++++++++
 test/testing.cmake  |  1 +
 3 files changed, 31 insertions(+)
 create mode 100644 test/swapDoubles.cc

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 835135020..d21dbb3ae 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -133,6 +133,7 @@ endif()
 util_exe_func(print_pumipic_partition print_pumipic_partition.cc)
 
 # Unit tests / functionality regression tests
+test_exe_func(swapDoubles swapDoubles.cc)
 test_exe_func(qr qr.cc)
 test_exe_func(eigen_test eigen_test.cc)
 test_exe_func(integrate integrate.cc)
diff --git a/test/swapDoubles.cc b/test/swapDoubles.cc
new file mode 100644
index 000000000..beacc27d2
--- /dev/null
+++ b/test/swapDoubles.cc
@@ -0,0 +1,29 @@
+#include <PCU.h>
+#include <pcu_io.h> //pcu_swap_doubles
+#include <pcu_util.h> //PCU_ALWAYS_ASSERT
+#include <numeric> //iota
+#include <iostream> //cerr
+
+int main(int argc, char** argv) {
+  MPI_Init(&argc,&argv);
+  PCU_Comm_Init();
+  const size_t n = 2;
+  double *d_orig = new double[n];
+  std::iota(d_orig,d_orig+n,0);
+  double *d = new double[n];
+  std::iota(d,d+n,0);
+  pcu_swap_doubles(d, n);
+  pcu_swap_doubles(d, n);
+  for(size_t i=0; i<n; i++) {
+    if(d[i] != d_orig[i]) {
+      std::cerr << "ERROR: swap does not match"
+                << " d[" << i << "] = " << d[i]
+                << " d_orig[" << i << "] = " << d_orig[i] << "\n";
+
+    }
+    PCU_ALWAYS_ASSERT(d[i]==d_orig[i]);
+  }
+  delete [] d_orig;
+  delete [] d;
+  return 0;
+}
diff --git a/test/testing.cmake b/test/testing.cmake
index 2e012f2c1..de1120f82 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -32,6 +32,7 @@ mpi_test(align 1 ./align)
 mpi_test(eigen_test 1 ./eigen_test)
 mpi_test(integrate 1 ./integrate)
 mpi_test(qr_test 1 ./qr)
+mpi_test(swapDoubles 1 ./swapDoubles)
 mpi_test(base64 1 ./base64)
 mpi_test(tensor_test 1 ./tensor)
 mpi_test(verify_convert 1 ./verify_convert)

From 830be9b64c6eac7287acd392bd5b44441cf0e604 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Thu, 3 Mar 2022 20:33:22 -0500
Subject: [PATCH 452/555] borrow a simpler byteswap from omegah #360

from src/Omega_h_file.cpp
---
 pcu/pcu_io.c | 46 +++++++++++++++++++++++++---------------------
 1 file changed, 25 insertions(+), 21 deletions(-)

diff --git a/pcu/pcu_io.c b/pcu/pcu_io.c
index 01462ac4d..a706f6155 100644
--- a/pcu/pcu_io.c
+++ b/pcu/pcu_io.c
@@ -244,31 +244,35 @@ static const uint16_t pcu_endian_value = 1;
 #define PCU_BIG_ENDIAN 0
 #define PCU_ENCODED_ENDIAN PCU_BIG_ENDIAN //consistent with network byte order
 
-static void pcu_swap_16(uint16_t* p)
-{
-  uint8_t* p2 = (uint8_t*)p;
-  uint8_t temp = p2[0];
-  p2[0] = p2[1];
-  p2[1] = temp;
-}
-
 static void pcu_swap_32(uint32_t* p)
 {
-  uint16_t* p2 = (uint16_t*)p;
-  uint16_t temp = p2[0];
-  p2[0] = p2[1];
-  p2[1] = temp;
-  pcu_swap_16(p2);
-  pcu_swap_16(p2+1);
+  uint32_t a = *p;
+#if defined(__GNUC__)
+  a = __builtin_bswap32(a);
+#elif defined(_MSC_VER)
+  a = _byteswap_ulong(a);
+#else
+  a = ((a & 0x000000FF) << 24) | ((a & 0x0000FF00) << 8) |
+      ((a & 0x00FF0000) >> 8) | ((a & 0xFF000000) >> 24);
+#endif
+  *p = a;
 }
 
-static void pcu_swap_64(uint32_t* p)
+static void pcu_swap_64(uint64_t* p)
 {
-  uint32_t temp = p[0];
-  p[0] = p[1];
-  p[1] = temp;
-  pcu_swap_32(p);
-  pcu_swap_32(p+1);
+  uint64_t a = *p;
+#if defined(__GNUC__) && !defined(__CUDA_ARCH__)
+  a = __builtin_bswap64(a);
+#elif defined(_MSC_VER) && !defined(__CUDA_ARCH__)
+  a = _byteswap_uint64(a);
+#else
+  a = ((a & 0x00000000000000FFULL) << 56) |
+      ((a & 0x000000000000FF00ULL) << 40) |
+      ((a & 0x0000000000FF0000ULL) << 24) | ((a & 0x00000000FF000000ULL) << 8) |
+      ((a & 0x000000FF00000000ULL) >> 8) | ((a & 0x0000FF0000000000ULL) >> 24) |
+      ((a & 0x00FF000000000000ULL) >> 40) | ((a & 0xFF00000000000000ULL) >> 56);
+#endif
+  *p = a;
 }
 
 void pcu_swap_unsigneds(unsigned* p, size_t n)
@@ -282,7 +286,7 @@ void pcu_swap_doubles(double* p, size_t n)
 {
   PCU_ALWAYS_ASSERT(sizeof(double)==8);
   for (size_t i=0; i < n; ++i)
-    pcu_swap_64((uint32_t*)(p++));
+    pcu_swap_64((uint64_t*)(p++));
 }
 
 void pcu_write_unsigneds(pcu_file* f, unsigned* p, size_t n)

From 72cb33f05948259ee8d628fc8ea5e5915aa35342 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Mon, 14 Mar 2022 12:34:23 -0400
Subject: [PATCH 453/555] mds: increase smb version #360

update chef reference meshes
---
 mds/mds_smb.c | 2 +-
 pumi-meshes   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/mds/mds_smb.c b/mds/mds_smb.c
index 5a6898957..f0590d679 100644
--- a/mds/mds_smb.c
+++ b/mds/mds_smb.c
@@ -22,7 +22,7 @@
 #include <sys/stat.h> /*using POSIX mkdir call for SMB "foo/" path*/
 #include <errno.h> /* for checking the error from mkdir */
 
-enum { SMB_VERSION = 5 };
+enum { SMB_VERSION = 6 };
 
 enum {
   SMB_VERT,
diff --git a/pumi-meshes b/pumi-meshes
index d2be1da48..98f48ca2c 160000
--- a/pumi-meshes
+++ b/pumi-meshes
@@ -1 +1 @@
-Subproject commit d2be1da48f4b0a5fdcd07e1f4ef388c2fba9ae34
+Subproject commit 98f48ca2cb8f1f3ad3db0c5fe4ad1c8df1f20811

From a295720d7b4828282484f2b78bac1f6504512de4 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Mon, 14 Mar 2022 14:52:27 -0400
Subject: [PATCH 454/555] pumi version 2.2.7

see issue #357
---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 961506a67..3bcb909a7 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -6,7 +6,7 @@ endif()
 # This is the top level CMake file for the SCOREC build
 cmake_minimum_required(VERSION 3.0)
 
-project(SCOREC VERSION 2.2.6 LANGUAGES CXX C)
+project(SCOREC VERSION 2.2.7 LANGUAGES CXX C)
 
 include(cmake/bob.cmake)
 include(cmake/xsdk.cmake)

From 1e604ecc4cbed9ff7bd562c78781d6fe9c039acc Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Thu, 31 Mar 2022 10:32:59 -0400
Subject: [PATCH 455/555] disable warnings/errors by default

- see request in #363
- explicitly enable warnings/errors in CI and nightly builds
---
 .github/workflows/cmake.yml | 2 +-
 cdash/nightly.cmake         | 1 +
 cmake/bob.cmake             | 2 +-
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index 6ed2bd420..39ed4a701 100644
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -30,7 +30,7 @@ jobs:
       env:
         MPICH_CXX: ${{matrix.compiler.CXX}}
         MPICH_CC: ${{matrix.compiler.CC}}
-      run: cmake -S ${{github.workspace}} -B ${{github.workspace}}/build -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_C_COMPILER=mpicc -DCMAKE_VERBOSE_MAKEFILE=ON -DMESHES=${{github.workspace}}/pumi-meshes -DIS_TESTING=ON -DCMAKE_BUILD_TYPE=${{matrix.build_type}}
+      run: cmake -S ${{github.workspace}} -B ${{github.workspace}}/build -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_C_COMPILER=mpicc -DCMAKE_VERBOSE_MAKEFILE=ON -DMESHES=${{github.workspace}}/pumi-meshes -DIS_TESTING=ON -DSCOREC_CXX_WARNINGS=ON -DCMAKE_BUILD_TYPE=${{matrix.build_type}}
 
     - name: Build
       env:
diff --git a/cdash/nightly.cmake b/cdash/nightly.cmake
index 8bd6d01c8..32fcbec69 100644
--- a/cdash/nightly.cmake
+++ b/cdash/nightly.cmake
@@ -305,6 +305,7 @@ SET(CONFIGURE_OPTIONS
   "-DENABLE_ZOLTAN:BOOL=ON"
   "-DPCU_COMPRESS:BOOL=ON"
   "-DIS_TESTING:BOOL=True"
+  "-DSCOREC_CXX_WARNINGS:BOOL=ON"
   "-DMESHES:STRING=${MESHES}/meshes"
 )
 
diff --git a/cmake/bob.cmake b/cmake/bob.cmake
index fb68a8dc9..5478c9ffc 100644
--- a/cmake/bob.cmake
+++ b/cmake/bob.cmake
@@ -50,7 +50,7 @@ endmacro(bob_set_shared_libs)
 function(bob_begin_cxx_flags)
   option(${PROJECT_NAME}_CXX_OPTIMIZE "Compile C++ with optimization" ON)
   option(${PROJECT_NAME}_CXX_SYMBOLS "Compile C++ with debug symbols" ON)
-  option(${PROJECT_NAME}_CXX_WARNINGS "Compile C++ with warnings" ON)
+  option(${PROJECT_NAME}_CXX_WARNINGS "Compile C++ with warnings" OFF)
   set(FLAGS "")
   if(${PROJECT_NAME}_CXX_OPTIMIZE)
     if (NOT ("${CMAKE_BUILD_TYPE}" STREQUAL "Release"))

From 72ffbab05185a88b10bed14b6aa5a2cf62165daa Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimm2@rpi.edu>
Date: Tue, 12 Apr 2022 15:16:36 -0400
Subject: [PATCH 456/555] Adds a utility to construct the cavities in a mesh

---
 test/makeAllCavities.cc | 153 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 153 insertions(+)
 create mode 100644 test/makeAllCavities.cc

diff --git a/test/makeAllCavities.cc b/test/makeAllCavities.cc
new file mode 100644
index 000000000..d425ff2bc
--- /dev/null
+++ b/test/makeAllCavities.cc
@@ -0,0 +1,153 @@
+#include <apf.h>
+#include <apfMDS.h>
+#include <gmi_mesh.h>
+#include <PCU.h>
+#include <pcu_util.h>
+#include <apfDynamicVector.h>
+#include <apfDynamicMatrix.h>
+#include <crv.h>
+#include <cassert>
+#include <stdlib.h>
+#include <sstream>
+#include <fstream>
+#ifdef HAVE_SIMMETRIX
+#include <gmi_sim.h>
+#include <SimUtil.h>
+#include <MeshSim.h>
+#include <SimModel.h>
+#endif
+
+// === includes for safe_mkdir ===
+#include <reel.h>
+#include <sys/types.h> /*required for mode_t for mkdir on some systems*/
+#include <sys/stat.h> /*using POSIX mkdir call for SMB "foo/" path*/
+#include <errno.h> /* for checking the error from mkdir */
+
+
+static void safe_mkdir(
+    const char* path);
+static apf::Mesh2* makeEntMesh(
+    apf::Mesh2* m,
+    apf::MeshEntity* e);
+static void makeCavityMeshes(
+    apf::Mesh2* m,
+    apf::MeshEntity* e,
+    apf::Mesh2* cavityMeshLinear,
+    apf::Mesh2* cavityMeshCurved);
+
+int main(int argc, char** argv)
+{
+
+  MPI_Init(&argc,&argv);
+  PCU_Comm_Init();
+  if (PCU_Comm_Peers() > 1) {
+    printf("%s should only be used for serial (single part) meshes!\n", argv[0]);
+    printf("use the serialize utility to get a serial mesh, and retry!\n");
+  }
+  if (argc < 5) {
+    if (PCU_Comm_Self() == 0) {
+      printf("USAGE: %s <model> <mesh> <enttype> <prefix>\n", argv[0]);
+    }
+    MPI_Finalize();
+    exit(EXIT_FAILURE);
+  }
+
+#ifdef HAVE_SIMMETRIX
+  MS_init();
+  SimModel_start();
+  Sim_readLicenseFile(0);
+  gmi_sim_start();
+  gmi_register_sim();
+#endif
+
+  const char* modelFile = argv[1];
+  const char* meshFile = argv[2];
+  apf::Mesh::Type enttype = atoi(argv[3]);
+  const char* prefix = argv[4];
+
+  // load the mesh and get the order
+  apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile);
+  int order = m->getShape()->getOrder();
+
+  int dim = m->getDimension(); // mesh dimension
+  int d = apf::Mesh::typeDimension[enttype]; // the dim we iterate on to create cavities
+
+  apf::MeshEntity* e;
+  apf::MeshIterator it = m->begin(d);
+  int index = 0;
+
+  // for now cavities are defined as follows
+  // all the upward adjacent entities of dimension "dim" that
+  // are adjacent to the verts of the "e". E.g., in case of edges,
+  // this would give us the bi-directional edge collapse cavity.
+  while ( (e = m->iterate(it)) ) {
+    int etype = m->getType(e);
+    int mtype = m->getModelType(m->toModel(e));
+    apf::Mesh2* entMesh = makeEntMesh(m, e);
+    apf::Mesh2* cavityMeshCurved = 0;
+    apf::Mesh2* cavityMeshLinear = 0;
+    makeCavityMeshes(m, e, cavityMeshLinear, cavityMeshCurved);
+
+    // write the curved cavity mesh in native format for future retrieval
+
+    // write the entMesh and cavityMeshLinear and cavityMeshCurved in vtk format
+
+    // destroy the entMesh and cavityMeshes
+    destroyMesh(entMesh);
+    destroyMesh(cavityMeshLinear);
+    destroyMesh(cavityMeshCurved);
+    index++;
+  }
+  m->end(it);
+
+
+  safe_mkdir(inPrefix);
+
+  // rest of the clean up
+  m->destroyNative();
+  apf::destroyMesh(m);
+
+#ifdef HAVE_SIMMETRIX
+  gmi_sim_stop();
+  Sim_unregisterAllKeys();
+  SimModel_stop();
+  MS_exit();
+#endif
+
+  PCU_Comm_Free();
+  MPI_Finalize();
+}
+
+static void safe_mkdir(
+    const char* path)
+{
+  mode_t const mode = S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH;
+  int err;
+  errno = 0;
+  err = mkdir(path, mode);
+  if (err != 0 && errno != EEXIST)
+  {
+    reel_fail("Err: could not create directory \"%s\"\n", path);
+  }
+}
+
+static apf::Mesh2* makeEntMesh(
+    apf::Mesh2* m,
+    apf::MeshEntity* e)
+{
+}
+
+static void makeCavityMeshes(
+    apf::Mesh2* m,
+    apf::MeshEntity* e,
+    apf::Mesh2* cavityMeshLinear,
+    apf::Mesh2* cavityMeshCurved)
+{
+  PCU_ALWAYS_ASSERT(!cavityMeshLinear);
+  PCU_ALWAYS_ASSERT(!cavityMeshCurved);
+
+  int dim = m->getDimension();
+
+  cavityMeshLinear = apf::makeEmptyMdsMesh(0, dim, false);
+  cavityMeshCurved = apf::makeEmptyMdsMesh(0, dim, false);
+}

From 2964b073b785685821ba14005fecb15a6c9db913 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimm2@rpi.edu>
Date: Tue, 12 Apr 2022 20:07:00 -0400
Subject: [PATCH 457/555] Makes list of entities in the cavity using adjacency

---
 test/makeAllCavities.cc | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/test/makeAllCavities.cc b/test/makeAllCavities.cc
index d425ff2bc..e34d9923f 100644
--- a/test/makeAllCavities.cc
+++ b/test/makeAllCavities.cc
@@ -148,6 +148,22 @@ static void makeCavityMeshes(
 
   int dim = m->getDimension();
 
+  std::set<apf::MeshEntity*> cavity;
+  cavity.clear();
+
+  apf::Downward dv;
+  int nv = m->getDownward(e, 0, dv);
+  for (int i = 0; i < nv; i++) {
+    apf::Adjacent adj;
+    m->getAdjacent(dv[i], dim, adj);
+    APF_ITERATE(apf::Adjacent, adj, it)
+      if (!cavity.count(*it))
+      	cavity.insert(*it);
+  }
+
+
+
+
   cavityMeshLinear = apf::makeEmptyMdsMesh(0, dim, false);
   cavityMeshCurved = apf::makeEmptyMdsMesh(0, dim, false);
 }

From bcf1d629193ca851d7c10e3afe08e330d3edfcb4 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimm2@rpi.edu>
Date: Wed, 13 Apr 2022 21:03:21 -0400
Subject: [PATCH 458/555] Adds cavity maker to the CMake

---
 test/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index d21dbb3ae..256e58671 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -195,6 +195,7 @@ endif()
 if(ENABLE_SIMMETRIX)
   test_exe_func(curvetest curvetest.cc)
   test_exe_func(curve_to_bezier curve_to_bezier.cc)
+  test_exe_func(make_all_cavities makeAllCavities.cc)
   test_exe_func(inClosureOf_test inClosureOf_test.cc)
   test_exe_func(degenerate_test degenerateSurfs.cc)
   test_exe_func(simZBalance simZBalance.cc)

From 53bd5e2f3c181470406b27414833287f8260274c Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimm2@rpi.edu>
Date: Wed, 13 Apr 2022 21:03:38 -0400
Subject: [PATCH 459/555] Adds routines to make a mesh of driving enity

---
 test/makeAllCavities.cc | 567 +++++++++++++++++++++++++++++++++++++---
 1 file changed, 527 insertions(+), 40 deletions(-)

diff --git a/test/makeAllCavities.cc b/test/makeAllCavities.cc
index e34d9923f..f7add4e2a 100644
--- a/test/makeAllCavities.cc
+++ b/test/makeAllCavities.cc
@@ -1,6 +1,7 @@
 #include <apf.h>
 #include <apfMDS.h>
 #include <gmi_mesh.h>
+#include <gmi_null.h>
 #include <PCU.h>
 #include <pcu_util.h>
 #include <apfDynamicVector.h>
@@ -10,6 +11,8 @@
 #include <stdlib.h>
 #include <sstream>
 #include <fstream>
+#include <algorithm>
+#include <vector>
 #ifdef HAVE_SIMMETRIX
 #include <gmi_sim.h>
 #include <SimUtil.h>
@@ -24,16 +27,22 @@
 #include <errno.h> /* for checking the error from mkdir */
 
 
-static void safe_mkdir(
-    const char* path);
-static apf::Mesh2* makeEntMesh(
-    apf::Mesh2* m,
-    apf::MeshEntity* e);
+/* static void safe_mkdir( */
+/*     const char* path); */
+/* static apf::Mesh2* makeEntMesh( */
+/*     apf::Mesh2* m, */
+/*     apf::MeshEntity* e); */
 static void makeCavityMeshes(
     apf::Mesh2* m,
     apf::MeshEntity* e,
-    apf::Mesh2* cavityMeshLinear,
-    apf::Mesh2* cavityMeshCurved);
+    apf::Mesh2* &cavityMeshLinear,
+    apf::Mesh2* &cavityMeshCurved);
+
+static void makeEntMeshes(
+    apf::Mesh2* m,
+    apf::MeshEntity* e,
+    apf::Mesh2* &entMeshLinear,
+    apf::Mesh2* &entMeshCurved);
 
 int main(int argc, char** argv)
 {
@@ -44,7 +53,7 @@ int main(int argc, char** argv)
     printf("%s should only be used for serial (single part) meshes!\n", argv[0]);
     printf("use the serialize utility to get a serial mesh, and retry!\n");
   }
-  if (argc < 5) {
+  if (argc < 6) {
     if (PCU_Comm_Self() == 0) {
       printf("USAGE: %s <model> <mesh> <enttype> <prefix>\n", argv[0]);
     }
@@ -60,20 +69,27 @@ int main(int argc, char** argv)
   gmi_register_sim();
 #endif
 
+  gmi_register_null();
+
   const char* modelFile = argv[1];
-  const char* meshFile = argv[2];
-  apf::Mesh::Type enttype = atoi(argv[3]);
-  const char* prefix = argv[4];
+  const char* meshFile  = argv[2];
+  int         enttype   = atoi(argv[3]);
+  const char* prefix    = argv[4];
+
+  printf("%s\n", prefix);
 
   // load the mesh and get the order
   apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile);
-  int order = m->getShape()->getOrder();
+  m->changeShape(crv::getBezier(3), true);
+  /* int order = m->getShape()->getOrder(); */
+  crv::writeCurvedVtuFiles(m, apf::Mesh::TRIANGLE, 10, "in_mesh");
+  crv::writeCurvedWireFrame(m, 10, "in_mesh");
 
-  int dim = m->getDimension(); // mesh dimension
+  /* int dim = m->getDimension(); // mesh dimension */
   int d = apf::Mesh::typeDimension[enttype]; // the dim we iterate on to create cavities
 
   apf::MeshEntity* e;
-  apf::MeshIterator it = m->begin(d);
+  apf::MeshIterator* it = m->begin(d);
   int index = 0;
 
   // for now cavities are defined as follows
@@ -81,27 +97,61 @@ int main(int argc, char** argv)
   // are adjacent to the verts of the "e". E.g., in case of edges,
   // this would give us the bi-directional edge collapse cavity.
   while ( (e = m->iterate(it)) ) {
-    int etype = m->getType(e);
-    int mtype = m->getModelType(m->toModel(e));
-    apf::Mesh2* entMesh = makeEntMesh(m, e);
+    if (index != atoi(argv[5])) {
+      index++;
+      continue;
+    }
+    /* int etype = m->getType(e); */
+    /* int mtype = m->getModelType(m->toModel(e)); */
+
     apf::Mesh2* cavityMeshCurved = 0;
     apf::Mesh2* cavityMeshLinear = 0;
     makeCavityMeshes(m, e, cavityMeshLinear, cavityMeshCurved);
 
+    apf::Mesh2* entMeshCurved = 0;
+    apf::Mesh2* entMeshLinear = 0;
+    makeEntMeshes(m, e, entMeshLinear, entMeshCurved);
+
+    apf::writeVtkFiles("cavity_linear", cavityMeshLinear);
+    crv::writeCurvedVtuFiles(cavityMeshCurved, apf::Mesh::TRIANGLE, 10, "cavity_curved");
+    crv::writeCurvedWireFrame(cavityMeshCurved, 10, "cavity_curved");
+
+    if (m->getType(e) == apf::Mesh::VERTEX)
+    {
+      apf::writeVtkFiles("entity_linear", entMeshLinear);
+      apf::writeVtkFiles("entity_curved", entMeshCurved);
+    }
+    else if (m->getType(e) == apf::Mesh::EDGE)
+    {
+      apf::writeVtkFiles("entity_linear", entMeshLinear);
+      /* crv::writeCurvedVtuFiles(entMeshCurved, apf::Mesh::TRIANGLE, 10, "entity_curved"); */
+      crv::writeCurvedWireFrame(entMeshCurved, 10, "entity_curved");
+    }
+    else if (m->getType(e) == apf::Mesh::TRIANGLE)
+    {
+      apf::writeVtkFiles("entity_linear", entMeshLinear);
+      crv::writeCurvedVtuFiles(entMeshCurved, apf::Mesh::TRIANGLE, 10, "entity_curved");
+      crv::writeCurvedWireFrame(entMeshCurved, 10, "entity_curved");
+    }
+    else
+      PCU_ALWAYS_ASSERT(0);
+
     // write the curved cavity mesh in native format for future retrieval
 
     // write the entMesh and cavityMeshLinear and cavityMeshCurved in vtk format
 
     // destroy the entMesh and cavityMeshes
-    destroyMesh(entMesh);
-    destroyMesh(cavityMeshLinear);
-    destroyMesh(cavityMeshCurved);
+    /* destroyMesh(entMesh); */
+    cavityMeshLinear->destroyNative();
+    cavityMeshCurved->destroyNative();
+    apf::destroyMesh(cavityMeshLinear);
+    apf::destroyMesh(cavityMeshCurved);
     index++;
   }
   m->end(it);
 
 
-  safe_mkdir(inPrefix);
+  /* safe_mkdir(inPrefix); */
 
   // rest of the clean up
   m->destroyNative();
@@ -118,52 +168,489 @@ int main(int argc, char** argv)
   MPI_Finalize();
 }
 
-static void safe_mkdir(
-    const char* path)
+/* static void safe_mkdir( */
+/*     const char* path) */
+/* { */
+/*   mode_t const mode = S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH; */
+/*   int err; */
+/*   errno = 0; */
+/*   err = mkdir(path, mode); */
+/*   if (err != 0 && errno != EEXIST) */
+/*   { */
+/*     reel_fail("Err: could not create directory \"%s\"\n", path); */
+/*   } */
+/* } */
+
+/* static apf::Mesh2* makeEntMesh( */
+/*     apf::Mesh2* m, */
+/*     apf::MeshEntity* e) */
+/* { */
+/*   return 0; */
+/* } */
+
+static apf::Vector3 getEdgeCenter(
+    apf::Mesh2* m,
+    apf::MeshEntity* e)
 {
-  mode_t const mode = S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH;
-  int err;
-  errno = 0;
-  err = mkdir(path, mode);
-  if (err != 0 && errno != EEXIST)
-  {
-    reel_fail("Err: could not create directory \"%s\"\n", path);
+  PCU_ALWAYS_ASSERT(m->getType(e) == apf::Mesh::EDGE);
+  apf::MeshEntity* dv[2];
+  m->getDownward(e, 0, dv);
+  apf::Vector3 center(0., 0., 0.);
+  for (int i = 0; i < 2; i++) {
+    apf::Vector3 p;
+    m->getPoint(dv[i], 0, p);
+    center = center + p;
   }
+  center = center * (0.5);
+  return center;
 }
 
-static apf::Mesh2* makeEntMesh(
+static apf::Mesh2*  makePoint(
     apf::Mesh2* m,
     apf::MeshEntity* e)
 {
+  PCU_ALWAYS_ASSERT(m->getType(e) == apf::Mesh::VERTEX);
+  apf::Mesh2* sphMesh = apf::makeEmptyMdsMesh(gmi_load(".null"), 1, false);
+  double xrange[2] = {1.e16, -1.e16};
+  double yrange[2] = {1.e16, -1.e16};
+  double zrange[2] = {1.e16, -1.e16};
+
+  apf::Adjacent adj;
+  m->getAdjacent(e, 1, adj);
+
+  for (int i = 0; i < (int)adj.getSize(); i++) {
+    apf::Vector3 p = getEdgeCenter(m, adj[i]);
+    if (p[0] < xrange[0]) xrange[0] = p[0];
+    if (p[0] > xrange[1]) xrange[1] = p[0];
+
+    if (p[1] < yrange[0]) yrange[0] = p[1];
+    if (p[1] > yrange[1]) yrange[1] = p[1];
+
+    if (p[2] < zrange[0]) zrange[0] = p[2];
+    if (p[2] > zrange[1]) zrange[1] = p[2];
+  }
+
+  double minsize = 1.e16;
+  if (xrange[1]-xrange[0] < minsize) minsize = xrange[1] - xrange[0];
+  if (yrange[1]-yrange[0] < minsize) minsize = yrange[1] - yrange[0];
+  if (zrange[1]-zrange[0] < minsize) minsize = zrange[1] - zrange[0];
+
+  double radius = minsize / 20.;
+  apf::Vector3 center;
+  m->getPoint(e, 0, center);
+
+  // z = 0 plane
+  int n = 20;
+  const double pi = 3.141595;
+  const apf::Vector3 param(0., 0., 0.);
+  std::vector<apf::MeshEntity*> vs;
+  vs.clear();
+  for (int i = 0; i < n; i++) {
+    apf::Vector3 p(0., 0., 0.);
+    p[0] = center[0] + radius * std::cos(i*pi/n);
+    p[1] = center[1] + radius * std::sin(i*pi/n);
+    p[2] = center[2];
+    apf::MeshEntity* newV = sphMesh->createVertex(m->toModel(e), p, param);
+    vs.push_back(newV);
+  }
+
+  for (int i = 0; i < n; i++) {
+    apf::MeshEntity* dv[2];
+    dv[0] = vs[i];
+    dv[1] = vs[(i+1)%20];
+    sphMesh->createEntity(apf::Mesh::EDGE, m->toModel(e), dv);
+  }
+
+  // y = 0 plane
+  vs.clear();
+  for (int i = 0; i < n; i++) {
+    apf::Vector3 p(0., 0., 0.);
+    p[0] = center[0] + radius * std::cos(i*pi/n);
+    p[1] = center[1];
+    p[2] = center[2] + radius * std::sin(i*pi/n);
+    apf::MeshEntity* newV = sphMesh->createVertex(m->toModel(e), p, param);
+    vs.push_back(newV);
+  }
+
+  for (int i = 0; i < n; i++) {
+    apf::MeshEntity* dv[2];
+    dv[0] = vs[i];
+    dv[1] = vs[(i+1)%20];
+    sphMesh->createEntity(apf::Mesh::EDGE, m->toModel(e), dv);
+  }
+
+  // x = 0 plane
+  vs.clear();
+  for (int i = 0; i < n; i++) {
+    apf::Vector3 p(0., 0., 0.);
+    p[0] = center[0];
+    p[1] = center[1] + radius * std::cos(i*pi/n);
+    p[2] = center[2] + radius * std::sin(i*pi/n);
+    apf::MeshEntity* newV = sphMesh->createVertex(m->toModel(e), p, param);
+    vs.push_back(newV);
+  }
+
+  for (int i = 0; i < n; i++) {
+    apf::MeshEntity* dv[2];
+    dv[0] = vs[i];
+    dv[1] = vs[(i+1)%20];
+    sphMesh->createEntity(apf::Mesh::EDGE, m->toModel(e), dv);
+  }
+  sphMesh->acceptChanges();
+  return sphMesh;
+}
+
+static void makeEntMeshes(
+    apf::Mesh2* m,
+    apf::MeshEntity* e,
+    apf::Mesh2* &entMeshLinear,
+    apf::Mesh2* &entMeshCurved)
+{
+  PCU_ALWAYS_ASSERT(!entMeshLinear);
+  PCU_ALWAYS_ASSERT(!entMeshCurved);
+
+  const apf::Vector3 param(0., 0., 0.);
+
+  if (m->getType(e) == apf::Mesh::VERTEX)
+  {
+    entMeshLinear = makePoint(m, e);
+    entMeshCurved = makePoint(m, e);
+    return;
+  }
+  if (m->getType(e) == apf::Mesh::EDGE)
+  {
+    entMeshLinear = apf::makeEmptyMdsMesh(gmi_load(".null"), 1, false);
+    entMeshCurved = apf::makeEmptyMdsMesh(gmi_load(".null"), 1, false);
+    apf::MeshEntity* vs[2];
+    m->getDownward(e, 0, vs);
+    apf::Vector3 p[2];
+    m->getPoint(vs[0], 0, p[0]);
+    m->getPoint(vs[1], 0, p[1]);
+    apf::MeshEntity* newVs[2];
+    newVs[0] = entMeshLinear->createVertex(0, p[0], param);
+    newVs[1] = entMeshLinear->createVertex(0, p[1], param);
+    entMeshLinear->createEntity(apf::Mesh::EDGE, 0, newVs);
+
+    apf::MeshEntity* newVsc[2];
+    newVsc[0] = entMeshCurved->createVertex(0, p[0], param);
+    newVsc[1] = entMeshCurved->createVertex(0, p[1], param);
+    apf::MeshEntity* edge = entMeshCurved->createEntity(apf::Mesh::EDGE, 0, newVsc);
+
+    apf::FieldShape* fs = m->getShape();
+    entMeshCurved->changeShape(fs, true);
+    if (fs->countNodesOn(apf::Mesh::EDGE))
+    {
+      for (int i = 0; i < fs->countNodesOn(apf::Mesh::EDGE); i++) {
+	apf::Vector3 p;
+	m->getPoint(e, i, p);
+	entMeshCurved->setPoint(edge, i, p);
+      }
+    }
+    entMeshCurved->acceptChanges();
+    return;
+  }
+  if (m->getType(e) == apf::Mesh::TRIANGLE)
+  {
+    entMeshLinear = apf::makeEmptyMdsMesh(gmi_load(".null"), 2, false);
+    entMeshCurved = apf::makeEmptyMdsMesh(gmi_load(".null"), 2, false);
+    apf::MeshEntity* vs[3];
+    m->getDownward(e, 0, vs);
+    apf::Vector3 p[3];
+    m->getPoint(vs[0], 0, p[0]);
+    m->getPoint(vs[1], 0, p[1]);
+    m->getPoint(vs[2], 0, p[2]);
+    apf::MeshEntity* newVs[3];
+    newVs[0] = entMeshLinear->createVertex(0, p[0], param);
+    newVs[1] = entMeshLinear->createVertex(0, p[1], param);
+    newVs[2] = entMeshLinear->createVertex(0, p[2], param);
+    entMeshLinear->createEntity(apf::Mesh::TRIANGLE, 0, newVs);
+
+    apf::MeshEntity* newVsc[3];
+    newVsc[0] = entMeshCurved->createVertex(0, p[0], param);
+    newVsc[1] = entMeshCurved->createVertex(0, p[1], param);
+    newVsc[2] = entMeshCurved->createVertex(0, p[2], param);
+    apf::MeshEntity* face = entMeshCurved->createEntity(apf::Mesh::TRIANGLE, 0, newVsc);
+
+    apf::FieldShape* fs = m->getShape();
+    entMeshCurved->changeShape(fs, true);
+    if (fs->countNodesOn(apf::Mesh::TRIANGLE))
+    {
+      for (int i = 0; i < fs->countNodesOn(apf::Mesh::TRIANGLE); i++) {
+	apf::Vector3 p;
+	m->getPoint(e, i, p);
+	entMeshCurved->setPoint(face, i, p);
+      }
+    }
+    entMeshCurved->acceptChanges();
+    return;
+  }
 }
 
 static void makeCavityMeshes(
     apf::Mesh2* m,
     apf::MeshEntity* e,
-    apf::Mesh2* cavityMeshLinear,
-    apf::Mesh2* cavityMeshCurved)
+    apf::Mesh2* &cavityMeshLinear,
+    apf::Mesh2* &cavityMeshCurved)
 {
+  typedef std::vector<apf::MeshEntity*> Cavity;
+  typedef std::vector<apf::MeshEntity*>::iterator CavityIter;
   PCU_ALWAYS_ASSERT(!cavityMeshLinear);
   PCU_ALWAYS_ASSERT(!cavityMeshCurved);
 
   int dim = m->getDimension();
 
-  std::set<apf::MeshEntity*> cavity;
-  cavity.clear();
+  // input cavities
+  Cavity icavity3; // tets in the cavity
+  Cavity icavity2; // faces in the cavity
+  Cavity icavity1; // edges in the cavity
+  Cavity icavity0; // verts in the cavity
+  icavity3.clear();
+  icavity2.clear();
+  icavity1.clear();
+  icavity0.clear();
+
+
+  // connectivity tables
+  std::vector<std::vector<int>> tet_face;
+  std::vector<std::vector<int>> face_edge;
+  std::vector<std::vector<int>> edge_vert;
 
   apf::Downward dv;
   int nv = m->getDownward(e, 0, dv);
   for (int i = 0; i < nv; i++) {
     apf::Adjacent adj;
     m->getAdjacent(dv[i], dim, adj);
-    APF_ITERATE(apf::Adjacent, adj, it)
-      if (!cavity.count(*it))
-      	cavity.insert(*it);
+    for (int j = 0; j < (int)adj.getSize(); j++) {
+      if (std::find(icavity3.begin(), icavity3.end(), adj[j]) == icavity3.end())
+      	icavity3.push_back(adj[j]);
+    }
+  }
+
+
+  // construct the unique vector of faces and tet_face connectivity table
+  for (int i = 0; i < (int)icavity3.size(); i++) {
+    apf::Downward dents;
+    int nents = m->getDownward(icavity3[i], 2, dents);
+    for (int j = 0; j < nents; j++) {
+      if (std::find(icavity2.begin(), icavity2.end(), dents[j]) == icavity2.end())
+      	icavity2.push_back(dents[j]);
+    }
+    std::vector<int> conn;
+    for (int j = 0; j < nents; j++) {
+      CavityIter it = std::find(icavity2.begin(), icavity2.end(), dents[j]);
+      PCU_ALWAYS_ASSERT(it != icavity2.end());
+      conn.push_back(std::distance(icavity2.begin(), it));
+    }
+    PCU_ALWAYS_ASSERT((int)conn.size() == nents);
+    tet_face.push_back(conn);
   }
+  PCU_ALWAYS_ASSERT(icavity3.size() == tet_face.size());
+
+  // construct the unique vector of edges and face_edge connectivity table
+  for (int i = 0; i < (int)icavity2.size(); i++) {
+    apf::Downward dents;
+    int nents = m->getDownward(icavity2[i], 1, dents);
+    for (int j = 0; j < nents; j++) {
+      if (std::find(icavity1.begin(), icavity1.end(), dents[j]) == icavity1.end())
+      	icavity1.push_back(dents[j]);
+    }
+    std::vector<int> conn;
+    for (int j = 0; j < nents; j++) {
+      CavityIter it = std::find(icavity1.begin(), icavity1.end(), dents[j]);
+      PCU_ALWAYS_ASSERT(it != icavity1.end());
+      conn.push_back(std::distance(icavity1.begin(), it));
+    }
+    PCU_ALWAYS_ASSERT((int)conn.size() == nents);
+    face_edge.push_back(conn);
+  }
+  PCU_ALWAYS_ASSERT(icavity2.size() == face_edge.size());
+
+  // construct the unique vector of verts and edge_vert connectivity table
+  for (int i = 0; i < (int)icavity1.size(); i++) {
+    apf::Downward dents;
+    int nents = m->getDownward(icavity1[i], 0, dents);
+    for (int j = 0; j < nents; j++) {
+      if (std::find(icavity0.begin(), icavity0.end(), dents[j]) == icavity0.end())
+      	icavity0.push_back(dents[j]);
+    }
+    std::vector<int> conn;
+    for (int j = 0; j < nents; j++) {
+      CavityIter it = std::find(icavity0.begin(), icavity0.end(), dents[j]);
+      PCU_ALWAYS_ASSERT(it != icavity0.end());
+      conn.push_back(std::distance(icavity0.begin(), it));
+    }
+    PCU_ALWAYS_ASSERT((int)conn.size() == nents);
+    edge_vert.push_back(conn);
+  }
+  PCU_ALWAYS_ASSERT(icavity1.size() == edge_vert.size());
+
+
+  // output cavities
+  Cavity ocavity3linear; // tets in the cavity
+  Cavity ocavity2linear; // faces in the cavity
+  Cavity ocavity1linear; // edges in the cavity
+  Cavity ocavity0linear; // verts in the cavity
+  Cavity ocavity3curved; // tets in the cavity
+  Cavity ocavity2curved; // faces in the cavity
+  Cavity ocavity1curved; // edges in the cavity
+  Cavity ocavity0curved; // verts in the cavity
+  ocavity3linear.clear();
+  ocavity2linear.clear();
+  ocavity1linear.clear();
+  ocavity0linear.clear();
+  ocavity3curved.clear();
+  ocavity2curved.clear();
+  ocavity1curved.clear();
+  ocavity0curved.clear();
+
+
 
+  // construct the cavity meshes
+  // we do this in a bottom up fashion, ie vets first then edges, faces and tets
+  /* cavityMeshLinear = apf::makeEmptyMdsMesh(m->getModel(), dim, false); */
+  /* cavityMeshCurved = apf::makeEmptyMdsMesh(m->getModel(), dim, false); */
+  cavityMeshLinear = apf::makeEmptyMdsMesh(gmi_load(".null"), dim, false);
+  cavityMeshCurved = apf::makeEmptyMdsMesh(gmi_load(".null"), dim, false);
 
+  // verts
+  for (int i = 0; i < (int) icavity0.size(); i++) {
+    apf::MeshEntity* ent = icavity0[i];
+    apf::ModelEntity* c = m->toModel(ent);
+    apf::Vector3 coords;
+    apf::Vector3 params;
+    m->getPoint(ent, 0, coords);
+    m->getParam(ent, params);
 
+    apf::MeshEntity* newEnt = cavityMeshLinear->createVertex(c, coords, params);
+    ocavity0linear.push_back(newEnt);
+
+    apf::MeshEntity* newEntc = cavityMeshCurved->createVertex(c, coords, params);
+    ocavity0curved.push_back(newEntc);
+  }
+  PCU_ALWAYS_ASSERT(icavity0.size() == ocavity0linear.size());
+  PCU_ALWAYS_ASSERT(icavity0.size() == ocavity0curved.size());
+
+  // edges
+  for (int i = 0; i < (int) icavity1.size(); i++) {
+    apf::MeshEntity* ent = icavity1[i];
+    apf::ModelEntity* c = m->toModel(ent);
+    apf::MeshEntity* downv[2];
+    downv[0] = ocavity0linear[edge_vert[i][0]];
+    downv[1] = ocavity0linear[edge_vert[i][1]];
+    apf::MeshEntity* newEnt = cavityMeshLinear->createEntity(
+    	apf::Mesh::EDGE, c, downv);
+    ocavity1linear.push_back(newEnt);
+
+    downv[0] = ocavity0curved[edge_vert[i][0]];
+    downv[1] = ocavity0curved[edge_vert[i][1]];
+    apf::MeshEntity* newEntc = cavityMeshCurved->createEntity(
+    	apf::Mesh::EDGE, c, downv);
+    ocavity1curved.push_back(newEntc);
+  }
+  PCU_ALWAYS_ASSERT(icavity1.size() == ocavity1linear.size());
+  PCU_ALWAYS_ASSERT(icavity1.size() == ocavity1curved.size());
+
+  // faces
+  for (int i = 0; i < (int) icavity2.size(); i++) {
+    apf::MeshEntity* ent = icavity2[i];
+    apf::ModelEntity* c = m->toModel(ent);
+    apf::MeshEntity* downe[3];
+    downe[0] = ocavity1linear[face_edge[i][0]];
+    downe[1] = ocavity1linear[face_edge[i][1]];
+    downe[2] = ocavity1linear[face_edge[i][2]];
+    apf::MeshEntity* newEnt = cavityMeshLinear->createEntity(
+    	apf::Mesh::TRIANGLE, c, downe);
+    ocavity2linear.push_back(newEnt);
+
+    downe[0] = ocavity1curved[face_edge[i][0]];
+    downe[1] = ocavity1curved[face_edge[i][1]];
+    downe[2] = ocavity1curved[face_edge[i][2]];
+    apf::MeshEntity* newEntc = cavityMeshCurved->createEntity(
+    	apf::Mesh::TRIANGLE, c, downe);
+    ocavity2curved.push_back(newEntc);
+  }
+  PCU_ALWAYS_ASSERT(icavity2.size() == ocavity2linear.size());
+  PCU_ALWAYS_ASSERT(icavity2.size() == ocavity2curved.size());
+
+
+  // tets
+  for (int i = 0; i < (int) icavity3.size(); i++) {
+    apf::MeshEntity* ent = icavity3[i];
+    apf::ModelEntity* c = m->toModel(ent);
+    apf::MeshEntity* downf[4];
+    downf[0] = ocavity2linear[tet_face[i][0]];
+    downf[1] = ocavity2linear[tet_face[i][1]];
+    downf[2] = ocavity2linear[tet_face[i][2]];
+    downf[3] = ocavity2linear[tet_face[i][3]];
+    apf::MeshEntity* newEnt = cavityMeshLinear->createEntity(
+    	apf::Mesh::TET, c, downf);
+    ocavity3linear.push_back(newEnt);
+
+    downf[0] = ocavity2curved[tet_face[i][0]];
+    downf[1] = ocavity2curved[tet_face[i][1]];
+    downf[2] = ocavity2curved[tet_face[i][2]];
+    downf[3] = ocavity2curved[tet_face[i][3]];
+    apf::MeshEntity* newEntc = cavityMeshCurved->createEntity(
+    	apf::Mesh::TET, c, downf);
+    ocavity3curved.push_back(newEntc);
+  }
+  PCU_ALWAYS_ASSERT(icavity3.size() == ocavity3linear.size());
+  PCU_ALWAYS_ASSERT(icavity3.size() == ocavity3curved.size());
+
+
+  cavityMeshLinear->acceptChanges();
+  /* cavityMeshLinear->verify(); */
+
+  cavityMeshCurved->acceptChanges();
+  /* cavityMeshCurved->verify(); */
+
+  // curve cavityMeshCurved
+  // this can be done by
+  // a) setting the shape of cavityMeshCurved to that of m
+  // b) setting the node coordinates of the entities
+  // in ocavity{1,2,3}curved to those of entities
+  // in icavity{1,2,3}
+  cavityMeshCurved->changeShape(m->getShape(), true);
+  apf::FieldShape* fs = cavityMeshCurved->getShape();
+
+  int nnodes = fs->countNodesOn(apf::Mesh::TET);
+  if (nnodes) {
+    for (int i = 0; i < (int)icavity3.size(); i++) {
+      apf::MeshEntity* fromEnt = icavity3[i];
+      apf::MeshEntity* toEnt   = ocavity3curved[i];
+      for (int j = 0; j < nnodes; j++) {
+	apf::Vector3 p;
+	m->getPoint(fromEnt, j, p);
+	cavityMeshCurved->setPoint(toEnt, j, p);
+      }
+    }
+  }
+
+  nnodes = fs->countNodesOn(apf::Mesh::TRIANGLE);
+  if (nnodes) {
+    for (int i = 0; i < (int)icavity2.size(); i++) {
+      apf::MeshEntity* fromEnt = icavity2[i];
+      apf::MeshEntity* toEnt   = ocavity2curved[i];
+      for (int j = 0; j < nnodes; j++) {
+	apf::Vector3 p;
+	m->getPoint(fromEnt, j, p);
+	cavityMeshCurved->setPoint(toEnt, j, p);
+      }
+    }
+  }
+
+  nnodes = fs->countNodesOn(apf::Mesh::EDGE);
+  if (nnodes) {
+    for (int i = 0; i < (int)icavity1.size(); i++) {
+      apf::MeshEntity* fromEnt = icavity1[i];
+      apf::MeshEntity* toEnt   = ocavity1curved[i];
+      for (int j = 0; j < nnodes; j++) {
+	apf::Vector3 p;
+	m->getPoint(fromEnt, j, p);
+	cavityMeshCurved->setPoint(toEnt, j, p);
+      }
+    }
+  }
 
-  cavityMeshLinear = apf::makeEmptyMdsMesh(0, dim, false);
-  cavityMeshCurved = apf::makeEmptyMdsMesh(0, dim, false);
+  cavityMeshCurved->acceptChanges();
 }

From bf349ffc907e773607d8dbe7428688bb6eab545b Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimm2@rpi.edu>
Date: Fri, 22 Apr 2022 09:36:27 -0400
Subject: [PATCH 460/555] A working version of makeAllCavities

---
 test/makeAllCavities.cc | 367 +++++++++++++++++++++++++++-------------
 1 file changed, 249 insertions(+), 118 deletions(-)

diff --git a/test/makeAllCavities.cc b/test/makeAllCavities.cc
index f7add4e2a..d34a683b0 100644
--- a/test/makeAllCavities.cc
+++ b/test/makeAllCavities.cc
@@ -27,11 +27,9 @@
 #include <errno.h> /* for checking the error from mkdir */
 
 
-/* static void safe_mkdir( */
-/*     const char* path); */
-/* static apf::Mesh2* makeEntMesh( */
-/*     apf::Mesh2* m, */
-/*     apf::MeshEntity* e); */
+static void safe_mkdir(
+    const char* path);
+
 static void makeCavityMeshes(
     apf::Mesh2* m,
     apf::MeshEntity* e,
@@ -44,6 +42,14 @@ static void makeEntMeshes(
     apf::Mesh2* &entMeshLinear,
     apf::Mesh2* &entMeshCurved);
 
+static void writeMeshes(
+    apf::Mesh2* m,
+    const char* prefix0,
+    const char* prefix1,
+    const char* prefix2,
+    const char* name,
+    int res = 10);
+
 int main(int argc, char** argv)
 {
 
@@ -53,10 +59,15 @@ int main(int argc, char** argv)
     printf("%s should only be used for serial (single part) meshes!\n", argv[0]);
     printf("use the serialize utility to get a serial mesh, and retry!\n");
   }
-  if (argc < 6) {
-    if (PCU_Comm_Self() == 0) {
-      printf("USAGE: %s <model> <mesh> <enttype> <prefix>\n", argv[0]);
-    }
+  if (argc < 5) {
+    printf("USAGE: %s <model> <mesh> <enttype> <prefix> <tagname:optional>\n", argv[0]);
+    printf("CASE1:if tagname IS given && such a tag exists in the mesh \n");
+    printf("      then enttype is ignored and cavities are made for all \n");
+    printf("      the entities of dim 0,1,2 that have the tag.\n");
+    printf("CASE2:if tagname IS NOT given \n");
+    printf("      then cavities are made for all the entities \n");
+    printf("      of dim enttype.\n");
+    printf("NOTE: for large meshes users are encouraged to use CASE1!\n");
     MPI_Finalize();
     exit(EXIT_FAILURE);
   }
@@ -76,79 +87,86 @@ int main(int argc, char** argv)
   int         enttype   = atoi(argv[3]);
   const char* prefix    = argv[4];
 
-  printf("%s\n", prefix);
 
-  // load the mesh and get the order
-  apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile);
-  m->changeShape(crv::getBezier(3), true);
-  /* int order = m->getShape()->getOrder(); */
-  crv::writeCurvedVtuFiles(m, apf::Mesh::TRIANGLE, 10, "in_mesh");
-  crv::writeCurvedWireFrame(m, 10, "in_mesh");
 
-  /* int dim = m->getDimension(); // mesh dimension */
-  int d = apf::Mesh::typeDimension[enttype]; // the dim we iterate on to create cavities
+  // load the mesh and check if the tag exists on the mesh
+  apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile);
 
-  apf::MeshEntity* e;
-  apf::MeshIterator* it = m->begin(d);
-  int index = 0;
-
-  // for now cavities are defined as follows
-  // all the upward adjacent entities of dimension "dim" that
-  // are adjacent to the verts of the "e". E.g., in case of edges,
-  // this would give us the bi-directional edge collapse cavity.
-  while ( (e = m->iterate(it)) ) {
-    if (index != atoi(argv[5])) {
-      index++;
-      continue;
+  apf::MeshTag* tag = 0;
+  if (argc == 6) {
+    const char* tname = argv[5];
+    tag = m->findTag(tname);
+    if (!tag) {
+      printf("input mesh does not have the tag with name %s\n", tname);
+      printf("aborting!\n");
+      MPI_Finalize();
+      exit(EXIT_FAILURE);
     }
-    /* int etype = m->getType(e); */
-    /* int mtype = m->getModelType(m->toModel(e)); */
+  }
 
-    apf::Mesh2* cavityMeshCurved = 0;
-    apf::Mesh2* cavityMeshLinear = 0;
-    makeCavityMeshes(m, e, cavityMeshLinear, cavityMeshCurved);
+  // make the root directory to save the cavity info
+  safe_mkdir(prefix);
+  writeMeshes(m, prefix, "mesh", NULL, "linear");
+  // change the order of the mesh
+  m->changeShape(crv::getBezier(3), true);
+  writeMeshes(m, prefix, "mesh", NULL, "curved");
 
-    apf::Mesh2* entMeshCurved = 0;
-    apf::Mesh2* entMeshLinear = 0;
-    makeEntMeshes(m, e, entMeshLinear, entMeshCurved);
 
-    apf::writeVtkFiles("cavity_linear", cavityMeshLinear);
-    crv::writeCurvedVtuFiles(cavityMeshCurved, apf::Mesh::TRIANGLE, 10, "cavity_curved");
-    crv::writeCurvedWireFrame(cavityMeshCurved, 10, "cavity_curved");
+  int targetd = apf::Mesh::typeDimension[enttype]; // the dim we iterate on to create cavities
+
+  apf::MeshEntity* e;
+  apf::MeshIterator* it;
 
-    if (m->getType(e) == apf::Mesh::VERTEX)
-    {
-      apf::writeVtkFiles("entity_linear", entMeshLinear);
-      apf::writeVtkFiles("entity_curved", entMeshCurved);
-    }
-    else if (m->getType(e) == apf::Mesh::EDGE)
-    {
-      apf::writeVtkFiles("entity_linear", entMeshLinear);
-      /* crv::writeCurvedVtuFiles(entMeshCurved, apf::Mesh::TRIANGLE, 10, "entity_curved"); */
-      crv::writeCurvedWireFrame(entMeshCurved, 10, "entity_curved");
-    }
-    else if (m->getType(e) == apf::Mesh::TRIANGLE)
-    {
-      apf::writeVtkFiles("entity_linear", entMeshLinear);
-      crv::writeCurvedVtuFiles(entMeshCurved, apf::Mesh::TRIANGLE, 10, "entity_curved");
-      crv::writeCurvedWireFrame(entMeshCurved, 10, "entity_curved");
-    }
-    else
-      PCU_ALWAYS_ASSERT(0);
 
-    // write the curved cavity mesh in native format for future retrieval
+  for (int d = 0; d < 3; d++) {
+    if (!tag && d != targetd) continue;
 
-    // write the entMesh and cavityMeshLinear and cavityMeshCurved in vtk format
+    it = m->begin(d);
+    int index = 0;
 
-    // destroy the entMesh and cavityMeshes
-    /* destroyMesh(entMesh); */
-    cavityMeshLinear->destroyNative();
-    cavityMeshCurved->destroyNative();
-    apf::destroyMesh(cavityMeshLinear);
-    apf::destroyMesh(cavityMeshCurved);
-    index++;
+    // for now cavities are defined as follows
+    // all the upward adjacent entities of dimension "dim" that
+    // are adjacent to the verts of the "e". E.g., in case of edges,
+    // this would give us the bi-directional edge collapse cavity.
+    while ( (e = m->iterate(it)) ) {
+      if (tag && !m->hasTag(e, tag)) {
+      	index++;
+      	continue;
+      }
+      int etype = m->getType(e);
+
+      apf::Mesh2* cavityMeshCurved = 0;
+      apf::Mesh2* cavityMeshLinear = 0;
+      makeCavityMeshes(m, e, cavityMeshLinear, cavityMeshCurved);
+
+      apf::Mesh2* entMeshCurved = 0;
+      apf::Mesh2* entMeshLinear = 0;
+      makeEntMeshes(m, e, entMeshLinear, entMeshCurved);
+
+      char cavityFolderName[128];
+      char cavityFileNameLinear[128];
+      char cavityFileNameCurved[128];
+      char entityFileNameLinear[128];
+      char entityFileNameCurved[128];
+      sprintf(cavityFolderName, "%s_%05d", apf::Mesh::typeName[etype], index);
+      sprintf(cavityFileNameLinear, "%s", "cavity_linear");
+      sprintf(cavityFileNameCurved, "%s", "cavity_curved");
+      sprintf(entityFileNameLinear, "%s", "entity_linear");
+      sprintf(entityFileNameCurved, "%s", "entity_curved");
+      writeMeshes(cavityMeshLinear, prefix, "cavities", cavityFolderName, cavityFileNameLinear);
+      writeMeshes(cavityMeshCurved, prefix, "cavities", cavityFolderName, cavityFileNameCurved);
+
+      writeMeshes(entMeshLinear, prefix, "cavities", cavityFolderName, entityFileNameLinear);
+      writeMeshes(entMeshCurved, prefix, "cavities", cavityFolderName, entityFileNameCurved);
+
+      cavityMeshLinear->destroyNative();
+      cavityMeshCurved->destroyNative();
+      apf::destroyMesh(cavityMeshLinear);
+      apf::destroyMesh(cavityMeshCurved);
+      index++;
+    }
+    m->end(it);
   }
-  m->end(it);
 
 
   /* safe_mkdir(inPrefix); */
@@ -168,25 +186,18 @@ int main(int argc, char** argv)
   MPI_Finalize();
 }
 
-/* static void safe_mkdir( */
-/*     const char* path) */
-/* { */
-/*   mode_t const mode = S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH; */
-/*   int err; */
-/*   errno = 0; */
-/*   err = mkdir(path, mode); */
-/*   if (err != 0 && errno != EEXIST) */
-/*   { */
-/*     reel_fail("Err: could not create directory \"%s\"\n", path); */
-/*   } */
-/* } */
-
-/* static apf::Mesh2* makeEntMesh( */
-/*     apf::Mesh2* m, */
-/*     apf::MeshEntity* e) */
-/* { */
-/*   return 0; */
-/* } */
+static void safe_mkdir(
+    const char* path)
+{
+  mode_t const mode = S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH;
+  int err;
+  errno = 0;
+  err = mkdir(path, mode);
+  if (err != 0 && errno != EEXIST)
+  {
+    reel_fail("Err: could not create directory \"%s\"\n", path);
+  }
+}
 
 static apf::Vector3 getEdgeCenter(
     apf::Mesh2* m,
@@ -247,8 +258,8 @@ static apf::Mesh2*  makePoint(
   vs.clear();
   for (int i = 0; i < n; i++) {
     apf::Vector3 p(0., 0., 0.);
-    p[0] = center[0] + radius * std::cos(i*pi/n);
-    p[1] = center[1] + radius * std::sin(i*pi/n);
+    p[0] = center[0] + radius * std::cos(2.*i*pi/n);
+    p[1] = center[1] + radius * std::sin(2.*i*pi/n);
     p[2] = center[2];
     apf::MeshEntity* newV = sphMesh->createVertex(m->toModel(e), p, param);
     vs.push_back(newV);
@@ -265,9 +276,9 @@ static apf::Mesh2*  makePoint(
   vs.clear();
   for (int i = 0; i < n; i++) {
     apf::Vector3 p(0., 0., 0.);
-    p[0] = center[0] + radius * std::cos(i*pi/n);
+    p[0] = center[0] + radius * std::cos(2.*i*pi/n);
     p[1] = center[1];
-    p[2] = center[2] + radius * std::sin(i*pi/n);
+    p[2] = center[2] + radius * std::sin(2.*i*pi/n);
     apf::MeshEntity* newV = sphMesh->createVertex(m->toModel(e), p, param);
     vs.push_back(newV);
   }
@@ -284,8 +295,8 @@ static apf::Mesh2*  makePoint(
   for (int i = 0; i < n; i++) {
     apf::Vector3 p(0., 0., 0.);
     p[0] = center[0];
-    p[1] = center[1] + radius * std::cos(i*pi/n);
-    p[2] = center[2] + radius * std::sin(i*pi/n);
+    p[1] = center[1] + radius * std::cos(2.*i*pi/n);
+    p[2] = center[2] + radius * std::sin(2.*i*pi/n);
     apf::MeshEntity* newV = sphMesh->createVertex(m->toModel(e), p, param);
     vs.push_back(newV);
   }
@@ -297,6 +308,8 @@ static apf::Mesh2*  makePoint(
     sphMesh->createEntity(apf::Mesh::EDGE, m->toModel(e), dv);
   }
   sphMesh->acceptChanges();
+  apf::deriveMdsModel(sphMesh);
+  sphMesh->verify();
   return sphMesh;
 }
 
@@ -336,6 +349,14 @@ static void makeEntMeshes(
     newVsc[1] = entMeshCurved->createVertex(0, p[1], param);
     apf::MeshEntity* edge = entMeshCurved->createEntity(apf::Mesh::EDGE, 0, newVsc);
 
+    entMeshLinear->acceptChanges();
+    apf::deriveMdsModel(entMeshLinear);
+    entMeshLinear->verify();
+
+    entMeshCurved->acceptChanges();
+    apf::deriveMdsModel(entMeshCurved);
+    entMeshCurved->verify();
+
     apf::FieldShape* fs = m->getShape();
     entMeshCurved->changeShape(fs, true);
     if (fs->countNodesOn(apf::Mesh::EDGE))
@@ -353,34 +374,90 @@ static void makeEntMeshes(
   {
     entMeshLinear = apf::makeEmptyMdsMesh(gmi_load(".null"), 2, false);
     entMeshCurved = apf::makeEmptyMdsMesh(gmi_load(".null"), 2, false);
-    apf::MeshEntity* vs[3];
-    m->getDownward(e, 0, vs);
-    apf::Vector3 p[3];
-    m->getPoint(vs[0], 0, p[0]);
-    m->getPoint(vs[1], 0, p[1]);
-    m->getPoint(vs[2], 0, p[2]);
-    apf::MeshEntity* newVs[3];
-    newVs[0] = entMeshLinear->createVertex(0, p[0], param);
-    newVs[1] = entMeshLinear->createVertex(0, p[1], param);
-    newVs[2] = entMeshLinear->createVertex(0, p[2], param);
-    entMeshLinear->createEntity(apf::Mesh::TRIANGLE, 0, newVs);
+    apf::MeshEntity* downverts[3];
+    apf::MeshEntity* downedges[3];
+    m->getDownward(e, 0, downverts);
+    m->getDownward(e, 1, downedges);
+    int edge_vert[3][2];
+    for (int i = 0; i < 3; i++) {
+      apf::MeshEntity* dv[2];
+      m->getDownward(downedges[i], 0, dv);
+      int i0 = apf::findIn(downverts, 3, dv[0]);
+      int i1 = apf::findIn(downverts, 3, dv[1]);
+      PCU_ALWAYS_ASSERT(i0 != -1);
+      PCU_ALWAYS_ASSERT(i1 != -1);
+      edge_vert[i][0] = i0;
+      edge_vert[i][1] = i1;
+    }
+
+    apf::MeshEntity* newvertsLinear[3];
+    apf::MeshEntity* newedgesLinear[3];
+    apf::MeshEntity* newvertsCurved[3];
+    apf::MeshEntity* newedgesCurved[3];
+    apf::Vector3 param(0.,0.,0.);
+    for (int i = 0; i < 3; i++) {
+      apf::Vector3 p;
+      m->getPoint(downverts[i], 0, p);
+      newvertsLinear[i] = entMeshLinear->createVertex(0, p, param);
+      newvertsCurved[i] = entMeshCurved->createVertex(0, p, param);
+    }
+
+    for (int i = 0; i < 3; i++) {
+      apf::MeshEntity* evLinear[2] = {
+      	newvertsLinear[edge_vert[i][0]],
+      	newvertsLinear[edge_vert[i][1]]
+      };
+      newedgesLinear[i] = entMeshLinear->createEntity(
+    	apf::Mesh::EDGE, 0, evLinear);
+
+      apf::MeshEntity* evCurved[2] = {
+      	newvertsCurved[edge_vert[i][0]],
+      	newvertsCurved[edge_vert[i][1]]
+      };
+      newedgesCurved[i] = entMeshCurved->createEntity(
+    	apf::Mesh::EDGE, 0, evCurved);
+    }
 
-    apf::MeshEntity* newVsc[3];
-    newVsc[0] = entMeshCurved->createVertex(0, p[0], param);
-    newVsc[1] = entMeshCurved->createVertex(0, p[1], param);
-    newVsc[2] = entMeshCurved->createVertex(0, p[2], param);
-    apf::MeshEntity* face = entMeshCurved->createEntity(apf::Mesh::TRIANGLE, 0, newVsc);
+
+    entMeshLinear->createEntity(
+    	apf::Mesh::TRIANGLE, 0, newedgesLinear);
+
+    apf::MeshEntity* face =
+    entMeshCurved->createEntity(
+    	apf::Mesh::TRIANGLE, 0, newedgesCurved);
+
+
+    entMeshLinear->acceptChanges();
+    apf::deriveMdsModel(entMeshLinear);
+    entMeshLinear->verify();
+
+    entMeshCurved->acceptChanges();
+    apf::deriveMdsModel(entMeshCurved);
+    entMeshCurved->verify();
 
     apf::FieldShape* fs = m->getShape();
     entMeshCurved->changeShape(fs, true);
-    if (fs->countNodesOn(apf::Mesh::TRIANGLE))
-    {
-      for (int i = 0; i < fs->countNodesOn(apf::Mesh::TRIANGLE); i++) {
+
+    int nnodes = fs->countNodesOn(apf::Mesh::EDGE);
+    if (nnodes) {
+      for (int i = 0; i < 3; i++) {
+	for (int n = 0; n < nnodes; n++) {
+	  apf::Vector3 p;
+	  m->getPoint(downedges[i], n, p);
+	  entMeshCurved->setPoint(newedgesCurved[i], n, p);
+	}
+      }
+    }
+
+    nnodes = fs->countNodesOn(apf::Mesh::TRIANGLE);
+    if (nnodes) {
+      for (int n = 0; n < nnodes; n++) {
 	apf::Vector3 p;
-	m->getPoint(e, i, p);
-	entMeshCurved->setPoint(face, i, p);
+	m->getPoint(e, n, p);
+	entMeshCurved->setPoint(face, n, p);
       }
     }
+
     entMeshCurved->acceptChanges();
     return;
   }
@@ -599,10 +676,12 @@ static void makeCavityMeshes(
 
 
   cavityMeshLinear->acceptChanges();
-  /* cavityMeshLinear->verify(); */
+  apf::deriveMdsModel(cavityMeshLinear);
+  cavityMeshLinear->verify();
 
   cavityMeshCurved->acceptChanges();
-  /* cavityMeshCurved->verify(); */
+  apf::deriveMdsModel(cavityMeshCurved);
+  cavityMeshCurved->verify();
 
   // curve cavityMeshCurved
   // this can be done by
@@ -654,3 +733,55 @@ static void makeCavityMeshes(
 
   cavityMeshCurved->acceptChanges();
 }
+
+static void writeMeshes(
+    apf::Mesh2* m,
+    const char* prefix0,
+    const char* prefix1,
+    const char* prefix2,
+    const char* name,
+    int res)
+{
+  PCU_ALWAYS_ASSERT(prefix0);
+  PCU_ALWAYS_ASSERT(name);
+  if (!prefix1)
+    PCU_ALWAYS_ASSERT(!prefix2);
+
+  int order = m->getShape()->getOrder();
+  std::stringstream ss;
+  ss << prefix0 << "/";
+  if (prefix1) {
+    ss << prefix1;
+    safe_mkdir(ss.str().c_str());
+    ss << "/";
+  }
+  if (prefix2) {
+    ss << prefix2;
+    safe_mkdir(ss.str().c_str());
+    ss << "/";
+  }
+  ss << name;
+
+  // also check if the cavity is made from a vertex
+  bool isVertCavity = false;
+  if (prefix2) {
+    std::string st(prefix2);
+    std::string subst = st.substr(0,4);
+    if (subst.compare(std::string("vert")) == 0)
+      isVertCavity = true;
+  }
+
+  if (order == 1) {
+    apf::writeVtkFiles(ss.str().c_str(), m);
+  }
+  else {
+    if (isVertCavity)
+      apf::writeVtkFiles(ss.str().c_str(), m);
+    else {
+      crv::writeCurvedVtuFiles(m, apf::Mesh::TRIANGLE, res, ss.str().c_str());
+      crv::writeCurvedWireFrame(m, res, ss.str().c_str());
+    }
+  }
+  ss << ".smb";
+  m->writeNative(ss.str().c_str());
+}

From 835bf79ba54a581ea213a42d34661097ab8931f4 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimm2@rpi.edu>
Date: Fri, 22 Apr 2022 10:28:53 -0400
Subject: [PATCH 461/555] Adds input modes to the makeAllCavities utility

---
 test/makeAllCavities.cc | 131 ++++++++++++++++++++++++++++++----------
 1 file changed, 100 insertions(+), 31 deletions(-)

diff --git a/test/makeAllCavities.cc b/test/makeAllCavities.cc
index d34a683b0..f700403dc 100644
--- a/test/makeAllCavities.cc
+++ b/test/makeAllCavities.cc
@@ -50,6 +50,11 @@ static void writeMeshes(
     const char* name,
     int res = 10);
 
+static apf::MeshTag* tagMesh(
+    apf::Mesh2* m,
+    int dim,
+    int model);
+
 int main(int argc, char** argv)
 {
 
@@ -58,16 +63,25 @@ int main(int argc, char** argv)
   if (PCU_Comm_Peers() > 1) {
     printf("%s should only be used for serial (single part) meshes!\n", argv[0]);
     printf("use the serialize utility to get a serial mesh, and retry!\n");
+    MPI_Finalize();
+    exit(EXIT_FAILURE);
   }
-  if (argc < 5) {
-    printf("USAGE: %s <model> <mesh> <enttype> <prefix> <tagname:optional>\n", argv[0]);
-    printf("CASE1:if tagname IS given && such a tag exists in the mesh \n");
-    printf("      then enttype is ignored and cavities are made for all \n");
-    printf("      the entities of dim 0,1,2 that have the tag.\n");
-    printf("CASE2:if tagname IS NOT given \n");
-    printf("      then cavities are made for all the entities \n");
-    printf("      of dim enttype.\n");
-    printf("NOTE: for large meshes users are encouraged to use CASE1!\n");
+  if (argc != 6) {
+    printf("USAGE: %s <model> <mesh> <prefix> <resolution> <mode>\n", argv[0]);
+    printf("modes are as follows \n");
+    printf("aa: creates all vert, edge, face cavities\n");
+    printf("ai: creates all vert, edge, face cavities classified on interior\n");
+    printf("ab: creates all vert, edge, face cavities classified on boundary\n");
+    printf("va: creates all vert cavities\n");
+    printf("vi: creates all vert cavities classified on interior\n");
+    printf("vb: creates all vert cavities classified on boundary\n");
+    printf("ea: creates all edge cavities\n");
+    printf("ei: creates all edge cavities classified on interior\n");
+    printf("eb: creates all edge cavities classified on boundary\n");
+    printf("fa: creates all face cavities\n");
+    printf("fi: creates all face cavities classified on interior\n");
+    printf("fb: creates all face cavities classified on boundary\n");
+    printf("tagname: creates cavities for all entities that have tagname\n");
     MPI_Finalize();
     exit(EXIT_FAILURE);
   }
@@ -84,43 +98,65 @@ int main(int argc, char** argv)
 
   const char* modelFile = argv[1];
   const char* meshFile  = argv[2];
-  int         enttype   = atoi(argv[3]);
-  const char* prefix    = argv[4];
+  const char* prefix    = argv[3];
+  int         res       = atoi(argv[4]);
+  std::string mode(argv[5]);
 
+  apf::MeshTag* tag = 0;
 
 
   // load the mesh and check if the tag exists on the mesh
   apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile);
 
-  apf::MeshTag* tag = 0;
-  if (argc == 6) {
-    const char* tname = argv[5];
-    tag = m->findTag(tname);
+  if (mode.compare(std::string("aa")) == 0)
+    tag = tagMesh(m, -1, 1);
+  else if (mode.compare(std::string("ai")) == 0)
+    tag = tagMesh(m, -1, 2);
+  else if (mode.compare(std::string("ab")) == 0)
+    tag = tagMesh(m, -1, 3);
+  else if (mode.compare(std::string("va")) == 0)
+    tag = tagMesh(m, 0, 1);
+  else if (mode.compare(std::string("vi")) == 0)
+    tag = tagMesh(m, 0, 2);
+  else if (mode.compare(std::string("vb")) == 0)
+    tag = tagMesh(m, 0, 3);
+  else if (mode.compare(std::string("ea")) == 0)
+    tag = tagMesh(m, 1, 1);
+  else if (mode.compare(std::string("ei")) == 0)
+    tag = tagMesh(m, 1, 2);
+  else if (mode.compare(std::string("eb")) == 0)
+    tag = tagMesh(m, 1, 3);
+  else if (mode.compare(std::string("fa")) == 0)
+    tag = tagMesh(m, 2, 1);
+  else if (mode.compare(std::string("fi")) == 0)
+    tag = tagMesh(m, 2, 2);
+  else if (mode.compare(std::string("fb")) == 0)
+    tag = tagMesh(m, 2, 3);
+  else {
+    tag = m->findTag(mode.c_str());
     if (!tag) {
-      printf("input mesh does not have the tag with name %s\n", tname);
-      printf("aborting!\n");
+      printf("tag with name %s was not found on the mesh. Aborting!\n", mode.c_str());
       MPI_Finalize();
       exit(EXIT_FAILURE);
     }
   }
 
+  PCU_ALWAYS_ASSERT(tag);
+
+
   // make the root directory to save the cavity info
   safe_mkdir(prefix);
-  writeMeshes(m, prefix, "mesh", NULL, "linear");
+  writeMeshes(m, prefix, "mesh", NULL, "linear", res);
   // change the order of the mesh
   m->changeShape(crv::getBezier(3), true);
-  writeMeshes(m, prefix, "mesh", NULL, "curved");
-
+  writeMeshes(m, prefix, "mesh", NULL, "curved", res);
 
-  int targetd = apf::Mesh::typeDimension[enttype]; // the dim we iterate on to create cavities
 
   apf::MeshEntity* e;
   apf::MeshIterator* it;
 
 
   for (int d = 0; d < 3; d++) {
-    if (!tag && d != targetd) continue;
-
     it = m->begin(d);
     int index = 0;
 
@@ -129,7 +165,7 @@ int main(int argc, char** argv)
     // are adjacent to the verts of the "e". E.g., in case of edges,
     // this would give us the bi-directional edge collapse cavity.
     while ( (e = m->iterate(it)) ) {
-      if (tag && !m->hasTag(e, tag)) {
+      if (!m->hasTag(e, tag)) {
       	index++;
       	continue;
       }
@@ -153,11 +189,15 @@ int main(int argc, char** argv)
       sprintf(cavityFileNameCurved, "%s", "cavity_curved");
       sprintf(entityFileNameLinear, "%s", "entity_linear");
       sprintf(entityFileNameCurved, "%s", "entity_curved");
-      writeMeshes(cavityMeshLinear, prefix, "cavities", cavityFolderName, cavityFileNameLinear);
-      writeMeshes(cavityMeshCurved, prefix, "cavities", cavityFolderName, cavityFileNameCurved);
+      writeMeshes(cavityMeshLinear, prefix, "cavities",
+      	  cavityFolderName, cavityFileNameLinear, res);
+      writeMeshes(cavityMeshCurved, prefix, "cavities",
+      	  cavityFolderName, cavityFileNameCurved, res);
 
-      writeMeshes(entMeshLinear, prefix, "cavities", cavityFolderName, entityFileNameLinear);
-      writeMeshes(entMeshCurved, prefix, "cavities", cavityFolderName, entityFileNameCurved);
+      writeMeshes(entMeshLinear, prefix, "cavities",
+      	  cavityFolderName, entityFileNameLinear, res);
+      writeMeshes(entMeshCurved, prefix, "cavities",
+      	  cavityFolderName, entityFileNameCurved, res);
 
       cavityMeshLinear->destroyNative();
       cavityMeshCurved->destroyNative();
@@ -168,9 +208,6 @@ int main(int argc, char** argv)
     m->end(it);
   }
 
-
-  /* safe_mkdir(inPrefix); */
-
   // rest of the clean up
   m->destroyNative();
   apf::destroyMesh(m);
@@ -785,3 +822,35 @@ static void writeMeshes(
   ss << ".smb";
   m->writeNative(ss.str().c_str());
 }
+
+static apf::MeshTag* tagMesh(
+    apf::Mesh2* m,
+    int dim,
+    int model)
+{
+  // model = 1 means all
+  // model = 2 means only interior
+  // model = 3 means only boundary
+  // dim = -1 tags all dims
+  // dim =  0 tags verts only
+  // dim =  1 tags edges only
+  // dim =  2 tags faces only
+  apf::MeshEntity* e;
+  apf::MeshIterator* it;
+  apf::MeshTag* t = m->createIntTag("which_ent", 1);
+  for (int d = 0; d < 3; d++) {
+    if (dim == 0 && d != 0) continue;
+    if (dim == 1 && d != 1) continue;
+    if (dim == 2 && d != 2) continue;
+    it = m->begin(d);
+    while ( (e = m->iterate(it)) ) {
+      int mtype = m->getModelType(m->toModel(e));
+      if (model == 2 && mtype !=3) continue;
+      if (model == 3 && mtype ==3) continue;
+      int val = 1; // the value does not matter
+      m->setIntTag(e, t, &val);
+    }
+    m->end(it);
+  }
+  return t;
+}

From bd2e7d784fe01cb8d00a7865544d5ea94f4c5389 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimm2@rpi.edu>
Date: Fri, 22 Apr 2022 11:43:07 -0400
Subject: [PATCH 462/555] Fixes memory leaks in makeAllCavities

---
 test/makeAllCavities.cc | 21 ++++++---------------
 1 file changed, 6 insertions(+), 15 deletions(-)

diff --git a/test/makeAllCavities.cc b/test/makeAllCavities.cc
index f700403dc..0f98f7716 100644
--- a/test/makeAllCavities.cc
+++ b/test/makeAllCavities.cc
@@ -199,6 +199,10 @@ int main(int argc, char** argv)
       writeMeshes(entMeshCurved, prefix, "cavities",
       	  cavityFolderName, entityFileNameCurved, res);
 
+      entMeshLinear->destroyNative();
+      entMeshCurved->destroyNative();
+      apf::destroyMesh(entMeshLinear);
+      apf::destroyMesh(entMeshCurved);
       cavityMeshLinear->destroyNative();
       cavityMeshCurved->destroyNative();
       apf::destroyMesh(cavityMeshLinear);
@@ -799,25 +803,12 @@ static void writeMeshes(
   }
   ss << name;
 
-  // also check if the cavity is made from a vertex
-  bool isVertCavity = false;
-  if (prefix2) {
-    std::string st(prefix2);
-    std::string subst = st.substr(0,4);
-    if (subst.compare(std::string("vert")) == 0)
-      isVertCavity = true;
-  }
-
   if (order == 1) {
     apf::writeVtkFiles(ss.str().c_str(), m);
   }
   else {
-    if (isVertCavity)
-      apf::writeVtkFiles(ss.str().c_str(), m);
-    else {
-      crv::writeCurvedVtuFiles(m, apf::Mesh::TRIANGLE, res, ss.str().c_str());
-      crv::writeCurvedWireFrame(m, res, ss.str().c_str());
-    }
+    crv::writeCurvedVtuFiles(m, apf::Mesh::TRIANGLE, res, ss.str().c_str());
+    crv::writeCurvedWireFrame(m, res, ss.str().c_str());
   }
   ss << ".smb";
   m->writeNative(ss.str().c_str());

From acbe1ba0ee7a56eecb8d3c4575d52826732651f1 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimm2@rpi.edu>
Date: Tue, 26 Apr 2022 14:20:23 -0400
Subject: [PATCH 463/555] Adds another option to cavity maker

---
 test/makeAllCavities.cc | 91 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 91 insertions(+)

diff --git a/test/makeAllCavities.cc b/test/makeAllCavities.cc
index 0f98f7716..3e11d44fb 100644
--- a/test/makeAllCavities.cc
+++ b/test/makeAllCavities.cc
@@ -55,6 +55,10 @@ static apf::MeshTag* tagMesh(
     int dim,
     int model);
 
+static apf::MeshTag* tagMesh(
+    apf::Mesh2* m,
+    const std::vector<std::string>& ids);
+
 int main(int argc, char** argv)
 {
 
@@ -81,6 +85,7 @@ int main(int argc, char** argv)
     printf("fa: creates all face cavities\n");
     printf("fi: creates all face cavities classified on interior\n");
     printf("fb: creates all face cavities classified on boundary\n");
+    printf("ls: get a list from user and creates cavities for that list\n");
     printf("tagname: creates cavities for all entities that have tagname\n");
     MPI_Finalize();
     exit(EXIT_FAILURE);
@@ -132,6 +137,31 @@ int main(int argc, char** argv)
     tag = tagMesh(m, 2, 2);
   else if (mode.compare(std::string("fb")) == 0)
     tag = tagMesh(m, 2, 3);
+  else if (mode.compare(std::string("ls")) == 0) {
+    std::cout << "provide the list of entities format \"v_i,e_i,f_i\"" << std::endl;
+    std::cout << "example: v120 e23 f0 represents vert 120," << std::endl;
+    std::cout << "edge 23 and face 0." << std::endl;
+    std::cout << "total #verts=" << m->count(0);
+    std::cout << ", #edges=" << m->count(1);
+    std::cout << ", #faces=" << m->count(2) << std::endl;
+    int cnt=0;
+    std::cout << "enter #of ents in the list:" << std::endl;
+    std::cin >> cnt;
+    std::vector<std::string> ents;
+    ents.clear();
+    while ((int)ents.size() < cnt) {
+      std::string temp;
+      std::cin >> temp;
+      ents.push_back(temp);
+    }
+    PCU_ALWAYS_ASSERT((int)ents.size() == cnt);
+    std::cout << "creating cavities for " << std::endl;
+    for (int i = 0; i < (int)ents.size(); i++)
+      std::cout << ents[i] << " ";
+    std::cout << std::endl;
+
+    tag = tagMesh(m, ents);
+  }
   else {
     tag = m->findTag(mode.c_str());
     if (!tag) {
@@ -845,3 +875,64 @@ static apf::MeshTag* tagMesh(
   }
   return t;
 }
+
+static void getEntIds(
+    const std::vector<std::string>& ids,
+    std::vector<int>& vids,
+    std::vector<int>& eids,
+    std::vector<int>& fids)
+{
+  vids.clear();
+  eids.clear();
+  fids.clear();
+
+  for (std::size_t i = 0; i < ids.size(); i++) {
+    std::string key = ids[i].substr(0,1);
+    int value = atoi(ids[i].substr(1).c_str());
+    if (key.compare(std::string("v")) == 0)
+      vids.push_back(value);
+    if (key.compare(std::string("e")) == 0)
+      eids.push_back(value);
+    if (key.compare(std::string("f")) == 0)
+      fids.push_back(value);
+  }
+
+}
+
+static apf::MeshTag* tagMesh(
+    apf::Mesh2* m,
+    const std::vector<std::string>& ids)
+{
+  std::vector<int> vids;
+  std::vector<int> eids;
+  std::vector<int> fids;
+  getEntIds(ids, vids, eids, fids);
+  PCU_ALWAYS_ASSERT(ids.size() == vids.size()+eids.size()+fids.size());
+
+  std::vector<int>::iterator vit;
+  apf::MeshEntity* e;
+  apf::MeshIterator* it;
+  apf::MeshTag* t = m->createIntTag("which_ent", 1);
+  for (int d = 0; d < 3; d++) {
+    int index = 0;
+    it = m->begin(d);
+    while ( (e = m->iterate(it)) ) {
+      bool found = false;
+      if (d == 0 &&
+      	  std::find(vids.begin(), vids.end(), index) != vids.end())
+      	found = true;
+      if (d == 1 &&
+      	  std::find(eids.begin(), eids.end(), index) != eids.end())
+      	found = true;
+      if (d == 2 &&
+      	  std::find(fids.begin(), fids.end(), index) != fids.end())
+      	found = true;
+      int val = 1; // the value does not matter
+      if (found)
+      	m->setIntTag(e, t, &val);
+      index++;
+    }
+    m->end(it);
+  }
+  return t;
+}

From 1e04fa676aa4c6d5e5ae26dd51c37407ecadfd37 Mon Sep 17 00:00:00 2001
From: "Morteza H. Siboni" <hakimm2@rpi.edu>
Date: Sat, 14 May 2022 11:09:12 -0400
Subject: [PATCH 464/555] Adds a new utility to compute Bezeir shape values

---
 test/CMakeLists.txt     |   1 +
 test/bezierShapeEval.cc | 214 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 215 insertions(+)
 create mode 100644 test/bezierShapeEval.cc

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 256e58671..a5307fb7c 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -65,6 +65,7 @@ endif()
 util_exe_func(measureAnisoStats measureAnisoStats.cc)
 util_exe_func(measureIsoStats measureIsoStats.cc)
 util_exe_func(visualizeAnisoSizes visualizeAnisoSizes.cc)
+test_exe_func(bezierShapeEval bezierShapeEval.cc)
 if(ENABLE_SIMMETRIX)
   util_exe_func(runSimxAnisoAdapt runSimxAnisoAdapt.cc)
 endif()
diff --git a/test/bezierShapeEval.cc b/test/bezierShapeEval.cc
new file mode 100644
index 000000000..07abf4785
--- /dev/null
+++ b/test/bezierShapeEval.cc
@@ -0,0 +1,214 @@
+#include <crv.h>
+#include <crvBezier.h>
+#include <crvTables.h>
+#include <crvSnap.h>
+#include <crvMath.h>
+#include <crvBezierShapes.h>
+#include <gmi_analytic.h>
+#include <gmi_null.h>
+#include <apfMDS.h>
+#include <apfMesh2.h>
+#include <apf.h>
+#include <apfShape.h>
+#include <PCU.h>
+#include <lionPrint.h>
+#include <mth.h>
+#include <mth_def.h>
+#include <pcu_util.h>
+#include <ostream>
+/* This file contains miscellaneous tests relating to bezier shapes and
+ * blended bezier shapes
+ */
+
+static apf::Mesh2* makeOneTriMesh(int order, apf::MeshEntity* &ent);
+static apf::Mesh2* makeOneTetMesh(int order, apf::MeshEntity* &ent);
+
+
+int main(int argc, char** argv)
+{
+  MPI_Init(&argc,&argv);
+  PCU_Comm_Init();
+  lion_set_verbosity(1);
+  if ( argc != 7 ) {
+    if ( !PCU_Comm_Self() ) {
+      printf("Usage: %s <ent_type> <order> <blend_order> <xi_0> <xi_1> <xi_2>>\n", argv[0]);
+      printf("<ent_type>            can only be 2 (for triangles) and 4 (for tets)\n");
+      printf("<order>               is the order of bezier\n");
+      printf("<blend_order>         can be -1, 0, 1, 2 (-1 means no blending)\n");
+      printf("<xi_0> <xi_1> <xi_2>  inquiry point in the parent entity)\n");
+    }
+    MPI_Finalize();
+    exit(EXIT_FAILURE);
+  }
+
+  int type  = atoi(argv[1]);
+  int order = atoi(argv[2]);
+  int b     = atoi(argv[3]);
+  PCU_ALWAYS_ASSERT_VERBOSE(type == 2 || type == 4,
+      "<ent_type> can only be 2 or 4!");
+  PCU_ALWAYS_ASSERT_VERBOSE(b > -2 &&  b < 3,
+      "<blend_order> must be between -1 and 2!");
+
+  apf::MeshEntity* ent = 0;
+  apf::Mesh2* m = type == 2 ? makeOneTriMesh(order,ent) : makeOneTetMesh(order,ent);
+
+  double xi0 = atof(argv[4]);
+  double xi1 = atof(argv[5]);
+  double xi2 = atof(argv[6]);
+
+
+  // set the order
+  apf::FieldShape* bezierShape = crv::getBezier(order);
+  int non = bezierShape->getEntityShape(type)->countNodes();
+  apf::Vector3 xi(xi0, xi1, xi2);
+  apf::NewArray<double> vals(non);
+
+  if (b == -1) // regular shape functions
+     bezierShape->getEntityShape(type)->getValues(m,ent,xi,vals);
+  else // blended shape functions
+  {
+    crv::setBlendingOrder(type, b);
+    if (type == 2)
+      crv::BlendedTriangleGetValues(m,ent,xi,vals);
+    else
+      crv::BlendedTetGetValues(m,ent,xi,vals);
+  }
+
+
+  printf("shape values are \n");
+  for (int i = 0; i < non; i++) {
+    printf("%d,  %E \n", i, vals[i]);
+  }
+
+  PCU_Comm_Free();
+  MPI_Finalize();
+}
+
+static apf::Mesh2* makeOneTriMesh(int order, apf::MeshEntity* &ent)
+{
+  apf::Mesh2* mesh = apf::makeEmptyMdsMesh(0, 2, false);
+
+  double vert_coords[3][6] = {
+      {0.,0.,0., 0., 0., 0.},
+      {1.,0.,0., 0., 0., 0.},
+      {0.,1.,0., 0., 0., 0.},
+  };
+
+  // each edge is defined by the bounding verts
+  int edge_info[3][2] = {
+      {0,1},
+      {1,2},
+      {2,0}
+  };
+
+  apf::MeshEntity* verts[3];
+  apf::MeshEntity* edges[3];
+
+  for (int i = 0; i < 3; i++) {
+    apf::Vector3 coords(vert_coords[i][0],
+                        vert_coords[i][1],
+                        vert_coords[i][2]);
+    apf::Vector3 params(vert_coords[i][3],
+                        vert_coords[i][4],
+                        vert_coords[i][5]);
+    verts[i] = mesh->createVertex(0, coords, params);
+  }
+  for (int i = 0; i < 3; i++) {
+    apf::MeshEntity* down_vs[2] = {verts[edge_info[i][0]],
+                                   verts[edge_info[i][1]]};
+    edges[i] = mesh->createEntity(apf::Mesh::EDGE, 0, down_vs);
+  }
+
+  ent = mesh->createEntity(apf::Mesh::TRIANGLE, 0, edges);
+
+  mesh->acceptChanges();
+
+  apf::changeMeshShape(mesh, crv::getBezier(order),true);
+
+  printf ("Made tri with verts:\n");
+  for (int i = 0; i < 3; i++) {
+    printf("v0: (%e,%e,%e)\n", vert_coords[i][0], vert_coords[i][1], vert_coords[i][2]);
+  }
+
+  printf ("all nodes of the tri:\n");
+  apf::Element* elem = apf::createElement(mesh->getCoordinateField(),ent);
+  apf::NewArray<apf::Vector3> nodes;
+  apf::getVectorNodes(elem,nodes);
+  apf::destroyElement(elem);
+  for (int i=0; i<(int)nodes.size(); i++)
+  {
+    printf("node %d: (%e,%e,%e)\n", i, nodes[i][0], nodes[i][1], nodes[i][2]);
+  }
+  return mesh;
+}
+
+static apf::Mesh2* makeOneTetMesh(int order, apf::MeshEntity* &ent)
+{
+
+  apf::Mesh2* mesh = apf::makeEmptyMdsMesh(0, 3, false);
+
+  double vert_coords[4][6] = {
+      {0.,0.,0., 0., 0., 0.},
+      {1.,0.,0., 0., 0., 0.},
+      {0.,1.,0., 0., 0., 0.},
+      {0.,0.,1., 0., 0., 0.},
+  };
+
+  // each edge is defined by the bounding verts
+  int const (*edge_info)[2] = apf::tet_edge_verts;
+  int face_info[4][3] = {
+    {0,1,2},
+    {0,4,3},
+    {1,5,4},
+    {2,5,3}
+  };
+
+
+  apf::MeshEntity* verts[4];
+  apf::MeshEntity* edges[6];
+  apf::MeshEntity* faces[4];
+
+  for (int i = 0; i < 4; i++) {
+    apf::Vector3 coords(vert_coords[i][0],
+                        vert_coords[i][1],
+                        vert_coords[i][2]);
+    apf::Vector3 params(vert_coords[i][3],
+                        vert_coords[i][4],
+                        vert_coords[i][5]);
+    verts[i] = mesh->createVertex(0, coords, params);
+  }
+  for (int i = 0; i < 6; i++) {
+    apf::MeshEntity* down_vs[2] = {verts[edge_info[i][0]],
+                                   verts[edge_info[i][1]]};
+    edges[i] = mesh->createEntity(apf::Mesh::EDGE, 0, down_vs);
+  }
+
+  for (int i = 0; i < 4; i++) {
+    apf::MeshEntity* down_es[3] = {edges[face_info[i][0]],
+                                   edges[face_info[i][1]],
+                                   edges[face_info[i][2]]};
+    faces[i] = mesh->createEntity(apf::Mesh::TRIANGLE, 0, down_es);
+  }
+
+  ent = mesh->createEntity(apf::Mesh::TET, 0, faces);
+
+  mesh->acceptChanges();
+
+  apf::changeMeshShape(mesh, crv::getBezier(order),true);
+
+  printf ("Made tet with verts:\n");
+  for (int i = 0; i < 4; i++) {
+    printf("v0: (%e,%e,%e)\n", vert_coords[i][0], vert_coords[i][1], vert_coords[i][2]);
+  }
+
+  printf ("all nodes of the tet:\n");
+  apf::Element* elem = apf::createElement(mesh->getCoordinateField(),ent);
+  apf::NewArray<apf::Vector3> nodes;
+  apf::getVectorNodes(elem,nodes);
+  apf::destroyElement(elem);
+  for (int i=0; i<(int)nodes.size(); i++)
+  {
+    printf("node %d: (%e,%e,%e)\n", i, nodes[i][0], nodes[i][1], nodes[i][2]);
+  }
+  return mesh;
+}

From 39bcc6f5028bf26322f4d61e04ec6d2e36cb987b Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 22 Jul 2022 15:32:13 -0400
Subject: [PATCH 465/555] closureTest does not work with dmg models

see #369
---
 test/inClosureOf_test.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/inClosureOf_test.cc b/test/inClosureOf_test.cc
index 2078638cc..7b673dee3 100644
--- a/test/inClosureOf_test.cc
+++ b/test/inClosureOf_test.cc
@@ -29,7 +29,6 @@ int main(int argc, char** argv)
   gmi_sim_start();
   gmi_register_sim();
 #endif
-  gmi_register_mesh();
 
   gmi_model* model = gmi_load(modelFile); // read the model here
 

From 97ca1ef6139755a5d40cdabda0f0d182af785ee6 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 5 Oct 2022 12:01:06 -0400
Subject: [PATCH 466/555] support simmodsuite 18

---
 cmake/FindSimModSuite.cmake  |  4 ++--
 parma/diffMC/.clang_complete |  2 +-
 test/runSimxAnisoAdapt.cc    |  4 ----
 test/simTranslate.cc         | 23 ++++++++++++++++-------
 4 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/cmake/FindSimModSuite.cmake b/cmake/FindSimModSuite.cmake
index 8be42efb2..d935a8661 100644
--- a/cmake/FindSimModSuite.cmake
+++ b/cmake/FindSimModSuite.cmake
@@ -84,7 +84,7 @@ string(REGEX REPLACE
   "${SIM_VERSION}")
 
 set(MIN_VALID_SIM_VERSION 12.0.190225)
-set(MAX_VALID_SIM_VERSION 16.0-210623)
+set(MAX_VALID_SIM_VERSION 18.0-220930)
 if( ${SKIP_SIMMETRIX_VERSION_CHECK} )
   message(STATUS "Skipping Simmetrix SimModSuite version check."
     " This may result in undefined behavior")
@@ -113,7 +113,7 @@ message(STATUS "SIM_ARCHOS ${SIM_ARCHOS}")
 option(SIM_PARASOLID "Use Parasolid through Simmetrix" OFF)
 if (SIM_PARASOLID)
   set(MIN_SIM_PARASOLID_VERSION 290)
-  set(MAX_SIM_PARASOLID_VERSION 330)
+  set(MAX_SIM_PARASOLID_VERSION 350)
   foreach(version RANGE
       ${MAX_SIM_PARASOLID_VERSION}
       ${MIN_SIM_PARASOLID_VERSION} -10)
diff --git a/parma/diffMC/.clang_complete b/parma/diffMC/.clang_complete
index 93a06dfc0..ae7332293 100644
--- a/parma/diffMC/.clang_complete
+++ b/parma/diffMC/.clang_complete
@@ -1,4 +1,4 @@
--D_HAVE_ZOLTAN_
+-D_PUMI_HAS_ZOLTAN_
 -D_HAVE_PARMETIS_
 -DFMDB_PARALLEL
 -DFMDB
diff --git a/test/runSimxAnisoAdapt.cc b/test/runSimxAnisoAdapt.cc
index a0c0389d8..d0aeacc16 100644
--- a/test/runSimxAnisoAdapt.cc
+++ b/test/runSimxAnisoAdapt.cc
@@ -14,10 +14,6 @@
 #include "MeshSimAdapt.h"
 #include "SimDiscrete.h"
 #include "SimMessages.h"
-#include "SimError.h"
-#include "SimErrorCodes.h"
-#include "SimMeshingErrorCodes.h"
-#include "SimDiscreteErrorCodes.h"
 #include <iostream>
 
 #include <cassert>
diff --git a/test/simTranslate.cc b/test/simTranslate.cc
index 9b99255e2..c439f244f 100644
--- a/test/simTranslate.cc
+++ b/test/simTranslate.cc
@@ -11,8 +11,6 @@
 #include "SimAdvModel.h"
 #include "SimUtil.h"
 #include "SimMessages.h"
-#include "SimError.h"
-#include "SimErrorCodes.h"
 #include "MeshSim.h"
 #include "SimAttribute.h"
 #include "AttributeTypes.h"
@@ -26,6 +24,17 @@
 #include "gmi_sim_config.h"
 #include <gmi_sim.h>
 
+#if SIMMODSUITE_MAJOR_VERSION >= 18
+  #include "SimInfo.h"
+  #include "SimInfoCodes.h"
+  #define SIM_ERROR(suffix,err) SimInfo_##suffix(err)
+  typedef pSimInfo pSimError;
+#else
+  #include "SimError.h"
+  #include "SimErrorCodes.h"
+  #define SIM_ERROR(suffix,err) SimError_##suffix(err)
+#endif
+
 #ifdef SIM_PARASOLID
 #include "SimParasolidKrnl.h"
 #endif
@@ -46,14 +55,14 @@ static std::string acisExt = ".sat";
 static std::string paraExt = ".xmt_txt";
 static std::string paraExtshort = ".x_t";
 
-void printSimError(pSimError err)
-{
+void printSimError(pSimError err) {
   printf("Simmetrix error caught:\n");
-  printf("  Error code: %d\n",SimError_code(err));
-  printf("  Error string: %s\n",SimError_toString(err));
-  SimError_delete(err);
+  printf("  Error code: %d\n",SIM_ERROR(code,err));
+  printf("  Error string: %s\n",SIM_ERROR(code,err));
+  SIM_ERROR(delete,err);
 }
 
+
 void translateModel(std::string mdlName, pGModel* simmodel, pProgress& progress)
 {
   (void)progress;

From 4c8840e4243b3730bfd5b7f86f579e458e6ebebe Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Wed, 5 Oct 2022 11:31:09 -0600
Subject: [PATCH 467/555] support gmsh v4 (breaks gmsh v2 at this time)

---
 mds/apfMDS.h   |   2 +
 mds/mdsGmsh.cc | 180 ++++++++++++++++++++++++++++++++++++++++---------
 test/gmsh.cc   |  12 ++--
 3 files changed, 157 insertions(+), 37 deletions(-)

diff --git a/mds/apfMDS.h b/mds/apfMDS.h
index ef745e576..f32c5c6ca 100644
--- a/mds/apfMDS.h
+++ b/mds/apfMDS.h
@@ -194,6 +194,8 @@ MeshEntity* getMdsEntity(Mesh2* in, int dimension, int index);
 
 Mesh2* loadMdsFromGmsh(gmi_model* g, const char* filename);
 
+Mesh2* loadMdsDmgFromGmsh(const char* fnameDmg, const char* filename);
+
 Mesh2* loadMdsFromUgrid(gmi_model* g, const char* filename);
 
 void printUgridPtnStats(gmi_model* g, const char* ugridfile, const char* ptnfile,
diff --git a/mds/mdsGmsh.cc b/mds/mdsGmsh.cc
index 63e5ce463..cac98c6ce 100644
--- a/mds/mdsGmsh.cc
+++ b/mds/mdsGmsh.cc
@@ -97,6 +97,16 @@ long getLong(Reader* r)
   return x;
 }
 
+double getDouble(Reader* r)
+{
+  double x;
+  int nchars;
+  int ret = sscanf(r->word, "%lf%n", &x, &nchars);
+  PCU_ALWAYS_ASSERT(ret == 1);
+  r->word += nchars;
+  return x;
+}
+
 bool startsWith(char const* prefix, char const* s)
 {
   int ls = strlen(s);
@@ -118,23 +128,102 @@ void checkMarker(Reader* r, char const* marker)
   PCU_ALWAYS_ASSERT(startsWith(marker, r->line));
 }
 
-void readNode(Reader* r)
+void readNode(Reader* r, int bm)
 {
-  long id;
   Node n;
   apf::Vector3& p = n.point;
-  sscanf(r->line, "%ld %lf %lf %lf", &id, &p[0], &p[1], &p[2]);
-  r->nodeMap[id] = n;
+  sscanf(r->line, "%lf %lf %lf", &p[0], &p[1], &p[2]);
+  r->nodeMap[bm] = n;
   getLine(r);
 }
 
+void readEntities(Reader* r,const char* fnameDmg) 
+{
+  seekMarker(r, "$Entities");
+  long nlde,ilde,iud,tag,isign,nMV,nME,nMF,nMR;
+  double x,y,z;
+  FILE* f = fopen(fnameDmg, "w");
+  sscanf(r->line, "%ld %ld %ld %ld", &nMV, &nME, &nMF, &nMR);
+  fprintf(f, "%ld %ld %ld %ld \n", nMR, nMF, nME, nMV); // just reverse order 
+  fprintf(f, "%f %f %f \n ", 0.0, 0.0, 0.0); // Probaby model bounding box?
+  fprintf(f, "%f %f %f \n", 0.0, 0.0, 0.0); // 
+   
+  getLine(r); // because readNode gets the next line we need this outside  for Nodes_Block
+  for (long i = 0; i < nMV; ++i){  
+    sscanf(r->line, "%ld %lf %lf %lf %ld ", &tag, &x, &y, &z, &iud);
+    fprintf(f, "%ld %lf %lf %lf \n",tag,x,y,z);
+    getLine(r); 
+  }
+  for (long i = 0; i < nME; ++i){
+    tag = getLong(r); 
+    fprintf(f, "%ld", tag);
+    for (int i=0; i< 6; ++i) x=getDouble(r);  // read past min maxes
+    iud = getLong(r); 
+    for(long j =0; j < iud; ++j) isign=getLong(r); // read past iud user tags
+    nlde=getLong(r);  // 2 in straight edged models but...
+    for(long j =0; j < nlde; ++j) {
+      ilde=getLong(r);
+      fprintf(f, " %d", abs(ilde)); // modVerts started from 1
+    }
+    fprintf(f, "\n");
+    getLine(r); 
+  }
+  for (long i = 0; i < nMF; ++i){
+    tag = getLong(r); 
+    fprintf(f, "%ld %d\n", tag, 1);
+    for (int i=0; i< 6; ++i) x=getDouble(r);  // read past min maxes
+    iud = getLong(r); 
+    for(long j =0; j < iud; ++j) isign=getLong(r); // read past iud user tags
+    nlde=getLong(r); 
+    fprintf(f, "  %ld \n", nlde);
+    for(long j =0; j < nlde; ++j) {
+      ilde=getLong(r); 
+      if(ilde > 0 ) 
+        isign=1;
+      else
+        isign=0;
+      fprintf(f, "    %d %ld \n", abs(ilde),isign); 
+    }
+    getLine(r); 
+  }   
+  for (long i = 0; i < nMR; ++i){ 
+    tag = getLong(r); 
+    fprintf(f, "%ld %d \n", tag, 1);
+    for (int i=0; i< 6; ++i) x=getDouble(r);  // read past min maxes
+    iud = getLong(r); 
+    for(long j =0; j < iud; ++j) getLong(r); // read past iud user tags
+    nlde=getLong(r); 
+    fprintf(f, "%ld \n", nlde);
+    for(long j =0; j < nlde; ++j) {
+      ilde=getLong(r); 
+      if(ilde > 0 ) 
+        isign=1;
+      else
+        isign=0;
+      fprintf(f, "%d %ld \n", abs(ilde),isign); 
+    }
+    getLine(r); 
+  }   
+  checkMarker(r, "$EndEntities");
+  fclose(f);
+}
 void readNodes(Reader* r)
 {
   seekMarker(r, "$Nodes");
-  long n = getLong(r);
-  getLine(r);
-  for (long i = 0; i < n; ++i)
-    readNode(r);
+  long Num_EntityBlocks,Num_Nodes,Nodes_Block,edim,etag,junk1,junk2,junk3;
+  sscanf(r->line, "%ld %ld %ld %ld", &Num_EntityBlocks, &Num_Nodes, &junk1, &junk2);
+  getLine(r); // because readNode gets the next line we need this outside  for Nodes_Block
+  for (long i = 0; i < Num_EntityBlocks; ++i){
+    sscanf(r->line, "%ld %ld %ld %ld", &edim, &etag, &junk3, &Nodes_Block);
+    long blockMap[Nodes_Block];
+    for (long j = 0; j < Nodes_Block; ++j){
+      getLine(r);
+      sscanf(r->line, "%ld", &blockMap[j]);
+    }
+    getLine(r);
+    for (long j = 0; j < Nodes_Block; ++j)
+      readNode(r,blockMap[j]);  // has a genLine at end
+  }    
   checkMarker(r, "$EndNodes");
 }
 
@@ -149,34 +238,35 @@ apf::MeshEntity* lookupVert(Reader* r, long nodeId, apf::ModelEntity* g)
   return n.entity;
 }
 
-void readElement(Reader* r)
+void readElement(Reader* r, long gmshType,long gtag)
 {
   long id = getLong(r);
-  long gmshType = getLong(r);
   if (isQuadratic(gmshType))
     r->isQuadratic = true;
   int apfType = apfFromGmsh(gmshType);
   PCU_ALWAYS_ASSERT(0 <= apfType);
   int nverts = apf::Mesh::adjacentCount[apfType][0];
   int dim = apf::Mesh::typeDimension[apfType];
-  long ntags = getLong(r);
-  /* The Gmsh 4.9 documentation on the legacy 2.* format states:
-   * "By default, the first tag is the tag of the physical entity to which the
-   * element belongs; the second is the tag of the elementary model entity to
-   * which the element belongs; the third is the number of mesh partitions to
-   * which the element belongs, followed by the partition ids (negative
-   * partition ids indicate ghost cells). A zero tag is equivalent to no tag.
-   * Gmsh and most codes using the MSH 2 format require at least the first two
-   * tags (physical and elementary tags)."
-   * A physical entity is a user defined grouping of elementary model entities.
-   * An elementary model entity is a geometric model entity. */
-  PCU_ALWAYS_ASSERT(ntags >= 2);
-  const int physType = static_cast<int>(getLong(r));
-  PCU_ALWAYS_ASSERT(dim>=0 && dim<4);
-  r->physicalType[dim].push_back(physType);
-  long gtag = getLong(r);
-  for (long i = 2; i < ntags; ++i)
-    getLong(r); /* discard all other element tags */
+  if(false) { // FIXME
+    long ntags = getLong(r);
+    /* The Gmsh 4.9 documentation on the legacy 2.* format states:
+     * "By default, the first tag is the tag of the physical entity to which the
+     * element belongs; the second is the tag of the elementary model entity to
+     * which the element belongs; the third is the number of mesh partitions to
+     * which the element belongs, followed by the partition ids (negative
+     * partition ids indicate ghost cells). A zero tag is equivalent to no tag.
+     * Gmsh and most codes using the MSH 2 format require at least the first two
+     * tags (physical and elementary tags)."
+     * A physical entity is a user defined grouping of elementary model entities.
+     * An elementary model entity is a geometric model entity. */
+    PCU_ALWAYS_ASSERT(ntags >= 2);
+    const int physType = static_cast<int>(getLong(r));
+    PCU_ALWAYS_ASSERT(dim>=0 && dim<4);
+    r->physicalType[dim].push_back(physType);
+//FIXME blocks compilation    long gtag = getLong(r);
+    for (long i = 2; i < ntags; ++i)
+      getLong(r); /* discard all other element tags */
+  }
   apf::ModelEntity* g = r->mesh->findModelEntity(dim, gtag);
   apf::Downward verts;
   for (int i = 0; i < nverts; ++i) {
@@ -196,10 +286,16 @@ void readElement(Reader* r)
 void readElements(Reader* r)
 {
   seekMarker(r, "$Elements");
-  long n = getLong(r);
+  long Num_EntityBlocks,Num_Elements,Elements_Block,Edim,gtag,gmshType,junk1,junk2;
+  sscanf(r->line, "%ld %ld %ld %ld", &Num_EntityBlocks, &Num_Elements, &junk1, &junk2);
   getLine(r);
-  for (long i = 0; i < n; ++i)
-    readElement(r);
+  for (long i = 0; i < Num_EntityBlocks; ++i){
+    sscanf(r->line, "%ld %ld %ld %ld", &Edim, &gtag, &gmshType, &Elements_Block);
+    getLine(r);
+    for (long j = 0; j < Elements_Block; ++j) {
+       readElement(r,gmshType,gtag);
+    }
+  }
   checkMarker(r, "$EndElements");
 }
 
@@ -284,15 +380,25 @@ void readGmsh(apf::Mesh2* m, const char* filename)
   readNodes(&r);
   readElements(&r);
   m->acceptChanges();
-  setElmPhysicalType(&r,m);
+  if(false) // FIXME
+    setElmPhysicalType(&r,m);
   freeReader(&r);
   if (r.isQuadratic)
     readQuadratic(&r, m, filename);
 }
+}  // closes original namespace 
 
+namespace apf {
+void gmshFindDmg(const char* fnameDmg, const char* filename)
+{
+  Reader r;
+  
+  Mesh2* m=NULL;
+  initReader(&r, m,  filename);
+  readEntities(&r, fnameDmg);
+  freeReader(&r);
 }
 
-namespace apf {
 
 Mesh2* loadMdsFromGmsh(gmi_model* g, const char* filename)
 {
@@ -301,4 +407,12 @@ Mesh2* loadMdsFromGmsh(gmi_model* g, const char* filename)
   return m;
 }
 
+Mesh2* loadMdsDmgFromGmsh(const char*fnameDmg, const char* filename)
+{
+  gmshFindDmg(fnameDmg, filename);  // new function that scans $Entities and writes a dmg 
+  Mesh2* m = makeEmptyMdsMesh(gmi_load(fnameDmg), 0, false);
+  readGmsh(m, filename);
+  return m;
+}
+
 }
diff --git a/test/gmsh.cc b/test/gmsh.cc
index 307fab07e..7db3788f5 100644
--- a/test/gmsh.cc
+++ b/test/gmsh.cc
@@ -21,10 +21,14 @@ int main(int argc, char** argv)
   }
   gmi_register_null();
   gmi_register_mesh();
-  apf::Mesh2* m = apf::loadMdsFromGmsh(gmi_load(argv[1]), argv[2]);
-  // if input model is null derive a basic model for verify to pass.
-  if (std::string(argv[1]).compare(".null") == 0)
-    apf::deriveMdsModel(m);
+  if(false) {
+    apf::Mesh2* m = apf::loadMdsFromGmsh(gmi_load(argv[1]), argv[2]);
+    // if input model is null derive a basic model for verify to pass.
+    if (std::string(argv[1]).compare(".null") == 0)
+      apf::deriveMdsModel(m);
+  }
+  apf::Mesh2* m = apf::loadMdsDmgFromGmsh(argv[1], argv[2]);
+
   m->verify();
   m->writeNative(argv[3]);
   m->destroyNative();

From 1fa5ab1036d5966f2cba8409bea8853a70f1255e Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Wed, 5 Oct 2022 08:23:19 -0600
Subject: [PATCH 468/555] Parmetis from github needed some help from Ben M

---
 cmake/FindParmetis.cmake | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/cmake/FindParmetis.cmake b/cmake/FindParmetis.cmake
index b00a11d41..e9a544418 100644
--- a/cmake/FindParmetis.cmake
+++ b/cmake/FindParmetis.cmake
@@ -28,7 +28,13 @@ if(NOT EXISTS "${METIS_LIBRARY}")
   message(FATAL_ERROR "metis library not found")
 endif()
 
-set(PARMETIS_LIBRARIES ${PARMETIS_LIBRARY} ${METIS_LIBRARY})
+find_library(GK_LIBRARY GKlib PATHS "${PARMETIS_PREFIX}/lib")
+if(EXISTS "${GK_LIBRARY}")
+	set(PARMETIS_LIBRARIES ${PARMETIS_LIBRARY} ${METIS_LIBRARY} ${GK_LIBRARY})
+else()
+	set(PARMETIS_LIBRARIES ${PARMETIS_LIBRARY} ${METIS_LIBRARY})
+endif()
+
 set(PARMETIS_INCLUDE_DIRS ${PARMETIS_INCLUDE_DIR} ${METIS_INCLUDE_DIR})
 
 include(FindPackageHandleStandardArgs)

From 4e8b8e4065b7ab84ffec3c6a8edc4499e4922ac2 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 5 Oct 2022 13:46:38 -0400
Subject: [PATCH 469/555] latest pumi-meshes

---
 pumi-meshes | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pumi-meshes b/pumi-meshes
index 98f48ca2c..457c28125 160000
--- a/pumi-meshes
+++ b/pumi-meshes
@@ -1 +1 @@
-Subproject commit 98f48ca2cb8f1f3ad3db0c5fe4ad1c8df1f20811
+Subproject commit 457c28125284b015cb226883f5628a0b3a7af9c8

From ba87fb3ad748d393f8298a348d7b1da0a815bfc4 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 5 Oct 2022 14:02:53 -0400
Subject: [PATCH 470/555] gmsh v4 test

---
 test/testing.cmake | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/test/testing.cmake b/test/testing.cmake
index de1120f82..afe76acc6 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -191,12 +191,19 @@ mpi_test(create_misSquare 1
   mis_test)
 
 set(MDIR ${MESHES}/gmsh)
-mpi_test(twoQuads 1
+mpi_test(gmshv2TwoQuads 1
   ./from_gmsh
   ".null"
   "${MDIR}/twoQuads.msh"
   "${MDIR}/twoQuads.smb")
 
+set(MDIR ${MESHES}/gmsh/v4)
+mpi_test(gmshV4AirFoil 1
+  ./from_gmsh
+  "${MDIR}/AirfoilDemo.dmg"
+  "${MDIR}/AirfoilDemo.msh"
+  "${MDIR}/AirfoilDemo.smb")
+
 set(MDIR ${MESHES}/ugrid)
 mpi_test(naca_ugrid 2
   ./from_ugrid

From b27c9b2b93d01e018bfbec92d3a215a67192d19b Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 5 Oct 2022 14:38:14 -0400
Subject: [PATCH 471/555] gmsh: restore v2 support

---
 mds/apfMDS.h       |   2 +
 mds/mdsGmsh.cc     | 110 ++++++++++++++++++++++++++++++++++-----------
 test/gmsh.cc       |  39 ++++++++++++----
 test/testing.cmake |  10 +++--
 4 files changed, 122 insertions(+), 39 deletions(-)

diff --git a/mds/apfMDS.h b/mds/apfMDS.h
index f32c5c6ca..fbc8749d8 100644
--- a/mds/apfMDS.h
+++ b/mds/apfMDS.h
@@ -192,6 +192,8 @@ int getMdsIndex(Mesh2* in, MeshEntity* e);
   so call apf::reorderMdsMesh after any mesh modification. */
 MeshEntity* getMdsEntity(Mesh2* in, int dimension, int index);
 
+int gmshMajorVersion(const char* filename);
+
 Mesh2* loadMdsFromGmsh(gmi_model* g, const char* filename);
 
 Mesh2* loadMdsDmgFromGmsh(const char* fnameDmg, const char* filename);
diff --git a/mds/mdsGmsh.cc b/mds/mdsGmsh.cc
index cac98c6ce..5271198c0 100644
--- a/mds/mdsGmsh.cc
+++ b/mds/mdsGmsh.cc
@@ -52,6 +52,8 @@ struct Reader {
   char* line;
   char* word;
   size_t linecap;
+  int major_version;
+  int minor_version;
   bool isQuadratic;
   std::map<long, Node> nodeMap;
   std::map<long, apf::MeshEntity*> entMap[4];
@@ -60,19 +62,6 @@ struct Reader {
   std::vector<int> physicalType[4];
 };
 
-void initReader(Reader* r, apf::Mesh2* m, const char* filename)
-{
-  r->mesh = m;
-  r->file = fopen(filename, "r");
-  if (!r->file) {
-    lion_eprint(1,"couldn't open Gmsh file \"%s\"\n",filename);
-    abort();
-  }
-  r->line = static_cast<char*>(malloc(1));
-  r->line[0] = '\0';
-  r->linecap = 1;
-  r->isQuadratic = false;
-}
 
 void freeReader(Reader* r)
 {
@@ -128,12 +117,37 @@ void checkMarker(Reader* r, char const* marker)
   PCU_ALWAYS_ASSERT(startsWith(marker, r->line));
 }
 
-void readNode(Reader* r, int bm)
+void initReader(Reader* r, apf::Mesh2* m, const char* filename)
+{
+  r->mesh = m;
+  r->file = fopen(filename, "r");
+  if (!r->file) {
+    lion_eprint(1,"couldn't open Gmsh file \"%s\"\n",filename);
+    abort();
+  }
+  r->line = static_cast<char*>(malloc(1));
+  r->line[0] = '\0';
+  r->linecap = 1;
+  r->isQuadratic = false;
+  seekMarker(r, "$MeshFormat");
+  int fileType, dataSize;
+  int ret = sscanf(r->line, "%d.%d %d %d\n",
+      &r->major_version, &r->minor_version, &fileType, &dataSize);
+  PCU_ALWAYS_ASSERT(ret==4);
+}
+
+void readNode(Reader* r, int bm=-1)
 {
   Node n;
   apf::Vector3& p = n.point;
-  sscanf(r->line, "%lf %lf %lf", &p[0], &p[1], &p[2]);
-  r->nodeMap[bm] = n;
+  if(r->major_version == 4) {
+    sscanf(r->line, "%lf %lf %lf", &p[0], &p[1], &p[2]);
+    r->nodeMap[bm] = n;
+  } else if(r->major_version == 2) {
+    long id;
+    sscanf(r->line, "%ld %lf %lf %lf", &id, &p[0], &p[1], &p[2]);
+    r->nodeMap[id] = n;
+  }
   getLine(r);
 }
 
@@ -207,8 +221,21 @@ void readEntities(Reader* r,const char* fnameDmg)
   checkMarker(r, "$EndEntities");
   fclose(f);
 }
-void readNodes(Reader* r)
+
+void readNodesV2(Reader* r)
+{
+  PCU_ALWAYS_ASSERT(r->major_version == 2);
+  seekMarker(r, "$Nodes");
+  long n = getLong(r);
+  getLine(r);
+  for (long i = 0; i < n; ++i)
+    readNode(r);
+  checkMarker(r, "$EndNodes");
+}
+
+void readNodesV4(Reader* r)
 {
+  PCU_ALWAYS_ASSERT(r->major_version == 4);
   seekMarker(r, "$Nodes");
   long Num_EntityBlocks,Num_Nodes,Nodes_Block,edim,etag,junk1,junk2,junk3;
   sscanf(r->line, "%ld %ld %ld %ld", &Num_EntityBlocks, &Num_Nodes, &junk1, &junk2);
@@ -238,16 +265,19 @@ apf::MeshEntity* lookupVert(Reader* r, long nodeId, apf::ModelEntity* g)
   return n.entity;
 }
 
-void readElement(Reader* r, long gmshType,long gtag)
+void readElement(Reader* r, long gmshType=-1, long gtag=-1)
 {
   long id = getLong(r);
+  if(r->major_version == 2) {
+    gmshType = getLong(r);
+  }
   if (isQuadratic(gmshType))
     r->isQuadratic = true;
   int apfType = apfFromGmsh(gmshType);
   PCU_ALWAYS_ASSERT(0 <= apfType);
   int nverts = apf::Mesh::adjacentCount[apfType][0];
   int dim = apf::Mesh::typeDimension[apfType];
-  if(false) { // FIXME
+  if(r->major_version == 2) {
     long ntags = getLong(r);
     /* The Gmsh 4.9 documentation on the legacy 2.* format states:
      * "By default, the first tag is the tag of the physical entity to which the
@@ -263,7 +293,7 @@ void readElement(Reader* r, long gmshType,long gtag)
     const int physType = static_cast<int>(getLong(r));
     PCU_ALWAYS_ASSERT(dim>=0 && dim<4);
     r->physicalType[dim].push_back(physType);
-//FIXME blocks compilation    long gtag = getLong(r);
+    long gtag = getLong(r);
     for (long i = 2; i < ntags; ++i)
       getLong(r); /* discard all other element tags */
   }
@@ -283,8 +313,20 @@ void readElement(Reader* r, long gmshType,long gtag)
   getLine(r);
 }
 
-void readElements(Reader* r)
+void readElementsV2(Reader* r)
 {
+  PCU_ALWAYS_ASSERT(r->major_version == 2);
+  seekMarker(r, "$Elements");
+  long n = getLong(r);
+  getLine(r);
+  for (long i = 0; i < n; ++i)
+    readElement(r);
+  checkMarker(r, "$EndElements");
+}
+
+void readElementsV4(Reader* r)
+{
+  PCU_ALWAYS_ASSERT(r->major_version == 4);
   seekMarker(r, "$Elements");
   long Num_EntityBlocks,Num_Elements,Elements_Block,Edim,gtag,gmshType,junk1,junk2;
   sscanf(r->line, "%ld %ld %ld %ld", &Num_EntityBlocks, &Num_Elements, &junk1, &junk2);
@@ -377,24 +419,40 @@ void readGmsh(apf::Mesh2* m, const char* filename)
 {
   Reader r;
   initReader(&r, m, filename);
-  readNodes(&r);
-  readElements(&r);
-  m->acceptChanges();
-  if(false) // FIXME
+  if(r.major_version == 4) {
+    readNodesV4(&r);
+    readElementsV4(&r);
+    m->acceptChanges();
+  } else if(r.major_version == 2) {
+    readNodesV2(&r);
+    readElementsV2(&r);
+    m->acceptChanges();
     setElmPhysicalType(&r,m);
-  freeReader(&r);
+  }
   if (r.isQuadratic)
     readQuadratic(&r, m, filename);
+  freeReader(&r);
 }
 }  // closes original namespace 
 
 namespace apf {
+
+int gmshMajorVersion(const char* filename) {
+  Reader r;
+  Mesh2* m=NULL;
+  initReader(&r, m,  filename);
+  int version = r.major_version;
+  freeReader(&r);
+  return version;
+}
+
 void gmshFindDmg(const char* fnameDmg, const char* filename)
 {
   Reader r;
   
   Mesh2* m=NULL;
   initReader(&r, m,  filename);
+  PCU_ALWAYS_ASSERT(r.major_version==4);
   readEntities(&r, fnameDmg);
   freeReader(&r);
 }
diff --git a/test/gmsh.cc b/test/gmsh.cc
index 7db3788f5..77adab9fc 100644
--- a/test/gmsh.cc
+++ b/test/gmsh.cc
@@ -13,24 +13,45 @@ int main(int argc, char** argv)
   MPI_Init(&argc,&argv);
   PCU_Comm_Init();
   lion_set_verbosity(1);
-  if ( argc != 4 ) {
+  if ( argc != 5 ) {
     if ( !PCU_Comm_Self() )
-      printf("Usage: %s <model> <in .msh> <out .smb>\n", argv[0]);
+      printf("Usage: %s <in .dmg> <in .msh> <out .smb> <out .dmg>\n"
+             "The input .msh and output .smb file names are required. \n"
+             "If 'none' is specified as the input model file name then \n"
+             "a output model (.dmg) will be written to the specified filename. \n"
+             "When a **gmsh v2** .msh is passed in, a minimal model will be created from "
+             "the mesh.\n"
+             "When a **gmsh v4** .msh is passed in, a topological model will be created "
+             "from the geometric model entities defined in the gmsh input file.\n", argv[0]);
     MPI_Finalize();
     exit(EXIT_FAILURE);
   }
   gmi_register_null();
   gmi_register_mesh();
-  if(false) {
-    apf::Mesh2* m = apf::loadMdsFromGmsh(gmi_load(argv[1]), argv[2]);
-    // if input model is null derive a basic model for verify to pass.
-    if (std::string(argv[1]).compare(".null") == 0)
+
+  std::string model(argv[1]);
+  std::string gmsh(argv[2]);
+  std::string outMesh(argv[3]);
+  std::string outModel(argv[4]);
+
+  const int gmshVersion = apf::gmshMajorVersion(gmsh.c_str());
+  fprintf(stderr, "version %d\n", gmshVersion);
+  apf::Mesh2* m;
+  if (gmshVersion == 2) {
+    if (model.compare("none") == 0) {
+      m = apf::loadMdsFromGmsh(gmi_load(".null"), gmsh.c_str());
       apf::deriveMdsModel(m);
+      gmi_write_dmg(m->getModel(),outModel.c_str());
+    } else {
+      m = apf::loadMdsFromGmsh(gmi_load(model.c_str()), gmsh.c_str());
+    }
+  } else if (gmshVersion == 4) {
+    if (model.compare("none") == 0) {
+      m = apf::loadMdsDmgFromGmsh(outModel.c_str(), gmsh.c_str());
+    }
   }
-  apf::Mesh2* m = apf::loadMdsDmgFromGmsh(argv[1], argv[2]);
-
   m->verify();
-  m->writeNative(argv[3]);
+  m->writeNative(outMesh.c_str());
   m->destroyNative();
   apf::destroyMesh(m);
   PCU_Comm_Free();
diff --git a/test/testing.cmake b/test/testing.cmake
index afe76acc6..180ac9776 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -193,16 +193,18 @@ mpi_test(create_misSquare 1
 set(MDIR ${MESHES}/gmsh)
 mpi_test(gmshv2TwoQuads 1
   ./from_gmsh
-  ".null"
+  "none"
   "${MDIR}/twoQuads.msh"
-  "${MDIR}/twoQuads.smb")
+  "${MDIR}/twoQuads.smb"
+  "${MDIR}/twoQuads.dmg")
 
 set(MDIR ${MESHES}/gmsh/v4)
 mpi_test(gmshV4AirFoil 1
   ./from_gmsh
-  "${MDIR}/AirfoilDemo.dmg"
+  "none"
   "${MDIR}/AirfoilDemo.msh"
-  "${MDIR}/AirfoilDemo.smb")
+  "${MDIR}/AirfoilDemo.smb"
+  "${MDIR}/AirfoilDemo.dmg")
 
 set(MDIR ${MESHES}/ugrid)
 mpi_test(naca_ugrid 2

From e72c47391555a66debaf4e0f4d49b826724ee4ca Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 5 Oct 2022 15:57:45 -0400
Subject: [PATCH 472/555] gmsh: fix warning

this was a bug... thanks github actions
---
 mds/mdsGmsh.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mds/mdsGmsh.cc b/mds/mdsGmsh.cc
index 5271198c0..eab936442 100644
--- a/mds/mdsGmsh.cc
+++ b/mds/mdsGmsh.cc
@@ -293,7 +293,7 @@ void readElement(Reader* r, long gmshType=-1, long gtag=-1)
     const int physType = static_cast<int>(getLong(r));
     PCU_ALWAYS_ASSERT(dim>=0 && dim<4);
     r->physicalType[dim].push_back(physType);
-    long gtag = getLong(r);
+    gtag = getLong(r);
     for (long i = 2; i < ntags; ++i)
       getLong(r); /* discard all other element tags */
   }

From 7b523eb0345c130147c5609f95232af3cb7405b0 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 5 Oct 2022 16:11:35 -0400
Subject: [PATCH 473/555] gmsh: silence more warnings

---
 mds/mdsGmsh.cc | 6 +++---
 test/gmsh.cc   | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/mds/mdsGmsh.cc b/mds/mdsGmsh.cc
index eab936442..7d4a26567 100644
--- a/mds/mdsGmsh.cc
+++ b/mds/mdsGmsh.cc
@@ -177,7 +177,7 @@ void readEntities(Reader* r,const char* fnameDmg)
     nlde=getLong(r);  // 2 in straight edged models but...
     for(long j =0; j < nlde; ++j) {
       ilde=getLong(r);
-      fprintf(f, " %d", abs(ilde)); // modVerts started from 1
+      fprintf(f, " %ld", std::abs(ilde)); // modVerts started from 1
     }
     fprintf(f, "\n");
     getLine(r); 
@@ -196,7 +196,7 @@ void readEntities(Reader* r,const char* fnameDmg)
         isign=1;
       else
         isign=0;
-      fprintf(f, "    %d %ld \n", abs(ilde),isign); 
+      fprintf(f, "    %ld %ld \n", std::abs(ilde),isign); 
     }
     getLine(r); 
   }   
@@ -214,7 +214,7 @@ void readEntities(Reader* r,const char* fnameDmg)
         isign=1;
       else
         isign=0;
-      fprintf(f, "%d %ld \n", abs(ilde),isign); 
+      fprintf(f, "%ld %ld \n", std::abs(ilde),isign); 
     }
     getLine(r); 
   }   
diff --git a/test/gmsh.cc b/test/gmsh.cc
index 77adab9fc..b2f3911ec 100644
--- a/test/gmsh.cc
+++ b/test/gmsh.cc
@@ -36,7 +36,7 @@ int main(int argc, char** argv)
 
   const int gmshVersion = apf::gmshMajorVersion(gmsh.c_str());
   fprintf(stderr, "version %d\n", gmshVersion);
-  apf::Mesh2* m;
+  apf::Mesh2* m = NULL;
   if (gmshVersion == 2) {
     if (model.compare("none") == 0) {
       m = apf::loadMdsFromGmsh(gmi_load(".null"), gmsh.c_str());

From 7e306ce55ebf97663008fdd76ba0170b3adf4862 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Thu, 6 Oct 2022 12:40:17 -0400
Subject: [PATCH 474/555] gmsh: compare dmg

---
 pumi-meshes        | 2 +-
 test/testing.cmake | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/pumi-meshes b/pumi-meshes
index 457c28125..d03989485 160000
--- a/pumi-meshes
+++ b/pumi-meshes
@@ -1 +1 @@
-Subproject commit 457c28125284b015cb226883f5628a0b3a7af9c8
+Subproject commit d0398948585a3a0487f0e6cc61fe591573531fc4
diff --git a/test/testing.cmake b/test/testing.cmake
index 180ac9776..b9d7011e7 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -205,6 +205,9 @@ mpi_test(gmshV4AirFoil 1
   "${MDIR}/AirfoilDemo.msh"
   "${MDIR}/AirfoilDemo.smb"
   "${MDIR}/AirfoilDemo.dmg")
+add_test(NAME gmshV4AirFoil_dmgDiff
+  COMMAND diff -r ${MDIR}/AirfoilDemo.dmg AirfoilDemo_gold.dmg
+  WORKING_DIRECTORY ${MDIR})
 
 set(MDIR ${MESHES}/ugrid)
 mpi_test(naca_ugrid 2

From 95126d85f7d03ace81164556c1af1b6f07cff99c Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Sun, 9 Oct 2022 18:24:34 -0600
Subject: [PATCH 475/555] compiles but wont link...retreat to see if develop
 has same issue

---
 apf_sim/apfSIM.cc |  30 +++++++
 apf_sim/apfSIM.h  |   1 +
 test/convert.cc   | 208 +++++++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 237 insertions(+), 2 deletions(-)

diff --git a/apf_sim/apfSIM.cc b/apf_sim/apfSIM.cc
index a5b14a0d3..8132edfe7 100644
--- a/apf_sim/apfSIM.cc
+++ b/apf_sim/apfSIM.cc
@@ -542,6 +542,19 @@ class TagSIM
       comm = AttachDataCommu_new(unitSize/sizeof(int),0,count);
       PM_setMigrId(mesh,id);
     }
+    TagSIM(pParMesh m,
+           pMeshDataId simTag,
+           const char* n,
+           std::size_t unitSize,
+           int c):
+      count(c),
+      mesh(m),
+      id(simTag),
+      name(n)
+    {
+      comm = AttachDataCommu_new(unitSize/sizeof(int),0,count);
+      PM_setMigrId(mesh,id);
+    }
     virtual ~TagSIM()
     {
       MD_removeMeshCallback(id,CBdelete);
@@ -650,6 +663,14 @@ class IntTagSIM : public TagSIM
       return 1;
     }
 
+    IntTagSIM(pParMesh m, pMeshDataId simTag, const char* name, int c):
+      TagSIM(m,simTag,name,sizeof(int),c)
+    {
+      MD_setMeshCallback(id,CBdelete,deleteIntCB,this);
+      MD_setMeshCallback(id,CBmigrateOut,pm_sendIntArray,comm);
+      MD_setMeshCallback(id,CBmigrateIn,pm_recvIntArray,comm);
+    }
+
     IntTagSIM(pParMesh m, const char* name, int c):
       TagSIM(m,name,sizeof(int),c)
     {
@@ -750,6 +771,15 @@ MeshTag* MeshSIM::createIntTag(const char* name, int size)
   return reinterpret_cast<MeshTag*>(tag);
 }
 
+MeshTag* MeshSIM::createIntTag(const char* name, pMeshDataId simTag, int size)
+{
+  TagSIM* tag = new IntTagSIM(mesh,simTag,name,size);
+  tags.push_back(tag);
+  return reinterpret_cast<MeshTag*>(tag);
+}
+
+
+
 MeshTag* MeshSIM::createLongTag(const char* name, int size)
 {
   TagSIM* tag = new LongTagSIM(mesh,name,size);
diff --git a/apf_sim/apfSIM.h b/apf_sim/apfSIM.h
index dedfe020e..3d87bd4eb 100644
--- a/apf_sim/apfSIM.h
+++ b/apf_sim/apfSIM.h
@@ -60,6 +60,7 @@ class MeshSIM : public Mesh2
     void getResidence(MeshEntity* e, Parts& residence);
     MeshTag* createDoubleTag(const char* name, int size);
     MeshTag* createIntTag(const char* name, int size);
+    MeshTag* createIntTag(const char* name, pMeshDataId simTag, int size);
     MeshTag* createLongTag(const char* name, int size);
     MeshTag* findTag(const char* name);
     void destroyTag(MeshTag* tag);
diff --git a/test/convert.cc b/test/convert.cc
index bd27e6266..cc8943285 100644
--- a/test/convert.cc
+++ b/test/convert.cc
@@ -2,6 +2,7 @@
 #include <lionPrint.h>
 #include <MeshSim.h>
 #include <SimPartitionedMesh.h>
+#include <SimAdvMeshing.h>
 #include <SimUtil.h>
 #include <apfSIM.h>
 #include <apfMDS.h>
@@ -11,14 +12,30 @@
 #include <apfConvert.h>
 #include <apfMesh2.h>
 #include <apfNumbering.h>
+#include <apfShape.h>
 #include <ma.h>
 #include <pcu_util.h>
 #include <cstdlib>
 #include <cstring>
 #include <iostream>
+#include <cassert>
 
 #include <getopt.h>
 
+apf::Field* convert_my_tag(apf::Mesh* m, apf::MeshTag* t) {
+  apf::MeshEntity* vtx;
+  apf::MeshIterator* it = m->begin(0);
+  apf::Field* f = apf::createFieldOn(m, "fathers2D_field", apf::SCALAR);
+  int vals[1];
+  double vals_d;
+  while ((vtx = m->iterate(it))) {
+    m->getIntTag(vtx, t, vals);
+    vals_d = vals[0];
+    apf::setScalar(f, vtx, 0, vals_d);
+  }
+  m->end(it);
+  return f;
+}
 
 static void attachOrder(apf::Mesh* m)
 {
@@ -54,6 +71,7 @@ const char* smb_path = NULL;
 int should_log = 0;
 int should_fix_pyramids = 1;
 int should_attach_order = 0;
+int ExtruRootId =0;
 bool found_bad_arg = false;
 
 void getConfig(int argc, char** argv) {
@@ -64,6 +82,7 @@ void getConfig(int argc, char** argv) {
     {"no-pyramid-fix", no_argument, &should_fix_pyramids, 0},
     {"attach-order", no_argument, &should_attach_order, 1},
     {"enable-log", no_argument, &should_log, 2},
+    {"model-face-root", required_argument, 0, 'e'},
     {"native-model", required_argument, 0, 'n'},
     {0, 0, 0, 0}  // terminate the option array
   };
@@ -74,6 +93,7 @@ void getConfig(int argc, char** argv) {
     "  --no-pyramid-fix                Disable quad-connected pyramid tetrahedronization\n"
     "  --attach-order                  Attach the Simmetrix element order as a Numbering\n"
     "  --enable-log                    Enable Simmetrix logging\n"
+    "  --model-face-root               Model face that is root of extrusion from SimModeler\n"
     "  --native-model=/path/to/model   Load the native Parasolid or ACIS model that the GeomSim model uses\n";
 
   int option_index = 0;
@@ -85,6 +105,9 @@ void getConfig(int argc, char** argv) {
       case 1: // attach order flag
       case 2: // enable simmetrix logging
         break;
+      case 'e':
+        ExtruRootId = atoi(optarg);
+        break;
       case 'n':
         gmi_native_path = optarg;
         break;
@@ -110,8 +133,8 @@ void getConfig(int argc, char** argv) {
   smb_path = argv[i++];
 
   if (!PCU_Comm_Self()) {
-    printf ("fix_pyramids %d attach_order %d enable_log %d\n",
-            should_fix_pyramids, should_attach_order, should_log);
+    printf ("fix_pyramids %d attach_order %d enable_log %d ExtruRootId %d\n",
+            should_fix_pyramids, should_attach_order, should_log, ExtruRootId);
     printf ("native-model \'%s\' model \'%s\' simmetrix mesh \'%s\' output mesh \'%s\'\n",
       gmi_native_path, gmi_path, sms_path, smb_path);
   }
@@ -123,6 +146,7 @@ int main(int argc, char** argv)
   PCU_Comm_Init();
   lion_set_verbosity(1);
   MS_init();
+  SimAdvMeshing_start(); //for fancy BL/extrusion queries
   SimModel_start();
   Sim_readLicenseFile(NULL);
   SimPartitionedMesh_start(&argc,&argv);
@@ -153,7 +177,183 @@ int main(int argc, char** argv)
   double t1 = PCU_Time();
   if(!PCU_Comm_Self())
     fprintf(stderr, "read and created the simmetrix mesh in %f seconds\n", t1-t0);
+// put the extrude tagging here which 1) loops over the mesh faces classified on the model face that is the root of the extrude
+// create a tag on vertices fathers
+// get the list of mesh rootfaces classified on the source geometric model face
+// for each srcFace in rootfaces
+// get the ids of downward adjacent vertices, store that as an array of size 3
+// get the upward adjacent region srcRgn
+// call Extrusion_3DRegionsAndLayerFaces(srcRgn,...)
+// for each face in the returned list of faces
+// get the downward adjacent vertices of face - they will be in the same order as the srcFace ids
+// set the fathers tag
+// assert that the x,y coordinates of each vertex matches the srcFace vertex coordinates within some relaxed tolerance - sanity check my assumption that face-to-vtx adjaceny is always the same order
+  
+  // create a tag on vertices fathers   TODO
+  pMeshDataId myFather = MD_newMeshDataId( "fathers2D");
+  
+  pPList listV,listVn,regions,faces;
+  pFace face;
+  pRegion region;
+  pVertex vrts[4];
+//  pVertex vrtsN[4]; 
+  int dir, err;
+  int count2D=0;
+//  int ExtruRootId=6176;
+  pGFace gface;
+  pGFace ExtruRootFace=NULL;
+//  pEntity ent;
+  pVertex entV;
+  pMesh meshP= PM_mesh	(sim_mesh, 0 );	
+  
+  //find the root face of the extrusion
+  GFIter gfIter=GM_faceIter(simModel);
+  while ( (gface=GFIter_next(gfIter))) {
+    int id = GEN_tag(gface); 
+    if(id==ExtruRootId) ExtruRootFace=gface;
+  }
+  assert(ExtruRootFace != NULL);
+
+//  GEN_regions(); //FIXME - needed?
+//  GEN_faces(); //FIXME - need face iterator, then call int id = GEN_tag(face); if(id==wantedId) ExtruRootFace=face);
+//  pGEntity ExtruRootFace=??  // is this a model tag number and if yes how do I find it....is it the face ID from SimModeler?
+  // ExtruRootFace  needs to be read in but not sure how yet
+// get the list of mesh rootfaces classified on the source geometric model face
+// assume this iterator will do?
+
+
+  char coordfilename[64];
+  char cnnfilename[64];
+  sprintf(coordfilename, "geom.crd");
+  sprintf(cnnfilename, "geom.cnn");
+  FILE* fcr = fopen(coordfilename, "w");
+  FILE* fcn = fopen(cnnfilename, "w");
+
+  FIter fIter = M_classifiedFaceIter( meshP, ExtruRootFace, 0 ); // 0 says I don't want closure	  
+  while ((face = FIter_next(fIter))) {
+    dir=1;
+// get the ids of downward adjacent vertices, store that as an array of size 3
+//    int F_verticesArray	(face, v,  dir); // guessing zero as I think the element will number such that that the curl point inward
+//OR	
+    listV= F_vertices(face, dir); 
+    void *iter = 0;        // Must initialize to 0
+    int i=0;
+    while ((entV =(pVertex)PList_next(listV, &iter))) { //loop over plist of vertices
+      // Process each item in list
+      vrts[i] = (pVertex)entV;
+      i++;
+    }
+    int nvert=i;
+    PList_delete(listV);
+
+    //FIXME DELETE the plist
+
+//  We missed a step where we check to see if a 2D father has already been created or not and use previousy created ones for any of these
+//  three that already exist.  I suppose this is best accomplished by building a map of the 3D father nodes to a 2D father node number and
+//  increment that 2D father node counter as  we encounter nodes not yet on the list. This can aso take care of the fact that 3D father node
+//  needs to point a 2D fther node as well
+    double coordNewPt[nvert][3];
+    for(i=0; i< nvert ; i++) { //FIXME logic for quads
+       int* markedData;
+       if(!EN_getDataPtr((pEntity)vrts[i],myFather,(void**)&markedData)){  // not sure about marked yet
+         count2D++;
+         int* vtxData = new int[1];
+         vtxData[0] = count2D;
+         EN_attachDataPtr((pEntity)vrts[i],myFather,(void*)vtxData);
+         V_coord(vrts[i],coordNewPt[i]);
+
+//Worked         pGEntity pei=V_whatIn(vrts[i]);	
+//         if ((V_whatInType(vrts[i])==2) ;	
+//pGEntity V_whatIn	(	pVertex 	vertex	)	
+//Returns the model entity on which vertex is classified.
+
+//gType V_whatInType	(	pVertex 	vertex	)	
+         
+                    
+//           apfSIM::modelEntity* pei=meshP->toModel(vrts[i]);
+//         int mtag = meshP->getModelTag(meshP->toModel(vrts[i]));
+        fprintf ( fcr, "%.15E %.15E %d \n", coordNewPt[i][0],coordNewPt[i][1], V_whatInType(vrts[i]));
+       }  
+    }
+
+    double coordFather[nvert][3];
+    int fatherIds[4]; //store the ids of the fathers (vertices) on the root face 
+    for(i=0; i< nvert ; i++) { //FIXME logic for quads
+       int* fatherIdPtr;
+       const int exists = EN_getDataPtr((pEntity)vrts[i],myFather,(void**)&fatherIdPtr);
+       assert(exists);
+       fatherIds[i] = fatherIdPtr[0];
+       V_coord(vrts[i],coordFather[i]);
+       fprintf ( fcn, "%d ", fatherIds[i]);
+    }
+    fprintf ( fcn, "\n");
+//    fprintf ( fcn, "%d,%d,%d,\n", fatherIds[0],fatherIds[1],fatherIds[2]);
+
+   dir=0;  // 1 fails
+ // get the upward adjacent region srcRgn
+    region = F_region(face, dir );  // 0 is the negative normal which I assume for a face on the boundary in is interior. 	
+
+// call Extrusion_3DRegionsAndLayerFaces(srcRgn,...)
+    regions=PList_new();
+    faces=PList_new();
+   err = Extrusion_3DRegionsAndLayerFaces(region, regions, faces, 1); 
+   if(err!=1 && !PCU_Comm_Self())
+    fprintf(stderr, "Extrusion_3DRegionsAndLayerFaces returned %d for err \n", err);
+    
+// for each face in the returned list of faces
+// presumably there is a pPList iterator somewhere for faces??   assume this gives me a faceN 
+   iter=0;
+   //pEntity sonFace;
+   pFace sonFace;
+   int iface=0;
+   dir=0;
+   while( (sonFace = (pFace)PList_next(faces, &iter)) ) { //loop over plist of vertices
+    if(iface !=0) {  // root face is in the stack but we already took care of it above
+// get the downward adjacent vertices of face - they will be in the same order as the srcFace ids
+     listVn= F_vertices(sonFace, dir);
+     void *iter2=0; // Must initialize to 0
+     i=0;
+     int my2Dfath;
+     pVertex  sonVtx;
+     double dist, dx, dy, distMin;
+     double coordSon[3];
+     int iMin;
+     while( (sonVtx = (pVertex)PList_next(listVn, &iter2)) ) { //loop over plist of vertices
+        V_coord(sonVtx,coordSon);
+        distMin=1.0e7;
+        for(i=0; i< nvert; i++){
+          dx=coordSon[0]-coordFather[i][0];
+          dy=coordSon[1]-coordFather[i][1];
+          dist=dx*dx+dy*dy;
+          if(dist < distMin) {
+             iMin=i;
+             distMin=dist;
+          }
+        }
+//FAIL        my2Dfath=fatherIds[i];  FAIL would have worked if Simmetrix Extrusions followed root ordering with dir 0 or 1 but they don't
+        my2Dfath=fatherIds[iMin];
+        int* vtxData = new int[1];
+        vtxData[0] = my2Dfath;
+        EN_attachDataPtr((pEntity)sonVtx,myFather,(void*)vtxData);
+//FAIL        i++;
+     }
+    PList_delete(listVn);
+     //FIXME DELETE the plist
+    }
+    iface++;
+   }
+   //FIXME DELETE the plist
+    PList_delete(faces);
+
+// set the fathers tag  TODO
+// assert that the x,y coordinates of each vertex matches the srcFace vertex coordinates within some relaxed tolerance - sanity check my assump    tion that face-to-vtx adjaceny is always the same order
+  } //end root face iterator
+
+
+
   apf::Mesh* simApfMesh = apf::createMesh(sim_mesh);
+  apf::MeshSIM* cake = reinterpret_cast<apf::MeshSIM*>(simApfMesh);
+  cake->createIntTag("fathers2D", myFather, 1);
   double t2 = PCU_Time();
   if(!PCU_Comm_Self())
     fprintf(stderr, "created the apf_sim mesh in %f seconds\n", t2-t1);
@@ -172,6 +372,10 @@ int main(int argc, char** argv)
   mesh->verify();
   mesh->writeNative(smb_path);
 
+  auto ft = mesh->findTag("fathers2D");
+  convert_my_tag(mesh, ft);
+//  apf::writeVtkFiles("foo", mesh);
+
   mesh->destroyNative();
   apf::destroyMesh(mesh);
 

From 409efd175280740b7e350fa8e6737925dd905747 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Sun, 9 Oct 2022 18:53:16 -0600
Subject: [PATCH 476/555] cleaned up a lot of dead code with unclear comments
 that likely pertained to the dead code

---
 test/convert.cc | 44 ++------------------------------------------
 1 file changed, 2 insertions(+), 42 deletions(-)

diff --git a/test/convert.cc b/test/convert.cc
index cc8943285..17c141aec 100644
--- a/test/convert.cc
+++ b/test/convert.cc
@@ -196,13 +196,10 @@ int main(int argc, char** argv)
   pFace face;
   pRegion region;
   pVertex vrts[4];
-//  pVertex vrtsN[4]; 
   int dir, err;
   int count2D=0;
-//  int ExtruRootId=6176;
   pGFace gface;
   pGFace ExtruRootFace=NULL;
-//  pEntity ent;
   pVertex entV;
   pMesh meshP= PM_mesh	(sim_mesh, 0 );	
   
@@ -214,14 +211,6 @@ int main(int argc, char** argv)
   }
   assert(ExtruRootFace != NULL);
 
-//  GEN_regions(); //FIXME - needed?
-//  GEN_faces(); //FIXME - need face iterator, then call int id = GEN_tag(face); if(id==wantedId) ExtruRootFace=face);
-//  pGEntity ExtruRootFace=??  // is this a model tag number and if yes how do I find it....is it the face ID from SimModeler?
-  // ExtruRootFace  needs to be read in but not sure how yet
-// get the list of mesh rootfaces classified on the source geometric model face
-// assume this iterator will do?
-
-
   char coordfilename[64];
   char cnnfilename[64];
   sprintf(coordfilename, "geom.crd");
@@ -232,9 +221,6 @@ int main(int argc, char** argv)
   FIter fIter = M_classifiedFaceIter( meshP, ExtruRootFace, 0 ); // 0 says I don't want closure	  
   while ((face = FIter_next(fIter))) {
     dir=1;
-// get the ids of downward adjacent vertices, store that as an array of size 3
-//    int F_verticesArray	(face, v,  dir); // guessing zero as I think the element will number such that that the curl point inward
-//OR	
     listV= F_vertices(face, dir); 
     void *iter = 0;        // Must initialize to 0
     int i=0;
@@ -246,12 +232,6 @@ int main(int argc, char** argv)
     int nvert=i;
     PList_delete(listV);
 
-    //FIXME DELETE the plist
-
-//  We missed a step where we check to see if a 2D father has already been created or not and use previousy created ones for any of these
-//  three that already exist.  I suppose this is best accomplished by building a map of the 3D father nodes to a 2D father node number and
-//  increment that 2D father node counter as  we encounter nodes not yet on the list. This can aso take care of the fact that 3D father node
-//  needs to point a 2D fther node as well
     double coordNewPt[nvert][3];
     for(i=0; i< nvert ; i++) { //FIXME logic for quads
        int* markedData;
@@ -262,16 +242,6 @@ int main(int argc, char** argv)
          EN_attachDataPtr((pEntity)vrts[i],myFather,(void*)vtxData);
          V_coord(vrts[i],coordNewPt[i]);
 
-//Worked         pGEntity pei=V_whatIn(vrts[i]);	
-//         if ((V_whatInType(vrts[i])==2) ;	
-//pGEntity V_whatIn	(	pVertex 	vertex	)	
-//Returns the model entity on which vertex is classified.
-
-//gType V_whatInType	(	pVertex 	vertex	)	
-         
-                    
-//           apfSIM::modelEntity* pei=meshP->toModel(vrts[i]);
-//         int mtag = meshP->getModelTag(meshP->toModel(vrts[i]));
         fprintf ( fcr, "%.15E %.15E %d \n", coordNewPt[i][0],coordNewPt[i][1], V_whatInType(vrts[i]));
        }  
     }
@@ -287,29 +257,25 @@ int main(int argc, char** argv)
        fprintf ( fcn, "%d ", fatherIds[i]);
     }
     fprintf ( fcn, "\n");
-//    fprintf ( fcn, "%d,%d,%d,\n", fatherIds[0],fatherIds[1],fatherIds[2]);
 
    dir=0;  // 1 fails
  // get the upward adjacent region srcRgn
     region = F_region(face, dir );  // 0 is the negative normal which I assume for a face on the boundary in is interior. 	
 
-// call Extrusion_3DRegionsAndLayerFaces(srcRgn,...)
     regions=PList_new();
     faces=PList_new();
    err = Extrusion_3DRegionsAndLayerFaces(region, regions, faces, 1); 
    if(err!=1 && !PCU_Comm_Self())
     fprintf(stderr, "Extrusion_3DRegionsAndLayerFaces returned %d for err \n", err);
     
-// for each face in the returned list of faces
-// presumably there is a pPList iterator somewhere for faces??   assume this gives me a faceN 
+   // for each face in the returned list of faces
    iter=0;
-   //pEntity sonFace;
    pFace sonFace;
    int iface=0;
    dir=0;
    while( (sonFace = (pFace)PList_next(faces, &iter)) ) { //loop over plist of vertices
     if(iface !=0) {  // root face is in the stack but we already took care of it above
-// get the downward adjacent vertices of face - they will be in the same order as the srcFace ids
+     // get the downward adjacent vertices of face - they will be in the same order as the srcFace ids
      listVn= F_vertices(sonFace, dir);
      void *iter2=0; // Must initialize to 0
      i=0;
@@ -330,23 +296,17 @@ int main(int argc, char** argv)
              distMin=dist;
           }
         }
-//FAIL        my2Dfath=fatherIds[i];  FAIL would have worked if Simmetrix Extrusions followed root ordering with dir 0 or 1 but they don't
         my2Dfath=fatherIds[iMin];
         int* vtxData = new int[1];
         vtxData[0] = my2Dfath;
         EN_attachDataPtr((pEntity)sonVtx,myFather,(void*)vtxData);
-//FAIL        i++;
      }
     PList_delete(listVn);
-     //FIXME DELETE the plist
     }
     iface++;
    }
-   //FIXME DELETE the plist
     PList_delete(faces);
 
-// set the fathers tag  TODO
-// assert that the x,y coordinates of each vertex matches the srcFace vertex coordinates within some relaxed tolerance - sanity check my assump    tion that face-to-vtx adjaceny is always the same order
   } //end root face iterator
 
 

From 70d3e2efb94281f23276204c7d6d4d224d02851b Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Tue, 11 Oct 2022 14:12:02 -0600
Subject: [PATCH 477/555] a little cleanup

---
 test/convert.cc | 215 +++++++++++++++++++++++++-----------------------
 1 file changed, 113 insertions(+), 102 deletions(-)

diff --git a/test/convert.cc b/test/convert.cc
index 17c141aec..108d74cde 100644
--- a/test/convert.cc
+++ b/test/convert.cc
@@ -19,9 +19,10 @@
 #include <cstring>
 #include <iostream>
 #include <cassert>
-
 #include <getopt.h>
 
+#include <stdio.h>
+
 apf::Field* convert_my_tag(apf::Mesh* m, apf::MeshTag* t) {
   apf::MeshEntity* vtx;
   apf::MeshIterator* it = m->begin(0);
@@ -71,6 +72,7 @@ const char* smb_path = NULL;
 int should_log = 0;
 int should_fix_pyramids = 1;
 int should_attach_order = 0;
+int numExtruRootId =0;
 int ExtruRootId =0;
 bool found_bad_arg = false;
 
@@ -93,7 +95,7 @@ void getConfig(int argc, char** argv) {
     "  --no-pyramid-fix                Disable quad-connected pyramid tetrahedronization\n"
     "  --attach-order                  Attach the Simmetrix element order as a Numbering\n"
     "  --enable-log                    Enable Simmetrix logging\n"
-    "  --model-face-root               Model face that is root of extrusion from SimModeler\n"
+    "  --model-face-root               Number of Model faces that are roots of extrusions from SimModeler\n"
     "  --native-model=/path/to/model   Load the native Parasolid or ACIS model that the GeomSim model uses\n";
 
   int option_index = 0;
@@ -106,7 +108,7 @@ void getConfig(int argc, char** argv) {
       case 2: // enable simmetrix logging
         break;
       case 'e':
-        ExtruRootId = atoi(optarg);
+        numExtruRootId = atoi(optarg);
         break;
       case 'n':
         gmi_native_path = optarg;
@@ -133,8 +135,8 @@ void getConfig(int argc, char** argv) {
   smb_path = argv[i++];
 
   if (!PCU_Comm_Self()) {
-    printf ("fix_pyramids %d attach_order %d enable_log %d ExtruRootId %d\n",
-            should_fix_pyramids, should_attach_order, should_log, ExtruRootId);
+    printf ("fix_pyramids %d attach_order %d enable_log %d numExtruRootId %d\n",
+            should_fix_pyramids, should_attach_order, should_log, numExtruRootId);
     printf ("native-model \'%s\' model \'%s\' simmetrix mesh \'%s\' output mesh \'%s\'\n",
       gmi_native_path, gmi_path, sms_path, smb_path);
   }
@@ -189,7 +191,7 @@ int main(int argc, char** argv)
 // set the fathers tag
 // assert that the x,y coordinates of each vertex matches the srcFace vertex coordinates within some relaxed tolerance - sanity check my assumption that face-to-vtx adjaceny is always the same order
   
-  // create a tag on vertices fathers   TODO
+  // create a tag on vertices fathers
   pMeshDataId myFather = MD_newMeshDataId( "fathers2D");
   
   pPList listV,listVn,regions,faces;
@@ -202,14 +204,6 @@ int main(int argc, char** argv)
   pGFace ExtruRootFace=NULL;
   pVertex entV;
   pMesh meshP= PM_mesh	(sim_mesh, 0 );	
-  
-  //find the root face of the extrusion
-  GFIter gfIter=GM_faceIter(simModel);
-  while ( (gface=GFIter_next(gfIter))) {
-    int id = GEN_tag(gface); 
-    if(id==ExtruRootId) ExtruRootFace=gface;
-  }
-  assert(ExtruRootFace != NULL);
 
   char coordfilename[64];
   char cnnfilename[64];
@@ -218,97 +212,114 @@ int main(int argc, char** argv)
   FILE* fcr = fopen(coordfilename, "w");
   FILE* fcn = fopen(cnnfilename, "w");
 
-  FIter fIter = M_classifiedFaceIter( meshP, ExtruRootFace, 0 ); // 0 says I don't want closure	  
-  while ((face = FIter_next(fIter))) {
-    dir=1;
-    listV= F_vertices(face, dir); 
-    void *iter = 0;        // Must initialize to 0
-    int i=0;
-    while ((entV =(pVertex)PList_next(listV, &iter))) { //loop over plist of vertices
-      // Process each item in list
-      vrts[i] = (pVertex)entV;
-      i++;
-    }
-    int nvert=i;
-    PList_delete(listV);
-
-    double coordNewPt[nvert][3];
-    for(i=0; i< nvert ; i++) { //FIXME logic for quads
-       int* markedData;
-       if(!EN_getDataPtr((pEntity)vrts[i],myFather,(void**)&markedData)){  // not sure about marked yet
-         count2D++;
-         int* vtxData = new int[1];
-         vtxData[0] = count2D;
-         EN_attachDataPtr((pEntity)vrts[i],myFather,(void*)vtxData);
-         V_coord(vrts[i],coordNewPt[i]);
-
-        fprintf ( fcr, "%.15E %.15E %d \n", coordNewPt[i][0],coordNewPt[i][1], V_whatInType(vrts[i]));
-       }  
-    }
-
-    double coordFather[nvert][3];
-    int fatherIds[4]; //store the ids of the fathers (vertices) on the root face 
-    for(i=0; i< nvert ; i++) { //FIXME logic for quads
-       int* fatherIdPtr;
-       const int exists = EN_getDataPtr((pEntity)vrts[i],myFather,(void**)&fatherIdPtr);
-       assert(exists);
-       fatherIds[i] = fatherIdPtr[0];
-       V_coord(vrts[i],coordFather[i]);
-       fprintf ( fcn, "%d ", fatherIds[i]);
+  FILE* fid = fopen("ExruRootID.txt", "r"); // helper file that contains all faces with extrusions
+// fstream file("ExtruRootIDs.txt");
+  for(int i=0; i<numExtruRootId; ++i) { 
+ // file >> ExtruRootId;
+    fscanf(fid,"%d",&ExtruRootId);
+    fprintf(stderr,"ExtruRootId= %d \n",ExtruRootId);
+    //find the root face of the extrusion
+    GFIter gfIter=GM_faceIter(simModel);
+    while ( (gface=GFIter_next(gfIter))) {
+      int id = GEN_tag(gface); 
+      if(id==ExtruRootId) ExtruRootFace=gface;
     }
-    fprintf ( fcn, "\n");
-
-   dir=0;  // 1 fails
- // get the upward adjacent region srcRgn
-    region = F_region(face, dir );  // 0 is the negative normal which I assume for a face on the boundary in is interior. 	
-
-    regions=PList_new();
-    faces=PList_new();
-   err = Extrusion_3DRegionsAndLayerFaces(region, regions, faces, 1); 
-   if(err!=1 && !PCU_Comm_Self())
-    fprintf(stderr, "Extrusion_3DRegionsAndLayerFaces returned %d for err \n", err);
+    assert(ExtruRootFace != NULL);
+
+    FIter fIter = M_classifiedFaceIter( meshP, ExtruRootFace, 0 ); // 0 says I don't want closure	  
+    while ((face = FIter_next(fIter))) {
+      dir=1;
+      listV= F_vertices(face, dir); 
+      void *iter = 0;        // Must initialize to 0
+      int i=0;
+      while ((entV =(pVertex)PList_next(listV, &iter))) { //loop over plist of vertices
+        // Process each item in list
+        vrts[i] = (pVertex)entV;
+        i++;
+      }
+      int nvert=i;
+      PList_delete(listV);
+  
+      double coordNewPt[nvert][3];
+      for(i=0; i< nvert ; i++) { 
+         int* markedData;
+         if(!EN_getDataPtr((pEntity)vrts[i],myFather,(void**)&markedData)){  // not sure about marked yet
+           count2D++;
+           int* vtxData = new int[1];
+           vtxData[0] = count2D;
+           EN_attachDataPtr((pEntity)vrts[i],myFather,(void*)vtxData);
+           V_coord(vrts[i],coordNewPt[i]);
+  
+          fprintf ( fcr, "%.15E %.15E %d \n", coordNewPt[i][0],coordNewPt[i][1], V_whatInType(vrts[i]));
+         }  
+      }
+
+      double coordFather[nvert][3];
+      int fatherIds[4]; //store the ids of the fathers (vertices) on the root face 
+      for(i=0; i< nvert ; i++) { 
+         int* fatherIdPtr;
+         const int exists = EN_getDataPtr((pEntity)vrts[i],myFather,(void**)&fatherIdPtr);
+         assert(exists);
+         fatherIds[i] = fatherIdPtr[0];
+         V_coord(vrts[i],coordFather[i]);
+         fprintf ( fcn, "%d ", fatherIds[i]);
+      }
+      fprintf ( fcn, "\n");
+  
+      dir=0;  // 1 fails
+   // get the upward adjacent region srcRgn
+      region = F_region(face, dir );  // 0 is the negative normal which I assume for a face on the boundary in is interior. 	
+      if(region==NULL) { // try other dir
+        dir=1;  // 1 fails
+        region = F_region(face, dir );  // 0 is the negative normal which I assume for a face on the boundary in is interior. 	
+      }
+
+      regions=PList_new();
+      faces=PList_new();
+      err = Extrusion_3DRegionsAndLayerFaces(region, regions, faces, 1); 
+      if(err!=1 && !PCU_Comm_Self())
+       fprintf(stderr, "Extrusion_3DRegionsAndLayerFaces returned %d for err \n", err);
     
-   // for each face in the returned list of faces
-   iter=0;
-   pFace sonFace;
-   int iface=0;
-   dir=0;
-   while( (sonFace = (pFace)PList_next(faces, &iter)) ) { //loop over plist of vertices
-    if(iface !=0) {  // root face is in the stack but we already took care of it above
-     // get the downward adjacent vertices of face - they will be in the same order as the srcFace ids
-     listVn= F_vertices(sonFace, dir);
-     void *iter2=0; // Must initialize to 0
-     i=0;
-     int my2Dfath;
-     pVertex  sonVtx;
-     double dist, dx, dy, distMin;
-     double coordSon[3];
-     int iMin;
-     while( (sonVtx = (pVertex)PList_next(listVn, &iter2)) ) { //loop over plist of vertices
-        V_coord(sonVtx,coordSon);
-        distMin=1.0e7;
-        for(i=0; i< nvert; i++){
-          dx=coordSon[0]-coordFather[i][0];
-          dy=coordSon[1]-coordFather[i][1];
-          dist=dx*dx+dy*dy;
-          if(dist < distMin) {
-             iMin=i;
-             distMin=dist;
-          }
+     // for each face in the returned list of faces
+      iter=0;
+      pFace sonFace;
+      int iface=0;
+      dir=0;
+      while( (sonFace = (pFace)PList_next(faces, &iter)) ) { //loop over plist of vertices
+        if(iface !=0) {  // root face is in the stack but we already took care of it above
+         // get the downward adjacent vertices of face - they will be in the same order as the srcFace ids
+          listVn= F_vertices(sonFace, dir);
+          void *iter2=0; // Must initialize to 0
+          i=0;
+          int my2Dfath;
+          pVertex  sonVtx;
+          double dist, dx, dy, distMin;
+          double coordSon[3];
+          int iMin;
+          while( (sonVtx = (pVertex)PList_next(listVn, &iter2)) ) { //loop over plist of vertices
+             V_coord(sonVtx,coordSon);
+             distMin=1.0e7;
+             for(i=0; i< nvert; i++){
+               dx=coordSon[0]-coordFather[i][0];
+               dy=coordSon[1]-coordFather[i][1];
+               dist=dx*dx+dy*dy;
+               if(dist < distMin) {
+                  iMin=i;
+                  distMin=dist;
+               }
+             }
+             my2Dfath=fatherIds[iMin];
+             int* vtxData = new int[1];
+             vtxData[0] = my2Dfath;
+             EN_attachDataPtr((pEntity)sonVtx,myFather,(void*)vtxData);
+           }
+           PList_delete(listVn);
         }
-        my2Dfath=fatherIds[iMin];
-        int* vtxData = new int[1];
-        vtxData[0] = my2Dfath;
-        EN_attachDataPtr((pEntity)sonVtx,myFather,(void*)vtxData);
-     }
-    PList_delete(listVn);
-    }
-    iface++;
-   }
-    PList_delete(faces);
-
-  } //end root face iterator
-
+        iface++;
+      }
+      PList_delete(faces);
+    } //end root face iterator
+  }
 
 
   apf::Mesh* simApfMesh = apf::createMesh(sim_mesh);

From 6af5498df9e500f640e9f2a3cf79388af9338e3f Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Fri, 14 Oct 2022 09:03:18 -0600
Subject: [PATCH 478/555] add test

---
 pumi-meshes        | 2 +-
 test/testing.cmake | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/pumi-meshes b/pumi-meshes
index d03989485..0d4e5548c 160000
--- a/pumi-meshes
+++ b/pumi-meshes
@@ -1 +1 @@
-Subproject commit d0398948585a3a0487f0e6cc61fe591573531fc4
+Subproject commit 0d4e5548c2ea341b3c54e173da8344ffb5f53f29
diff --git a/test/testing.cmake b/test/testing.cmake
index b9d7011e7..f9dc19341 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -78,6 +78,12 @@ if(ENABLE_SIMMETRIX AND SIM_PARASOLID AND SIMMODSUITE_SimAdvMeshing_FOUND)
   add_test(NAME chef-BL_query-diff
     COMMAND diff -r run_case/4-procs_case/ good_case/4-procs_case
     WORKING_DIRECTORY ${MDIR})
+  set(MDIR ${MESHES}/simExtrusionInfo)
+  mpi_test(convertExtrudedRoots 1 ${CMAKE_CURRENT_BINARY_DIR}/convert
+    --model-face-root=1
+    --native_model=geom.xmt_txt
+    geom.smd geom.sms mdsMesh.smb
+    WORKING_DIRECTORY ${MDIR})
 endif()
 
 if(ENABLE_SIMMETRIX AND SIM_PARASOLID AND SIMMODSUITE_SimAdvMeshing_FOUND)

From a15f1318e1f915142b311c48f6bc35d2c709d54f Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 14 Oct 2022 14:33:52 -0400
Subject: [PATCH 479/555] take the source face file as the input arg

pumi-meshes: source extrusion face for test was incorrect
---
 pumi-meshes        |   2 +-
 test/convert.cc    | 214 ++++++++++++++++++++++-----------------------
 test/testing.cmake |   6 +-
 3 files changed, 109 insertions(+), 113 deletions(-)

diff --git a/pumi-meshes b/pumi-meshes
index 0d4e5548c..73950fdfe 160000
--- a/pumi-meshes
+++ b/pumi-meshes
@@ -1 +1 @@
-Subproject commit 0d4e5548c2ea341b3c54e173da8344ffb5f53f29
+Subproject commit 73950fdfee8547925a60dc3f456767d52a4e80ad
diff --git a/test/convert.cc b/test/convert.cc
index 108d74cde..bf9144e1d 100644
--- a/test/convert.cc
+++ b/test/convert.cc
@@ -72,7 +72,7 @@ const char* smb_path = NULL;
 int should_log = 0;
 int should_fix_pyramids = 1;
 int should_attach_order = 0;
-int numExtruRootId =0;
+const char* extruRootPath = NULL;
 int ExtruRootId =0;
 bool found_bad_arg = false;
 
@@ -108,7 +108,7 @@ void getConfig(int argc, char** argv) {
       case 2: // enable simmetrix logging
         break;
       case 'e':
-        numExtruRootId = atoi(optarg);
+        extruRootPath = optarg;
         break;
       case 'n':
         gmi_native_path = optarg;
@@ -135,8 +135,8 @@ void getConfig(int argc, char** argv) {
   smb_path = argv[i++];
 
   if (!PCU_Comm_Self()) {
-    printf ("fix_pyramids %d attach_order %d enable_log %d numExtruRootId %d\n",
-            should_fix_pyramids, should_attach_order, should_log, numExtruRootId);
+    printf ("fix_pyramids %d attach_order %d enable_log %d extruRootPath %s\n",
+            should_fix_pyramids, should_attach_order, should_log, extruRootPath);
     printf ("native-model \'%s\' model \'%s\' simmetrix mesh \'%s\' output mesh \'%s\'\n",
       gmi_native_path, gmi_path, sms_path, smb_path);
   }
@@ -212,113 +212,113 @@ int main(int argc, char** argv)
   FILE* fcr = fopen(coordfilename, "w");
   FILE* fcn = fopen(cnnfilename, "w");
 
-  FILE* fid = fopen("ExruRootID.txt", "r"); // helper file that contains all faces with extrusions
-// fstream file("ExtruRootIDs.txt");
-  for(int i=0; i<numExtruRootId; ++i) { 
- // file >> ExtruRootId;
-    fscanf(fid,"%d",&ExtruRootId);
-    fprintf(stderr,"ExtruRootId= %d \n",ExtruRootId);
-    //find the root face of the extrusion
-    GFIter gfIter=GM_faceIter(simModel);
-    while ( (gface=GFIter_next(gfIter))) {
-      int id = GEN_tag(gface); 
-      if(id==ExtruRootId) ExtruRootFace=gface;
-    }
-    assert(ExtruRootFace != NULL);
-
-    FIter fIter = M_classifiedFaceIter( meshP, ExtruRootFace, 0 ); // 0 says I don't want closure	  
-    while ((face = FIter_next(fIter))) {
-      dir=1;
-      listV= F_vertices(face, dir); 
-      void *iter = 0;        // Must initialize to 0
-      int i=0;
-      while ((entV =(pVertex)PList_next(listV, &iter))) { //loop over plist of vertices
-        // Process each item in list
-        vrts[i] = (pVertex)entV;
-        i++;
-      }
-      int nvert=i;
-      PList_delete(listV);
-  
-      double coordNewPt[nvert][3];
-      for(i=0; i< nvert ; i++) { 
-         int* markedData;
-         if(!EN_getDataPtr((pEntity)vrts[i],myFather,(void**)&markedData)){  // not sure about marked yet
-           count2D++;
-           int* vtxData = new int[1];
-           vtxData[0] = count2D;
-           EN_attachDataPtr((pEntity)vrts[i],myFather,(void*)vtxData);
-           V_coord(vrts[i],coordNewPt[i]);
-  
-          fprintf ( fcr, "%.15E %.15E %d \n", coordNewPt[i][0],coordNewPt[i][1], V_whatInType(vrts[i]));
-         }  
+  if(extruRootPath) {
+    FILE* fid = fopen(extruRootPath, "r"); // helper file that contains all faces with extrusions
+    assert(fid);
+    while(1 == fscanf(fid,"%d",&ExtruRootId)) {
+      fprintf(stderr,"ExtruRootId= %d \n",ExtruRootId);
+      //find the root face of the extrusion
+      GFIter gfIter=GM_faceIter(simModel);
+      while ( (gface=GFIter_next(gfIter))) {
+        int id = GEN_tag(gface);
+        if(id==ExtruRootId) ExtruRootFace=gface;
       }
+      assert(ExtruRootFace != NULL);
 
-      double coordFather[nvert][3];
-      int fatherIds[4]; //store the ids of the fathers (vertices) on the root face 
-      for(i=0; i< nvert ; i++) { 
-         int* fatherIdPtr;
-         const int exists = EN_getDataPtr((pEntity)vrts[i],myFather,(void**)&fatherIdPtr);
-         assert(exists);
-         fatherIds[i] = fatherIdPtr[0];
-         V_coord(vrts[i],coordFather[i]);
-         fprintf ( fcn, "%d ", fatherIds[i]);
-      }
-      fprintf ( fcn, "\n");
-  
-      dir=0;  // 1 fails
-   // get the upward adjacent region srcRgn
-      region = F_region(face, dir );  // 0 is the negative normal which I assume for a face on the boundary in is interior. 	
-      if(region==NULL) { // try other dir
-        dir=1;  // 1 fails
-        region = F_region(face, dir );  // 0 is the negative normal which I assume for a face on the boundary in is interior. 	
-      }
+      FIter fIter = M_classifiedFaceIter( meshP, ExtruRootFace, 0 ); // 0 says I don't want closure
+      while ((face = FIter_next(fIter))) {
+        dir=1;
+        listV= F_vertices(face, dir);
+        void *iter = 0;        // Must initialize to 0
+        int i=0;
+        while ((entV =(pVertex)PList_next(listV, &iter))) { //loop over plist of vertices
+          // Process each item in list
+          vrts[i] = (pVertex)entV;
+          i++;
+        }
+        int nvert=i;
+        PList_delete(listV);
 
-      regions=PList_new();
-      faces=PList_new();
-      err = Extrusion_3DRegionsAndLayerFaces(region, regions, faces, 1); 
-      if(err!=1 && !PCU_Comm_Self())
-       fprintf(stderr, "Extrusion_3DRegionsAndLayerFaces returned %d for err \n", err);
-    
-     // for each face in the returned list of faces
-      iter=0;
-      pFace sonFace;
-      int iface=0;
-      dir=0;
-      while( (sonFace = (pFace)PList_next(faces, &iter)) ) { //loop over plist of vertices
-        if(iface !=0) {  // root face is in the stack but we already took care of it above
-         // get the downward adjacent vertices of face - they will be in the same order as the srcFace ids
-          listVn= F_vertices(sonFace, dir);
-          void *iter2=0; // Must initialize to 0
-          i=0;
-          int my2Dfath;
-          pVertex  sonVtx;
-          double dist, dx, dy, distMin;
-          double coordSon[3];
-          int iMin;
-          while( (sonVtx = (pVertex)PList_next(listVn, &iter2)) ) { //loop over plist of vertices
-             V_coord(sonVtx,coordSon);
-             distMin=1.0e7;
-             for(i=0; i< nvert; i++){
-               dx=coordSon[0]-coordFather[i][0];
-               dy=coordSon[1]-coordFather[i][1];
-               dist=dx*dx+dy*dy;
-               if(dist < distMin) {
+        double coordNewPt[nvert][3];
+        for(i=0; i< nvert ; i++) {
+          int* markedData;
+          if(!EN_getDataPtr((pEntity)vrts[i],myFather,(void**)&markedData)){  // not sure about marked yet
+            count2D++;
+            int* vtxData = new int[1];
+            vtxData[0] = count2D;
+            EN_attachDataPtr((pEntity)vrts[i],myFather,(void*)vtxData);
+            V_coord(vrts[i],coordNewPt[i]);
+
+            fprintf ( fcr, "%.15E %.15E %d \n", coordNewPt[i][0],coordNewPt[i][1], V_whatInType(vrts[i]));
+          }
+        }
+
+        double coordFather[nvert][3];
+        int fatherIds[4]; //store the ids of the fathers (vertices) on the root face
+        for(i=0; i< nvert ; i++) {
+          int* fatherIdPtr;
+          const int exists = EN_getDataPtr((pEntity)vrts[i],myFather,(void**)&fatherIdPtr);
+          assert(exists);
+          fatherIds[i] = fatherIdPtr[0];
+          V_coord(vrts[i],coordFather[i]);
+          fprintf ( fcn, "%d ", fatherIds[i]);
+        }
+        fprintf ( fcn, "\n");
+
+        dir=0;  // 1 fails
+        // get the upward adjacent region srcRgn
+        region = F_region(face, dir );  // 0 is the negative normal which I assume for a face on the boundary in is interior.
+        if(region==NULL) { // try other dir
+          dir=1;  // 1 fails
+          region = F_region(face, dir );  // 0 is the negative normal which I assume for a face on the boundary in is interior.
+        }
+
+        regions=PList_new();
+        faces=PList_new();
+        err = Extrusion_3DRegionsAndLayerFaces(region, regions, faces, 1);
+        if(err!=1 && !PCU_Comm_Self())
+          fprintf(stderr, "Extrusion_3DRegionsAndLayerFaces returned %d for err \n", err);
+
+        // for each face in the returned list of faces
+        iter=0;
+        pFace sonFace;
+        int iface=0;
+        dir=0;
+        while( (sonFace = (pFace)PList_next(faces, &iter)) ) { //loop over plist of vertices
+          if(iface !=0) {  // root face is in the stack but we already took care of it above
+            // get the downward adjacent vertices of face - they will be in the same order as the srcFace ids
+            listVn= F_vertices(sonFace, dir);
+            void *iter2=0; // Must initialize to 0
+            i=0;
+            int my2Dfath;
+            pVertex  sonVtx;
+            double dist, dx, dy, distMin;
+            double coordSon[3];
+            int iMin;
+            while( (sonVtx = (pVertex)PList_next(listVn, &iter2)) ) { //loop over plist of vertices
+              V_coord(sonVtx,coordSon);
+              distMin=1.0e7;
+              for(i=0; i< nvert; i++){
+                dx=coordSon[0]-coordFather[i][0];
+                dy=coordSon[1]-coordFather[i][1];
+                dist=dx*dx+dy*dy;
+                if(dist < distMin) {
                   iMin=i;
                   distMin=dist;
-               }
-             }
-             my2Dfath=fatherIds[iMin];
-             int* vtxData = new int[1];
-             vtxData[0] = my2Dfath;
-             EN_attachDataPtr((pEntity)sonVtx,myFather,(void*)vtxData);
-           }
-           PList_delete(listVn);
+                }
+              }
+              my2Dfath=fatherIds[iMin];
+              int* vtxData = new int[1];
+              vtxData[0] = my2Dfath;
+              EN_attachDataPtr((pEntity)sonVtx,myFather,(void*)vtxData);
+            }
+            PList_delete(listVn);
+          }
+          iface++;
         }
-        iface++;
-      }
-      PList_delete(faces);
-    } //end root face iterator
+        PList_delete(faces);
+      } //end root face iterator
+    }
   }
 
 
@@ -343,10 +343,6 @@ int main(int argc, char** argv)
   mesh->verify();
   mesh->writeNative(smb_path);
 
-  auto ft = mesh->findTag("fathers2D");
-  convert_my_tag(mesh, ft);
-//  apf::writeVtkFiles("foo", mesh);
-
   mesh->destroyNative();
   apf::destroyMesh(mesh);
 
diff --git a/test/testing.cmake b/test/testing.cmake
index f9dc19341..bff7b17b9 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -80,9 +80,9 @@ if(ENABLE_SIMMETRIX AND SIM_PARASOLID AND SIMMODSUITE_SimAdvMeshing_FOUND)
     WORKING_DIRECTORY ${MDIR})
   set(MDIR ${MESHES}/simExtrusionInfo)
   mpi_test(convertExtrudedRoots 1 ${CMAKE_CURRENT_BINARY_DIR}/convert
-    --model-face-root=1
-    --native_model=geom.xmt_txt
-    geom.smd geom.sms mdsMesh.smb
+    --model-face-root=${MDIR}/ExtruRootID.txt
+    --native_model=${MDIR}/geom.xmt_txt
+    ${MDIR}/geom.smd ${MDIR}/geom.sms ${MDIR}/mdsMesh.smb
     WORKING_DIRECTORY ${MDIR})
 endif()
 

From e51fdad2940e07bdedbcb0f33477de59673b2fa9 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 14 Oct 2022 15:09:54 -0400
Subject: [PATCH 480/555] move father tag code into function

---
 test/convert.cc | 282 ++++++++++++++++++++++++------------------------
 1 file changed, 143 insertions(+), 139 deletions(-)

diff --git a/test/convert.cc b/test/convert.cc
index bf9144e1d..707a8d672 100644
--- a/test/convert.cc
+++ b/test/convert.cc
@@ -95,7 +95,7 @@ void getConfig(int argc, char** argv) {
     "  --no-pyramid-fix                Disable quad-connected pyramid tetrahedronization\n"
     "  --attach-order                  Attach the Simmetrix element order as a Numbering\n"
     "  --enable-log                    Enable Simmetrix logging\n"
-    "  --model-face-root               Number of Model faces that are roots of extrusions from SimModeler\n"
+    "  --model-face-root=/path/to/file ASCII input file with one integer per line listing the face ids that are the roots of mesh extrusions from SimModeler\n"
     "  --native-model=/path/to/model   Load the native Parasolid or ACIS model that the GeomSim model uses\n";
 
   int option_index = 0;
@@ -142,43 +142,6 @@ void getConfig(int argc, char** argv) {
   }
 }
 
-int main(int argc, char** argv)
-{
-  MPI_Init(&argc, &argv);
-  PCU_Comm_Init();
-  lion_set_verbosity(1);
-  MS_init();
-  SimAdvMeshing_start(); //for fancy BL/extrusion queries
-  SimModel_start();
-  Sim_readLicenseFile(NULL);
-  SimPartitionedMesh_start(&argc,&argv);
-
-  getConfig(argc, argv);
-  if( should_log )
-    Sim_logOn("convert.sim.log");
-
-  if (should_attach_order && should_fix_pyramids) {
-    if (!PCU_Comm_Self())
-      std::cout << "disabling pyramid fix because --attach-order was given\n";
-    should_fix_pyramids = false;
-  }
-
-  gmi_sim_start();
-  gmi_register_sim();
-  pProgress progress = Progress_new();
-  Progress_setDefaultCallback(progress);
-
-  gmi_model* mdl;
-  if( gmi_native_path )
-    mdl = gmi_sim_load(gmi_native_path,gmi_path);
-  else
-    mdl = gmi_load(gmi_path);
-  pGModel simModel = gmi_export_sim(mdl);
-  double t0 = PCU_Time();
-  pParMesh sim_mesh = PM_load(sms_path, simModel, progress);
-  double t1 = PCU_Time();
-  if(!PCU_Comm_Self())
-    fprintf(stderr, "read and created the simmetrix mesh in %f seconds\n", t1-t0);
 // put the extrude tagging here which 1) loops over the mesh faces classified on the model face that is the root of the extrude
 // create a tag on vertices fathers
 // get the list of mesh rootfaces classified on the source geometric model face
@@ -189,8 +152,10 @@ int main(int argc, char** argv)
 // for each face in the returned list of faces
 // get the downward adjacent vertices of face - they will be in the same order as the srcFace ids
 // set the fathers tag
-// assert that the x,y coordinates of each vertex matches the srcFace vertex coordinates within some relaxed tolerance - sanity check my assumption that face-to-vtx adjaceny is always the same order
-  
+// assert that the x,y coordinates of each vertex matches the srcFace vertex coordinates within some relaxed
+// tolerance - sanity check my assumption that face-to-vtx adjaceny is always the same order
+void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, const char* extrusionFaceFile) {
+  if(!extrusionFaceFile) return;
   // create a tag on vertices fathers
   pMeshDataId myFather = MD_newMeshDataId( "fathers2D");
   
@@ -212,119 +177,158 @@ int main(int argc, char** argv)
   FILE* fcr = fopen(coordfilename, "w");
   FILE* fcn = fopen(cnnfilename, "w");
 
-  if(extruRootPath) {
-    FILE* fid = fopen(extruRootPath, "r"); // helper file that contains all faces with extrusions
-    assert(fid);
-    while(1 == fscanf(fid,"%d",&ExtruRootId)) {
-      fprintf(stderr,"ExtruRootId= %d \n",ExtruRootId);
-      //find the root face of the extrusion
-      GFIter gfIter=GM_faceIter(simModel);
-      while ( (gface=GFIter_next(gfIter))) {
-        int id = GEN_tag(gface);
-        if(id==ExtruRootId) ExtruRootFace=gface;
+  FILE* fid = fopen(extrusionFaceFile, "r"); // helper file that contains all faces with extrusions
+  assert(fid);
+  while(1 == fscanf(fid,"%d",&ExtruRootId)) {
+    fprintf(stderr,"ExtruRootId= %d \n",ExtruRootId);
+    //find the root face of the extrusion
+    GFIter gfIter=GM_faceIter(simModel);
+    while ( (gface=GFIter_next(gfIter))) {
+      int id = GEN_tag(gface);
+      if(id==ExtruRootId) ExtruRootFace=gface;
+    }
+    assert(ExtruRootFace != NULL);
+
+    FIter fIter = M_classifiedFaceIter( meshP, ExtruRootFace, 0 ); // 0 says I don't want closure
+    while ((face = FIter_next(fIter))) {
+      dir=1;
+      listV= F_vertices(face, dir);
+      void *iter = 0;        // Must initialize to 0
+      int i=0;
+      while ((entV =(pVertex)PList_next(listV, &iter))) { //loop over plist of vertices
+        // Process each item in list
+        vrts[i] = (pVertex)entV;
+        i++;
       }
-      assert(ExtruRootFace != NULL);
-
-      FIter fIter = M_classifiedFaceIter( meshP, ExtruRootFace, 0 ); // 0 says I don't want closure
-      while ((face = FIter_next(fIter))) {
-        dir=1;
-        listV= F_vertices(face, dir);
-        void *iter = 0;        // Must initialize to 0
-        int i=0;
-        while ((entV =(pVertex)PList_next(listV, &iter))) { //loop over plist of vertices
-          // Process each item in list
-          vrts[i] = (pVertex)entV;
-          i++;
-        }
-        int nvert=i;
-        PList_delete(listV);
-
-        double coordNewPt[nvert][3];
-        for(i=0; i< nvert ; i++) {
-          int* markedData;
-          if(!EN_getDataPtr((pEntity)vrts[i],myFather,(void**)&markedData)){  // not sure about marked yet
-            count2D++;
-            int* vtxData = new int[1];
-            vtxData[0] = count2D;
-            EN_attachDataPtr((pEntity)vrts[i],myFather,(void*)vtxData);
-            V_coord(vrts[i],coordNewPt[i]);
-
-            fprintf ( fcr, "%.15E %.15E %d \n", coordNewPt[i][0],coordNewPt[i][1], V_whatInType(vrts[i]));
-          }
+      int nvert=i;
+      PList_delete(listV);
+
+      double coordNewPt[nvert][3];
+      for(i=0; i< nvert ; i++) {
+        int* markedData;
+        if(!EN_getDataPtr((pEntity)vrts[i],myFather,(void**)&markedData)){  // not sure about marked yet
+          count2D++;
+          int* vtxData = new int[1];
+          vtxData[0] = count2D;
+          EN_attachDataPtr((pEntity)vrts[i],myFather,(void*)vtxData);
+          V_coord(vrts[i],coordNewPt[i]);
+
+          fprintf ( fcr, "%.15E %.15E %d \n", coordNewPt[i][0],coordNewPt[i][1], V_whatInType(vrts[i]));
         }
+      }
 
-        double coordFather[nvert][3];
-        int fatherIds[4]; //store the ids of the fathers (vertices) on the root face
-        for(i=0; i< nvert ; i++) {
-          int* fatherIdPtr;
-          const int exists = EN_getDataPtr((pEntity)vrts[i],myFather,(void**)&fatherIdPtr);
-          assert(exists);
-          fatherIds[i] = fatherIdPtr[0];
-          V_coord(vrts[i],coordFather[i]);
-          fprintf ( fcn, "%d ", fatherIds[i]);
-        }
-        fprintf ( fcn, "\n");
+      double coordFather[nvert][3];
+      int fatherIds[4]; //store the ids of the fathers (vertices) on the root face
+      for(i=0; i< nvert ; i++) {
+        int* fatherIdPtr;
+        const int exists = EN_getDataPtr((pEntity)vrts[i],myFather,(void**)&fatherIdPtr);
+        assert(exists);
+        fatherIds[i] = fatherIdPtr[0];
+        V_coord(vrts[i],coordFather[i]);
+        fprintf ( fcn, "%d ", fatherIds[i]);
+      }
+      fprintf ( fcn, "\n");
 
-        dir=0;  // 1 fails
-        // get the upward adjacent region srcRgn
+      dir=0;  // 1 fails
+      // get the upward adjacent region srcRgn
+      region = F_region(face, dir );  // 0 is the negative normal which I assume for a face on the boundary in is interior.
+      if(region==NULL) { // try other dir
+        dir=1;  // 1 fails
         region = F_region(face, dir );  // 0 is the negative normal which I assume for a face on the boundary in is interior.
-        if(region==NULL) { // try other dir
-          dir=1;  // 1 fails
-          region = F_region(face, dir );  // 0 is the negative normal which I assume for a face on the boundary in is interior.
-        }
+      }
 
-        regions=PList_new();
-        faces=PList_new();
-        err = Extrusion_3DRegionsAndLayerFaces(region, regions, faces, 1);
-        if(err!=1 && !PCU_Comm_Self())
-          fprintf(stderr, "Extrusion_3DRegionsAndLayerFaces returned %d for err \n", err);
-
-        // for each face in the returned list of faces
-        iter=0;
-        pFace sonFace;
-        int iface=0;
-        dir=0;
-        while( (sonFace = (pFace)PList_next(faces, &iter)) ) { //loop over plist of vertices
-          if(iface !=0) {  // root face is in the stack but we already took care of it above
-            // get the downward adjacent vertices of face - they will be in the same order as the srcFace ids
-            listVn= F_vertices(sonFace, dir);
-            void *iter2=0; // Must initialize to 0
-            i=0;
-            int my2Dfath;
-            pVertex  sonVtx;
-            double dist, dx, dy, distMin;
-            double coordSon[3];
-            int iMin;
-            while( (sonVtx = (pVertex)PList_next(listVn, &iter2)) ) { //loop over plist of vertices
-              V_coord(sonVtx,coordSon);
-              distMin=1.0e7;
-              for(i=0; i< nvert; i++){
-                dx=coordSon[0]-coordFather[i][0];
-                dy=coordSon[1]-coordFather[i][1];
-                dist=dx*dx+dy*dy;
-                if(dist < distMin) {
-                  iMin=i;
-                  distMin=dist;
-                }
+      regions=PList_new();
+      faces=PList_new();
+      err = Extrusion_3DRegionsAndLayerFaces(region, regions, faces, 1);
+      if(err!=1 && !PCU_Comm_Self())
+        fprintf(stderr, "Extrusion_3DRegionsAndLayerFaces returned %d for err \n", err);
+
+      // for each face in the returned list of faces
+      iter=0;
+      pFace sonFace;
+      int iface=0;
+      dir=0;
+      while( (sonFace = (pFace)PList_next(faces, &iter)) ) { //loop over plist of vertices
+        if(iface !=0) {  // root face is in the stack but we already took care of it above
+          // get the downward adjacent vertices of face - they will be in the same order as the srcFace ids
+          listVn= F_vertices(sonFace, dir);
+          void *iter2=0; // Must initialize to 0
+          i=0;
+          int my2Dfath;
+          pVertex  sonVtx;
+          double dist, dx, dy, distMin;
+          double coordSon[3];
+          int iMin;
+          while( (sonVtx = (pVertex)PList_next(listVn, &iter2)) ) { //loop over plist of vertices
+            V_coord(sonVtx,coordSon);
+            distMin=1.0e7;
+            for(i=0; i< nvert; i++){
+              dx=coordSon[0]-coordFather[i][0];
+              dy=coordSon[1]-coordFather[i][1];
+              dist=dx*dx+dy*dy;
+              if(dist < distMin) {
+                iMin=i;
+                distMin=dist;
               }
-              my2Dfath=fatherIds[iMin];
-              int* vtxData = new int[1];
-              vtxData[0] = my2Dfath;
-              EN_attachDataPtr((pEntity)sonVtx,myFather,(void*)vtxData);
             }
-            PList_delete(listVn);
+            my2Dfath=fatherIds[iMin];
+            int* vtxData = new int[1];
+            vtxData[0] = my2Dfath;
+            EN_attachDataPtr((pEntity)sonVtx,myFather,(void*)vtxData);
           }
-          iface++;
+          PList_delete(listVn);
         }
-        PList_delete(faces);
-      } //end root face iterator
-    }
+        iface++;
+      }
+      PList_delete(faces);
+    } //end root face iterator
   }
+  apf::MeshSIM* cake = reinterpret_cast<apf::MeshSIM*>(simApfMesh);
+  cake->createIntTag("fathers2D", myFather, 1);
+}
+
+int main(int argc, char** argv)
+{
+  MPI_Init(&argc, &argv);
+  PCU_Comm_Init();
+  lion_set_verbosity(1);
+  MS_init();
+  SimAdvMeshing_start(); //for fancy BL/extrusion queries
+  SimModel_start();
+  Sim_readLicenseFile(NULL);
+  SimPartitionedMesh_start(&argc,&argv);
 
+  getConfig(argc, argv);
+  if( should_log )
+    Sim_logOn("convert.sim.log");
+
+  if (should_attach_order && should_fix_pyramids) {
+    if (!PCU_Comm_Self())
+      std::cout << "disabling pyramid fix because --attach-order was given\n";
+    should_fix_pyramids = false;
+  }
+
+  gmi_sim_start();
+  gmi_register_sim();
+  pProgress progress = Progress_new();
+  Progress_setDefaultCallback(progress);
+
+  gmi_model* mdl;
+  if( gmi_native_path )
+    mdl = gmi_sim_load(gmi_native_path,gmi_path);
+  else
+    mdl = gmi_load(gmi_path);
+  pGModel simModel = gmi_export_sim(mdl);
+  double t0 = PCU_Time();
+  pParMesh sim_mesh = PM_load(sms_path, simModel, progress);
+  double t1 = PCU_Time();
+  if(!PCU_Comm_Self())
+    fprintf(stderr, "read and created the simmetrix mesh in %f seconds\n", t1-t0);
 
   apf::Mesh* simApfMesh = apf::createMesh(sim_mesh);
-  apf::MeshSIM* cake = reinterpret_cast<apf::MeshSIM*>(simApfMesh);
-  cake->createIntTag("fathers2D", myFather, 1);
+
+  addFathersTag(simModel, sim_mesh, simApfMesh, extruRootPath);
+
   double t2 = PCU_Time();
   if(!PCU_Comm_Self())
     fprintf(stderr, "created the apf_sim mesh in %f seconds\n", t2-t1);

From 25e9f26a342635acff09f57d8814de636d9ccd76 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 21 Oct 2022 12:55:14 -0400
Subject: [PATCH 481/555] typo

---
 test/testing.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/testing.cmake b/test/testing.cmake
index bff7b17b9..866e02afe 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -81,7 +81,7 @@ if(ENABLE_SIMMETRIX AND SIM_PARASOLID AND SIMMODSUITE_SimAdvMeshing_FOUND)
   set(MDIR ${MESHES}/simExtrusionInfo)
   mpi_test(convertExtrudedRoots 1 ${CMAKE_CURRENT_BINARY_DIR}/convert
     --model-face-root=${MDIR}/ExtruRootID.txt
-    --native_model=${MDIR}/geom.xmt_txt
+    --native-model=${MDIR}/geom.xmt_txt
     ${MDIR}/geom.smd ${MDIR}/geom.sms ${MDIR}/mdsMesh.smb
     WORKING_DIRECTORY ${MDIR})
 endif()

From 26af7dbe81ebe01e8a80612da83792239019b93a Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 21 Oct 2022 13:02:14 -0400
Subject: [PATCH 482/555] update pumi-meshes

---
 pumi-meshes | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pumi-meshes b/pumi-meshes
index 73950fdfe..7b84de7e3 160000
--- a/pumi-meshes
+++ b/pumi-meshes
@@ -1 +1 @@
-Subproject commit 73950fdfee8547925a60dc3f456767d52a4e80ad
+Subproject commit 7b84de7e31d0d539acf83837cbce2dc0702c2bbc

From 6dfbc4fb7b89424379dd5f927741664ca0f10445 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 21 Oct 2022 13:04:59 -0400
Subject: [PATCH 483/555] compare output against expected results

---
 test/testing.cmake | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/test/testing.cmake b/test/testing.cmake
index 866e02afe..9dc958082 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -84,6 +84,12 @@ if(ENABLE_SIMMETRIX AND SIM_PARASOLID AND SIMMODSUITE_SimAdvMeshing_FOUND)
     --native-model=${MDIR}/geom.xmt_txt
     ${MDIR}/geom.smd ${MDIR}/geom.sms ${MDIR}/mdsMesh.smb
     WORKING_DIRECTORY ${MDIR})
+  add_test(NAME convertExtrudedRoots_diff_cnn
+    COMMAND diff -r geom.cnn geom_expected.cnn
+    WORKING_DIRECTORY ${MDIR})
+  add_test(NAME convertExtrudedRoots_diff_crd
+    COMMAND diff -r geom.crd geom_expected.crd
+    WORKING_DIRECTORY ${MDIR})
 endif()
 
 if(ENABLE_SIMMETRIX AND SIM_PARASOLID AND SIMMODSUITE_SimAdvMeshing_FOUND)

From d7e64eb7a4133cb29862c69587baf484abad068f Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Tue, 1 Nov 2022 11:55:41 -0400
Subject: [PATCH 484/555] simTranslate: string not code

---
 test/simTranslate.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/simTranslate.cc b/test/simTranslate.cc
index c439f244f..535ded1d1 100644
--- a/test/simTranslate.cc
+++ b/test/simTranslate.cc
@@ -58,7 +58,7 @@ static std::string paraExtshort = ".x_t";
 void printSimError(pSimError err) {
   printf("Simmetrix error caught:\n");
   printf("  Error code: %d\n",SIM_ERROR(code,err));
-  printf("  Error string: %s\n",SIM_ERROR(code,err));
+  printf("  Error string: %s\n",SIM_ERROR(toString,err));
   SIM_ERROR(delete,err);
 }
 

From d24a838cd08e0d20e1815a2cda416c2e1ddca1bf Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Tue, 1 Nov 2022 12:28:23 -0400
Subject: [PATCH 485/555] increase minimum simmetrix version to 15

Extrusion_3DRegionsAndLayerFaces is needed for test/convert
---
 cmake/FindSimModSuite.cmake | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmake/FindSimModSuite.cmake b/cmake/FindSimModSuite.cmake
index d935a8661..ccbb3d7a6 100644
--- a/cmake/FindSimModSuite.cmake
+++ b/cmake/FindSimModSuite.cmake
@@ -83,8 +83,8 @@ string(REGEX REPLACE
   "\\3" SIMMODSUITE_MINOR_VERSION
   "${SIM_VERSION}")
 
-set(MIN_VALID_SIM_VERSION 12.0.190225)
-set(MAX_VALID_SIM_VERSION 18.0-220930)
+set(MIN_VALID_SIM_VERSION 15.0.191017)
+set(MAX_VALID_SIM_VERSION 18.0.220930)
 if( ${SKIP_SIMMETRIX_VERSION_CHECK} )
   message(STATUS "Skipping Simmetrix SimModSuite version check."
     " This may result in undefined behavior")

From 2b958a91bc2fd3cc9eb5c771e8a6b36063830b91 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Tue, 1 Nov 2022 13:20:24 -0400
Subject: [PATCH 486/555] update modules

---
 cdash/nightly.sh | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/cdash/nightly.sh b/cdash/nightly.sh
index d0438e05b..80ac4af73 100755
--- a/cdash/nightly.sh
+++ b/cdash/nightly.sh
@@ -7,13 +7,14 @@ export PATH=/usr/share/lmod/lmod/libexec:$PATH
 
 #setup spack modules
 unset MODULEPATH
-module use /opt/scorec/spack/lmod/linux-rhel7-x86_64/Core/
 
-module load gcc/7.3.0-bt47fwr
-module load cmake/3.12.1-wfk2b7e
-module load mpich
-module load zoltan/3.83-int32-gx3prjr
-module load simmetrix-simmodsuite/14.0-190928-snlypqg
+module unuse /opt/scorec/spack/lmod/linux-rhel7-x86_64/Core
+module use /opt/scorec/spack/v0154_2/lmod/linux-rhel7-x86_64/Core
+module load gcc/10.1.0
+module load mpich/3.3.2
+module load simmetrix-simmodsuite/17.0-220516
+module load zoltan/3.83-int32
+module load cmake/3.20.0
 
 #cdash output root
 d=/lore/cwsmith/nightlyBuilds/

From b0a7ecf344ef7f0be5085ae4bae259bf49934cd0 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Tue, 1 Nov 2022 14:45:54 -0400
Subject: [PATCH 487/555] nightly test scripts

---
 cdash/nightly.cmake | 8 ++++----
 cdash/nightly.sh    | 5 +++--
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/cdash/nightly.cmake b/cdash/nightly.cmake
index 32fcbec69..596d3f5d5 100644
--- a/cdash/nightly.cmake
+++ b/cdash/nightly.cmake
@@ -1,10 +1,11 @@
-cmake_minimum_required(VERSION 2.8)
+cmake_minimum_required(VERSION 2.17)
 
 SET(CTEST_DO_SUBMIT ON)
 SET(CTEST_TEST_TYPE Nightly)
+set(CTEST_BUILD_CONFIGURATION RelWithDebInfo)
 
 set(CTEST_NIGHTLY_START_TIME "17:00:00 EST")
-set(CTEST_SITE "pachisi.scorec.rpi.edu" )
+set(CTEST_SITE "cranium.scorec.rpi.edu" )
 set(CTEST_DROP_METHOD "http")
 set(CTEST_DROP_SITE "my.cdash.org")
 set(CTEST_DROP_LOCATION "/submit.php?project=SCOREC")
@@ -13,7 +14,6 @@ set(CTEST_BUILD_NAME  "linux-gcc-${CTEST_BUILD_CONFIGURATION}")
 
 set(CTEST_DASHBOARD_ROOT "/lore/cwsmith/nightlyBuilds/" )
 set(CTEST_CMAKE_GENERATOR "Unix Makefiles")
-set(CTEST_BUILD_CONFIGURATION RelWithDebInfo)
 set(CTEST_BUILD_FLAGS -j4)
 
 set(CTEST_PROJECT_NAME "SCOREC")
@@ -313,7 +313,7 @@ SET(CONFIGURE_OPTIONS-sim
   "${CONFIGURE_OPTIONS}"
   "-DENABLE_SIMMETRIX:BOOL=ON"
   "-DSIM_PARASOLID:BOOL=ON"
-  "-DSIM_MPI:STRING=mpich3.3"
+  "-DSIM_MPI:STRING=mpich3.3.2"
 )
 
 setup_repo()
diff --git a/cdash/nightly.sh b/cdash/nightly.sh
index 80ac4af73..cf4c37b9b 100755
--- a/cdash/nightly.sh
+++ b/cdash/nightly.sh
@@ -9,6 +9,7 @@ export PATH=/usr/share/lmod/lmod/libexec:$PATH
 unset MODULEPATH
 
 module unuse /opt/scorec/spack/lmod/linux-rhel7-x86_64/Core
+module use /opt/scorec/modules
 module use /opt/scorec/spack/v0154_2/lmod/linux-rhel7-x86_64/Core
 module load gcc/10.1.0
 module load mpich/3.3.2
@@ -24,10 +25,10 @@ cd $d
 
 touch $d/startedCoreNightly
 #run nightly.cmake script
-ctest -V --script $d/repos/core/cdash/nightly.cmake
+ctest -V --script $d/nightly.cmake
 touch $d/doneCoreNightly
 
 #create doxygen docs
 cd build/master
 make doc
-cp -r doc/html/* /net/web/scorec/scorec-web/htdocs/pumi/doxygen/.
+cp -r doc/html /net/web/scorec/scorec-web/htdocs/pumi/doxygen

From 36ad9e838bd79fb9bb18dcff83d8ad4fc9f6da5a Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Tue, 8 Nov 2022 14:37:35 -0700
Subject: [PATCH 488/555] unknown state after Friday Nov 4th

---
 test/matchedNodeElmReader.cc | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index a0b5a236d..88aad9d65 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -684,17 +684,17 @@ void readElements(FILE* f, FILE* fh, unsigned &dim,  apf::Gid& numElms,
 struct MeshInfo {
   double* coords;
   double* solution;
-  apf::Gid* elements;
+  apf::Gid* elements; //TODO store per block
   apf::Gid* matches;
   int* classification;
   int* fathers2D;
   unsigned dim;
-  unsigned elementType;
+  unsigned elementType; //TODO remove
   apf::Gid numVerts;
   int localNumVerts;
-  apf::Gid numElms;
-  int localNumElms;
-  unsigned numVtxPerElm;
+  apf::Gid numElms; //TODO does not appear to be used
+  int localNumElms; //TODO store per block 
+  unsigned numVtxPerElm; //TODO can be a pumi query?
 };
 
 void readMesh(const char* meshfilename,
@@ -758,6 +758,7 @@ void readMesh(const char* meshfilename,
   FILE* fh = fopen(connHeadfilename, "r");
   PCU_ALWAYS_ASSERT(f);
   PCU_ALWAYS_ASSERT(fh);
+//  now we went to do a readElements for each topology so 
   readElements(f,fh, mesh.dim, mesh.numElms, mesh.numVtxPerElm,
       mesh.localNumElms, &(mesh.elements));
   mesh.elementType = getElmType(mesh.dim, mesh.numVtxPerElm);
@@ -809,6 +810,9 @@ int main(int argc, char** argv)
   apf::GlobalToVert outMap;
   PCU_Debug_Open();
   apf::construct(mesh, m.elements, m.localNumElms, m.elementType, outMap);
+// PLANNING: before we can call what used to be construct but is now assemble and finalise we will need to batch elements by topology
+// in the first pass we will keep the upstream reader that  has these as rectangular arrays and do a sort here into one 
+// group o
   delete [] m.elements;
   apf::alignMdsRemotes(mesh);
   apf::deriveMdsModel(mesh);

From 9901e72f758753794950c71f681c9cc3ca3ca3b8 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 9 Dec 2022 13:03:43 -0500
Subject: [PATCH 489/555] hack - avoid compilation error in phOutput.cc

---
 phasta/phInput.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/phasta/phInput.h b/phasta/phInput.h
index 79728b4f4..a6bf88c90 100644
--- a/phasta/phInput.h
+++ b/phasta/phInput.h
@@ -23,6 +23,7 @@ class Input
   public:
     Input();
     void load(const char* filename);
+    int txtCoord; //HACK added to get through compile
     int timeStepNumber;
     /** \brief this corresponds to the number of degrees of
       freedom in the solution field of the output restart file.

From 34823474c2eb9976e3871f0276d316b9a3efd507 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 9 Dec 2022 13:41:08 -0500
Subject: [PATCH 490/555] model only box creation function

---
 mds/apfBox.cc | 11 ++++++++++-
 mds/apfBox.h  |  7 ++++++-
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/mds/apfBox.cc b/mds/apfBox.cc
index 56154c916..bf20a192e 100644
--- a/mds/apfBox.cc
+++ b/mds/apfBox.cc
@@ -330,7 +330,15 @@ void BoxBuilder::buildMeshAndModel()
   m->acceptChanges();
 }
 
-gmi_model* makeMdsBox(
+Mesh2* makeMdsBox(
+    int nex, int ney, int nez,
+    double wx, double wy, double wz, bool is)
+{
+  BoxBuilder bb(nex, ney, nez, wx, wy, wz, is);
+  return bb.m;
+}
+
+gmi_model* makeMdsBoxModel(
     int nex, int ney, int nez,
     double wx, double wy, double wz, bool is)
 {
@@ -338,4 +346,5 @@ gmi_model* makeMdsBox(
   return bb.buildModel();
 }
 
+
 }
diff --git a/mds/apfBox.h b/mds/apfBox.h
index 33d6c94b5..035a29b78 100644
--- a/mds/apfBox.h
+++ b/mds/apfBox.h
@@ -70,9 +70,14 @@ struct BoxBuilder
   \param wz z dimension width
   \param is true = simplical mesh, false = quad/hex
   \details set ny,nz=0 for a 1D mesh, set nz=0 for a 2D mesh */
-gmi_model* makeMdsBox(
+Mesh2* makeMdsBox(
     int nx, int ny, int nz, double wx, double wy, double wz, bool is);
 
+/** \brief see makeMdsBox - only creates geometric model */
+gmi_model* makeMdsBoxModel(
+    int nx, int ny, int nz, double wx, double wy, double wz, bool is);
+
+
 }
 
 #endif

From 0d733d3f7f06bcccf893e6f8470b98c41caacb50 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 9 Dec 2022 13:42:55 -0500
Subject: [PATCH 491/555] use Gids in [con|de]struct call

---
 test/constructThenGhost.cc |  2 +-
 test/embedded_edges.cc     |  2 +-
 test/runSimxAnisoAdapt.cc  | 23 +++++++++++++++--------
 3 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/test/constructThenGhost.cc b/test/constructThenGhost.cc
index 4caafe06a..663fc01eb 100644
--- a/test/constructThenGhost.cc
+++ b/test/constructThenGhost.cc
@@ -18,7 +18,7 @@ int main(int argc, char** argv)
   lion_set_verbosity(1);
   gmi_register_mesh();
   gmi_register_null();
-  int* conn;
+  apf::Gid* conn;
   double* coords;
   int nelem;
   int etype;
diff --git a/test/embedded_edges.cc b/test/embedded_edges.cc
index 74531aae3..be0948569 100644
--- a/test/embedded_edges.cc
+++ b/test/embedded_edges.cc
@@ -21,7 +21,7 @@ int main(int argc, char** argv)
   lion_set_verbosity(1);
   gmi_register_mesh();
   gmi_register_null();
-  int* conn;
+  apf::Gid* conn;
   double* coords;
   int nelem;
   int etype;
diff --git a/test/runSimxAnisoAdapt.cc b/test/runSimxAnisoAdapt.cc
index d0aeacc16..bfb399a38 100644
--- a/test/runSimxAnisoAdapt.cc
+++ b/test/runSimxAnisoAdapt.cc
@@ -30,7 +30,7 @@ typedef vector<vec>    mat;
 void printModelStats(pGModel model);
 void makeSimxModelAndMesh(
     double* coords, int nverts,
-    int*    conn,   int nelem,
+    apf::Gid*    conn,   int nelem,
     pMesh& mesh, pDiscreteModel& model,
     pVertex* vReturn, pEntity* eReturn);
 bool checkVertexOrder(
@@ -147,7 +147,7 @@ int main(int argc, char** argv)
   m->verify();
 
   // extract the coordinates and connectivities
-  int* conn;
+  apf::Gid* conn;
   double* coords;
   int nelem;
   int etype;
@@ -258,7 +258,7 @@ void printModelStats(pGModel model)
 
 void makeSimxModelAndMesh(
     double* coords, int nverts,
-    int*    conn,   int nelem,
+    apf::Gid*    connGid,   int nelem,
     pMesh& mesh, pDiscreteModel& model,
     pVertex* vReturn, pEntity* eReturn)
 {
@@ -269,6 +269,12 @@ void makeSimxModelAndMesh(
 
   Sim_setMessageHandler(0);
 
+  const int connSize = 4*nelem;
+  int* conn = new int[connSize];
+  for(int i=0; i<connSize; i++) {
+    conn[i] = static_cast<int>(connGid[i]);
+  }
+
   mesh = M_new(0,0);
   if(M_importFromData(mesh,nverts,coords,nelem,
       elementType,conn,vReturn,eReturn,0)) { //check for error 
@@ -300,6 +306,7 @@ void makeSimxModelAndMesh(
     GM_release(model);
     return;
   }
+  delete [] conn;
 }
 
 bool checkVertexOrder(
@@ -500,7 +507,7 @@ static void getSizeAndFramesFromArray(
 void destructSimxMesh(
     pMesh mesh,
     double*& adaptedCoords,
-    int*& adaptedConns,
+    apf::Gid*& adaptedConns,
     int& nverts, int& nelem,
     vector<apf::Vector3>&  adaptedSizes,
     vector<apf::Matrix3x3>& adaptedFrames)
@@ -509,7 +516,7 @@ void destructSimxMesh(
   nelem  = countRegions(mesh);
 
   adaptedCoords = new double[3*nverts];
-  adaptedConns  = new int[4*nelem];
+  adaptedConns  = new apf::Gid[4*nelem];
 
   VIter vertices;
   RIter regions;
@@ -540,10 +547,10 @@ void destructSimxMesh(
   i=0;
   while( (region = RIter_next(regions)) ){
     regionVerts = R_vertices(region,0);
-    vector<int> ids;
+    vector<apf::Gid> ids;
     for(j=0; j < 4; j++){
       vertex = (pVertex)PList_item(regionVerts,j);
-      int id = EN_id((pEntity)vertex);
+      apf::Gid id = EN_id((pEntity)vertex);
       ids.push_back(id);
     }
     // simmetrix's local ordering is different from pumi's
@@ -581,7 +588,7 @@ apf::Mesh2* convertToPumi(
     const char* frameName)
 {
   double* adaptedCoords;
-  int*	  adaptedConns;
+  apf::Gid* adaptedConns;
   int adaptedNumVerts, adaptedNumElems;
   vector<apf::Vector3> adaptedSizes;
   vector<apf::Matrix3x3> adaptedFrames;

From ecfb60c2148983f90ce4e0044ade637fccc025ec Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 9 Dec 2022 13:43:10 -0500
Subject: [PATCH 492/555] remove diff markers

---
 apf/apfConstruct.cc | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index 45529a837..a87128bd0 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -202,13 +202,8 @@ static void constructRemotes(Mesh2* m, GlobalToVert& globalToVert)
   lion_eprint(1, "%d done inside remotes \n",PCU_Comm_Self());
 }
 
-<<<<<<< HEAD
-void assemble(Mesh2* m, const int* conn, int nelem, int etype,
+void assemble(Mesh2* m, const Gid* conn, int nelem, int etype,
      GlobalToVert& globalToVert)
-=======
-void construct(Mesh2* m, const Gid* conn, int nelem, int etype,
-    GlobalToVert& globalToVert)
->>>>>>> globalToVertCorruption
 {
   constructVerts(m, conn, nelem, etype, globalToVert);
   constructElements(m, conn, nelem, etype, globalToVert);
@@ -223,7 +218,7 @@ void finalise(Mesh2* m, GlobalToVert& globalToVert)
   m->acceptChanges();
 }
  
-void construct(Mesh2* m, const int* conn, int nelem, int etype,
+void construct(Mesh2* m, const Gid* conn, int nelem, int etype,
     GlobalToVert& globalToVert)
 {
   assemble(m, conn, nelem, etype, globalToVert);
@@ -330,9 +325,6 @@ void setCoords(Mesh2* m, const double* coords, int nverts,
   delete [] c;
 }
 
-<<<<<<< HEAD
-void destruct(Mesh2* m, int*& conn, int& nelem, int &etype, int cellDim)
-=======
 void setMatches(Mesh2* m, const Gid* matches, int nverts,
     GlobalToVert& globalToVert)
 {
@@ -539,8 +531,7 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts,
   m->destroyTag(matchGidTag);
 }
 
-void destruct(Mesh2* m, Gid*& conn, int& nelem, int &etype)
->>>>>>> globalToVertCorruption
+void destruct(Mesh2* m, Gid*& conn, int& nelem, int &etype, int cellDim)
 {
   if(cellDim == -1) cellDim = m->getDimension();
   //int dim = m->getDimension();

From 4d0c2a9b99e04c77d9fea257ba67737bc3f24e7a Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 9 Dec 2022 13:44:06 -0500
Subject: [PATCH 493/555] mner: c++ multi-topo reader... WIP

---
 test/matchedNodeElmReader.cc | 93 ++++++++++++++++++++++++++++++------
 1 file changed, 79 insertions(+), 14 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 88aad9d65..8d478d051 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -14,6 +14,12 @@
 #include <algorithm>
 #include <apfBox.h>
 
+#include <fstream>
+#include <sstream>
+#include <iostream>
+#include <string>
+#include <regex>
+
 /* from https://github.com/SCOREC/core/issues/205
 0=fully interior of the volume
 1-6 =classified on face (not edge or vertex)
@@ -652,16 +658,73 @@ void readMatches(FILE* f, apf::Gid numvtx, int localnumvtx, apf::Gid** matches)
 //  }
 }
 
-void readElements(FILE* f, FILE* fh, unsigned &dim,  apf::Gid& numElms,
+//static int starts_with(char const* with, char const* s) {
+//  int lw;
+//  int ls;
+//  lw = strlen(with);
+//  ls = strlen(s);
+//  if (ls < lw)
+//    return 0;
+//  return strncmp(with, s, lw) == 0;
+//}
+
+bool seekPart(std::ifstream& f, const std::string& marker) {
+  std::regex integer("^[[:digit:]]+$");
+  std::string line;
+  while (std::getline(f, line)) {
+    if (std::regex_match(line,integer)) {
+      std::cerr << "string object matched " << line << "\n";
+      if(line == marker) {
+        std::cerr << "found marker " << marker << "!\n";
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+void rewindStream(std::ifstream& f) {
+  f.clear();
+  f.seekg(0);
+}
+
+/**
+fh = header file
+- fh = header file (there is only one for all processes), containing:
+   1 1  #No idea why we write this
+   3    #also not sure what this is used for
+   <numelTotal>  <maxNodesPerElement>
+   Part0
+       <numel_topo_1>   <NodesInElementTopo1>
+       <nmel_topo_2>   < NodesInElementTopo2>
+       ... for as many topos as are in  Part 0
+   Repeat the above bock for each part.
+- f = part file, each part gets a file that contains a rectangular array, one for each topology
+  present on that part, that provides element to vertexGlobalId connectivity in
+  the order listed in the section of the header file for that part
+**/
+void readElements(std::ifstream& f, std::ifstream& fh, unsigned &dim,  apf::Gid& numElms,
     unsigned& numVtxPerElm, int& localNumElms, apf::Gid** elements) {
-  rewind(f);
-  rewind(fh);
+  const int self = PCU_Comm_Self();;
+  //silence warnings -----
+  (void)dim;
+  (void)localNumElms;
+  (void)elements;
+  (void)numVtxPerElm;
+  (void)numElms;
+  //silence warnings -----
+
+  rewindStream(fh);
+  //find my parts header block
+  bool ret = seekPart(fh, std::to_string(self));
+  assert(ret);
+  exit(EXIT_FAILURE);
+
+  rewindStream(f);
+  /*
   int dimHeader[2];
-  gmi_fscanf(fh, 2, "%u %u", dimHeader, dimHeader+1);
-//  assert( dimHeader[0] == 1 && dimHeader[1] == 1);
-  gmi_fscanf(fh, 1, "%u", &dim);
-  gmi_fscanf(fh, 2, "%ld %u", &numElms, &numVtxPerElm);
-  int self = PCU_Comm_Self();;
+  unsigned maxVtxPerElm;
+  gmi_fscanf(fh, 2, "%ld %u", &numElms, &maxVtxPerElm);
   for (int j=0; j< self+1;j++)
      gmi_fscanf(fh, 2, "%d %u", &localNumElms, &numVtxPerElm);
   *elements = new apf::Gid[localNumElms*numVtxPerElm];
@@ -679,6 +742,7 @@ void readElements(FILE* f, FILE* fh, unsigned &dim,  apf::Gid& numElms,
     elmIdx++;
   }
   delete [] elmVtx;
+  */
 }
 
 struct MeshInfo {
@@ -754,15 +818,16 @@ void readMesh(const char* meshfilename,
   }
 
   sprintf(filename, "%s.%d",meshfilename,self);
-  FILE* f = fopen(filename, "r");
-  FILE* fh = fopen(connHeadfilename, "r");
-  PCU_ALWAYS_ASSERT(f);
-  PCU_ALWAYS_ASSERT(fh);
+  std::ifstream f(filename, std::ios::in);
+  PCU_ALWAYS_ASSERT(f.is_open());
+  std::ifstream fh(connHeadfilename, std::ios::in);
+  PCU_ALWAYS_ASSERT(fh.is_open());
 //  now we went to do a readElements for each topology so 
   readElements(f,fh, mesh.dim, mesh.numElms, mesh.numVtxPerElm,
       mesh.localNumElms, &(mesh.elements));
   mesh.elementType = getElmType(mesh.dim, mesh.numVtxPerElm);
-  fclose(f);
+  fh.close();
+  f.close();
 }
 
 
@@ -805,7 +870,7 @@ int main(int argc, char** argv)
     fprintf(stderr, "isMatched %d\n", isMatched);
 
   //gmi_model* model = gmi_load(".null");
-  gmi_model* model = apf::makeMdsBox(2,2,2,1,1,1,0);
+  gmi_model* model = apf::makeMdsBoxModel(2,2,2,1,1,1,0);
   apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, m.dim, isMatched);
   apf::GlobalToVert outMap;
   PCU_Debug_Open();

From 5c78a0f2b82a65844a0c5707a3c7d259f1ae6fdf Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 9 Dec 2022 13:49:44 -0500
Subject: [PATCH 494/555] remove commented code

---
 mds/apfBox.cc | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/mds/apfBox.cc b/mds/apfBox.cc
index bf20a192e..b6dec78c0 100644
--- a/mds/apfBox.cc
+++ b/mds/apfBox.cc
@@ -86,7 +86,6 @@ BoxBuilder::BoxBuilder(int nx, int ny, int nz,
 void BoxBuilder::formModelTable()
 {
   int nd[4] = {0,0,0,0};
-//  int nds[4] = {31,11,1,0};
   for (int i = 0; i < mgrid.total(); ++i) {
     Indices mi = mgrid.out(i);
     int mdim = 0;
@@ -95,8 +94,6 @@ void BoxBuilder::formModelTable()
         ++mdim;
     modelTable[i].dim = mdim;
     modelTable[i].tag = nd[mdim]++;
-//    modelTable[i].tag = nds[mdim]++;
-//    nd[mdim]++;
   }
   for (int i = 0; i < 4; ++i)
     modelCounts[i] = nd[i];

From 8b60eb43b99e3dbeeb40401f7e7922bd1b8b8c55 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 9 Dec 2022 13:54:19 -0500
Subject: [PATCH 495/555] revert change

---
 ma/maSnap.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ma/maSnap.cc b/ma/maSnap.cc
index 72d4d9922..195dca005 100644
--- a/ma/maSnap.cc
+++ b/ma/maSnap.cc
@@ -60,7 +60,7 @@ static size_t isSurfUnderlyingFaceDegenerate(
     (range[periodicAxes][0] + range[periodicAxes][1]) / 2.0;
   for (int i = 0; i < 2; i++) {
     double candidateDegenParam = range[degenAxes][i];
-    double param[3] = {0,0,0};
+    double param[2];
     Vector uTan;
     Vector vTan;
     param[periodicAxes] = candidatePeriodicParam;

From ff14397dc2dab7d3356edb579546c7c4dd0d04b4 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 9 Dec 2022 13:57:05 -0500
Subject: [PATCH 496/555] remove commented code

---
 mds/mds_apf.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/mds/mds_apf.c b/mds/mds_apf.c
index 19166c070..262cd2542 100644
--- a/mds/mds_apf.c
+++ b/mds/mds_apf.c
@@ -269,8 +269,6 @@ static int align_copies(struct mds_net* net, struct mds* m)
   mds_id e;
   struct mds_copies* c;
   int did_change = 0;
-//  long countAll=0;
-//  long countDTC=0;
   for (d = 1; d < m->d; ++d){
     PCU_Comm_Begin();
     for (e = mds_begin(m, d); e != MDS_NONE; e = mds_next(m, e)) {

From 72495ac2bda4512fb7f6948763c946efd1b1db23 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 9 Dec 2022 13:58:32 -0500
Subject: [PATCH 497/555] remove commented code

---
 phasta/phGeomBC.cc | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/phasta/phGeomBC.cc b/phasta/phGeomBC.cc
index 84be1e226..e996b7818 100644
--- a/phasta/phGeomBC.cc
+++ b/phasta/phGeomBC.cc
@@ -17,16 +17,6 @@ static std::string buildGeomBCFileName(std::string timestep_or_dat)
   return ss.str();
 }
 
-/* abandoned 
-static std::string buildCoordsFileName()
-{
-  std::stringstream ss;
-  int rank = PCU_Comm_Self() + 1;
-  ss << "coords."  << "." << rank;
-  return ss.str();
-}
-*/
-
 enum {
   MAX_PARAMS = 12
 };

From a81efe41ec1fb9698686f9d39a99aa9790d607bf Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 6 Jan 2023 13:28:49 -0500
Subject: [PATCH 498/555] add ctest and pumi-meshes test case

---
 pumi-meshes        |  2 +-
 test/testing.cmake | 12 ++++++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/pumi-meshes b/pumi-meshes
index 635fff268..7733c3ee8 160000
--- a/pumi-meshes
+++ b/pumi-meshes
@@ -1 +1 @@
-Subproject commit 635fff268201e528628595595fd320ab5aa5b8e5
+Subproject commit 7733c3ee82cc589ef378816c1202322e264c98c1
diff --git a/test/testing.cmake b/test/testing.cmake
index 9dc958082..953e9cb6c 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -202,6 +202,18 @@ mpi_test(create_misSquare 1
   ${MESHES}/square/square.smb
   mis_test)
 
+set(MDIR ${MESHES}/matchedNodeElementReader)
+mpi_test(matchedNodeElementReader_p4 4
+  ./matchedNodeElmReader
+  "${MDIR}/geom3D.cnndt"
+  "${MDIR}/geom3D.coord"
+  "${MDIR}/geom3D.match"
+  "${MDIR}/geom3D.class"
+  "${MDIR}/geom3D.fathr"
+  "NULL"
+  "${MDIR}/geom3DHead.cnn"
+  "geom.dmg" "geom.smb")
+
 set(MDIR ${MESHES}/gmsh)
 mpi_test(gmshv2TwoQuads 1
   ./from_gmsh

From 32cbcecc5a6b2967bc4070c2df2fa4557c60db1f Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 6 Jan 2023 13:33:36 -0500
Subject: [PATCH 499/555] mixed topo reader WIP

64 chars for filename was overflowing... it should be replaced with stringstream
---
 test/matchedNodeElmReader.cc | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 8d478d051..150c97d24 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -718,9 +718,14 @@ void readElements(std::ifstream& f, std::ifstream& fh, unsigned &dim,  apf::Gid&
   //find my parts header block
   bool ret = seekPart(fh, std::to_string(self));
   assert(ret);
-  exit(EXIT_FAILURE);
 
   rewindStream(f);
+
+  typedef std::pair<unsigned,unsigned> TopoInfo;
+  exit(EXIT_FAILURE);
+
+  TopoInfo a;
+
   /*
   int dimHeader[2];
   unsigned maxVtxPerElm;
@@ -772,7 +777,7 @@ void readMesh(const char* meshfilename,
 
   int self = PCU_Comm_Self();
 
-  char filename[64];
+  char filename[1024];
   sprintf(filename, "%s.%d",coordfilename,self);
     
   FILE* fc = fopen(filename , "r");
@@ -817,17 +822,18 @@ void readMesh(const char* meshfilename,
     fclose(fm);
   }
 
-  sprintf(filename, "%s.%d",meshfilename,self);
-  std::ifstream f(filename, std::ios::in);
-  PCU_ALWAYS_ASSERT(f.is_open());
-  std::ifstream fh(connHeadfilename, std::ios::in);
-  PCU_ALWAYS_ASSERT(fh.is_open());
+  std::stringstream ss;
+  ss << meshfilename << "." << self;
+  std::ifstream meshConnStream(ss.str());
+  PCU_ALWAYS_ASSERT(meshConnStream.is_open());
+  std::ifstream connHeadStream(connHeadfilename, std::ios::in);
+  PCU_ALWAYS_ASSERT(connHeadStream.is_open());
 //  now we went to do a readElements for each topology so 
-  readElements(f,fh, mesh.dim, mesh.numElms, mesh.numVtxPerElm,
+  readElements(meshConnStream, connHeadStream, mesh.dim, mesh.numElms, mesh.numVtxPerElm,
       mesh.localNumElms, &(mesh.elements));
   mesh.elementType = getElmType(mesh.dim, mesh.numVtxPerElm);
-  fh.close();
-  f.close();
+  connHeadStream.close();
+  meshConnStream.close();
 }
 
 

From bfb7fd7840f0ad848ee90b65ccd049a316354b52 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 6 Jan 2023 14:26:23 -0500
Subject: [PATCH 500/555] part id regex works

---
 test/matchedNodeElmReader.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 150c97d24..47f1f8205 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -669,9 +669,11 @@ void readMatches(FILE* f, apf::Gid numvtx, int localnumvtx, apf::Gid** matches)
 //}
 
 bool seekPart(std::ifstream& f, const std::string& marker) {
-  std::regex integer("^[[:digit:]]+$");
+  std::cerr << "marker: " << marker << "\n";
+  std::regex integer("^\\s+[[:digit:]]+$");
   std::string line;
   while (std::getline(f, line)) {
+    std::cerr << "line: " << line << "\n";
     if (std::regex_match(line,integer)) {
       std::cerr << "string object matched " << line << "\n";
       if(line == marker) {

From 0eccb7487b1938f35417d3df5fa6ca85c49e9760 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 6 Jan 2023 16:43:44 -0500
Subject: [PATCH 501/555] find match with regex

---
 test/matchedNodeElmReader.cc | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 47f1f8205..77dd1fbc8 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -669,17 +669,15 @@ void readMatches(FILE* f, apf::Gid numvtx, int localnumvtx, apf::Gid** matches)
 //}
 
 bool seekPart(std::ifstream& f, const std::string& marker) {
-  std::cerr << "marker: " << marker << "\n";
-  std::regex integer("^\\s+[[:digit:]]+$");
+  std::stringstream ss;
+  ss << "^\\s+" << marker << "$";
+  std::regex partId(ss.str());
   std::string line;
   while (std::getline(f, line)) {
-    std::cerr << "line: " << line << "\n";
-    if (std::regex_match(line,integer)) {
+    if (std::regex_match(line,partId)) {
       std::cerr << "string object matched " << line << "\n";
-      if(line == marker) {
-        std::cerr << "found marker " << marker << "!\n";
-        return true;
-      }
+      std::cerr << "found marker " << marker << "!\n";
+      return true;
     }
   }
   return false;

From e0c261c0b99649ee4405b8ca3f6a6f72980b957e Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 13 Jan 2023 11:26:24 -0500
Subject: [PATCH 502/555] read the block info

---
 test/matchedNodeElmReader.cc | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 77dd1fbc8..04c4cfd31 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -683,6 +683,22 @@ bool seekPart(std::ifstream& f, const std::string& marker) {
   return false;
 }
 
+using BlockInfo = std::pair<long,int>;
+
+std::vector<BlockInfo> readTopoBlockInfo(std::ifstream& f) {
+  std::vector<BlockInfo> blocks;
+  long blockSize;
+  int vtxPerElement;
+
+  std::string line;
+  while (std::getline(f, line)) {
+    std::istringstream iss(line);
+    if (!(iss >> blockSize >> vtxPerElement)) { break; } // error
+    blocks.push_back(BlockInfo(blockSize,vtxPerElement));
+  }
+  return blocks;
+}
+
 void rewindStream(std::ifstream& f) {
   f.clear();
   f.seekg(0);
@@ -718,6 +734,12 @@ void readElements(std::ifstream& f, std::ifstream& fh, unsigned &dim,  apf::Gid&
   //find my parts header block
   bool ret = seekPart(fh, std::to_string(self));
   assert(ret);
+  auto blockInfo = readTopoBlockInfo(fh);
+  assert(ret);
+  for(auto b : blockInfo) {
+    std::cout << self << " " << b.first << " " << b.second << "\n";
+  }
+  PCU_Barrier();
 
   rewindStream(f);
 

From e9bbbb7e037685e1a9c5a84158dd3136371cf232 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 13 Jan 2023 14:38:33 -0500
Subject: [PATCH 503/555] assemble: use gids for connectivity

---
 apf/apfConvert.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apf/apfConvert.h b/apf/apfConvert.h
index 084cb5916..88c0effa1 100644
--- a/apf/apfConvert.h
+++ b/apf/apfConvert.h
@@ -39,7 +39,7 @@ typedef std::map<Gid, MeshEntity*> GlobalToVert;
   assemble and finalise. The premise of assemble being 
   that it is called multiple times for a given cell type,
   across several different cell types in the input mesh. */
-void assemble(Mesh2* m, const int* conn, int nelem, int etype,
+void assemble(Mesh2* m, const apf::Gid* conn, int nelem, int etype,
     GlobalToVert& globalToVert);
 
 /** \brief finalise construction of a mixed-cell-type mesh from just a connectivity array

From 26e2f32c611643294c49ecb4d514aa4b8bc5c2b4 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 13 Jan 2023 14:40:39 -0500
Subject: [PATCH 504/555] read element topo blocks, compiles, fails with FPE

---
 test/matchedNodeElmReader.cc | 107 ++++++++++++++---------------------
 1 file changed, 44 insertions(+), 63 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 04c4cfd31..90022a4b1 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -683,7 +683,10 @@ bool seekPart(std::ifstream& f, const std::string& marker) {
   return false;
 }
 
-using BlockInfo = std::pair<long,int>;
+struct BlockInfo {
+  long numElms;
+  int vtxPerElm;
+};
 
 std::vector<BlockInfo> readTopoBlockInfo(std::ifstream& f) {
   std::vector<BlockInfo> blocks;
@@ -694,7 +697,7 @@ std::vector<BlockInfo> readTopoBlockInfo(std::ifstream& f) {
   while (std::getline(f, line)) {
     std::istringstream iss(line);
     if (!(iss >> blockSize >> vtxPerElement)) { break; } // error
-    blocks.push_back(BlockInfo(blockSize,vtxPerElement));
+    blocks.push_back({blockSize,vtxPerElement});
   }
   return blocks;
 }
@@ -705,87 +708,62 @@ void rewindStream(std::ifstream& f) {
 }
 
 /**
-fh = header file
-- fh = header file (there is only one for all processes), containing:
-   1 1  #No idea why we write this
-   3    #also not sure what this is used for
-   <numelTotal>  <maxNodesPerElement>
+fh = header file (there is only one for all processes), containing:
    Part0
        <numel_topo_1>   <NodesInElementTopo1>
        <nmel_topo_2>   < NodesInElementTopo2>
        ... for as many topos as are in  Part 0
    Repeat the above bock for each part.
-- f = part file, each part gets a file that contains a rectangular array, one for each topology
-  present on that part, that provides element to vertexGlobalId connectivity in
-  the order listed in the section of the header file for that part
 **/
-void readElements(std::ifstream& f, std::ifstream& fh, unsigned &dim,  apf::Gid& numElms,
-    unsigned& numVtxPerElm, int& localNumElms, apf::Gid** elements) {
-  const int self = PCU_Comm_Self();;
-  //silence warnings -----
-  (void)dim;
-  (void)localNumElms;
-  (void)elements;
-  (void)numVtxPerElm;
-  (void)numElms;
-  //silence warnings -----
-
+std::vector<BlockInfo> readHeader(std::ifstream& fh) {
   rewindStream(fh);
-  //find my parts header block
+  const int self = PCU_Comm_Self();;
   bool ret = seekPart(fh, std::to_string(self));
   assert(ret);
   auto blockInfo = readTopoBlockInfo(fh);
-  assert(ret);
+  assert(blockInfo.size()>0);
   for(auto b : blockInfo) {
-    std::cout << self << " " << b.first << " " << b.second << "\n";
+    std::cout << self << " " << b.numElms << " " << b.vtxPerElm << "\n";
   }
-  PCU_Barrier();
+  return blockInfo;
+}
 
+/**
+- f = part file, each part gets a file that contains a rectangular array, one for each topology
+  present on that part, that provides element to vertexGlobalId connectivity in
+  the order listed in the section of the header file for that part
+**/
+void readElements(std::ifstream& f, apf::Gid numElms,
+    unsigned numVtxPerElm, apf::Gid* elements) {
   rewindStream(f);
-
-  typedef std::pair<unsigned,unsigned> TopoInfo;
-  exit(EXIT_FAILURE);
-
-  TopoInfo a;
-
-  /*
-  int dimHeader[2];
-  unsigned maxVtxPerElm;
-  gmi_fscanf(fh, 2, "%ld %u", &numElms, &maxVtxPerElm);
-  for (int j=0; j< self+1;j++)
-     gmi_fscanf(fh, 2, "%d %u", &localNumElms, &numVtxPerElm);
-  *elements = new apf::Gid[localNumElms*numVtxPerElm];
-  int i;
-  unsigned j;
   unsigned elmIdx = 0;
   apf::Gid* elmVtx = new apf::Gid[numVtxPerElm];
-  for (i = 0; i < localNumElms; i++) {
-    for (j = 0; j < numVtxPerElm; j++)
-      gmi_fscanf(f, 1, "%ld", elmVtx+j);
-    for (j = 0; j < numVtxPerElm; j++) {
+  for (int i = 0; i < numElms; i++) {
+    for (unsigned j = 0; j < numVtxPerElm; j++)
+      f >> elmVtx[j];
+    for (unsigned j = 0; j < numVtxPerElm; j++) {
       const unsigned elmVtxIdx = elmIdx*numVtxPerElm+j;
-      (*elements)[elmVtxIdx] = --(elmVtx[j]); //export from matlab using 1-based indices
+      elements[elmVtxIdx] = --(elmVtx[j]); //export from matlab using 1-based indices
     }
     elmIdx++;
   }
   delete [] elmVtx;
-  */
 }
 
 struct MeshInfo {
   double* coords;
   double* solution;
-  apf::Gid* elements; //TODO store per block
+  std::vector<apf::Gid*> elements;
   apf::Gid* matches;
   int* classification;
   int* fathers2D;
   unsigned dim;
-  unsigned elementType; //TODO remove
+  std::vector<unsigned> elementType;
   apf::Gid numVerts;
   int localNumVerts;
-  apf::Gid numElms; //TODO does not appear to be used
-  int localNumElms; //TODO store per block 
-  unsigned numVtxPerElm; //TODO can be a pumi query?
+  std::vector<apf::Gid> numElms; //TODO does not appear to be used
+  std::vector<int> localNumElms; //TODO store per block 
+  std::vector<unsigned> numVtxPerElm; //TODO can be a pumi query?
 };
 
 void readMesh(const char* meshfilename,
@@ -797,6 +775,8 @@ void readMesh(const char* meshfilename,
     const char* connHeadfilename,
     MeshInfo& mesh) {
 
+  mesh.dim = 3; //FIXME
+
   int self = PCU_Comm_Self();
 
   char filename[1024];
@@ -850,11 +830,15 @@ void readMesh(const char* meshfilename,
   PCU_ALWAYS_ASSERT(meshConnStream.is_open());
   std::ifstream connHeadStream(connHeadfilename, std::ios::in);
   PCU_ALWAYS_ASSERT(connHeadStream.is_open());
-//  now we went to do a readElements for each topology so 
-  readElements(meshConnStream, connHeadStream, mesh.dim, mesh.numElms, mesh.numVtxPerElm,
-      mesh.localNumElms, &(mesh.elements));
-  mesh.elementType = getElmType(mesh.dim, mesh.numVtxPerElm);
+  auto blockInfo = readHeader(connHeadStream);
   connHeadStream.close();
+  for(auto b : blockInfo) {
+    mesh.numElms.push_back(b.numElms);
+    mesh.numVtxPerElm.push_back(b.vtxPerElm);
+    apf::Gid* elements = new apf::Gid[b.numElms*b.vtxPerElm];
+    readElements(meshConnStream, b.numElms, b.vtxPerElm, elements);
+    mesh.elementType.push_back(getElmType(mesh.dim, b.vtxPerElm));
+  }
   meshConnStream.close();
 }
 
@@ -902,16 +886,13 @@ int main(int argc, char** argv)
   apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, m.dim, isMatched);
   apf::GlobalToVert outMap;
   PCU_Debug_Open();
-  apf::construct(mesh, m.elements, m.localNumElms, m.elementType, outMap);
-// PLANNING: before we can call what used to be construct but is now assemble and finalise we will need to batch elements by topology
-// in the first pass we will keep the upstream reader that  has these as rectangular arrays and do a sort here into one 
-// group o
-  delete [] m.elements;
+  for( size_t i=0; i< m.elements.size(); i++) {
+    apf::assemble(mesh, m.elements[i], m.numElms[i], m.elementType[i], outMap);
+    delete [] m.elements[i];
+  }
+  apf::finalise(mesh, outMap);
   apf::alignMdsRemotes(mesh);
   apf::deriveMdsModel(mesh);
-  /*for (int i=0; i<81; i++) {
-  std::cout<<m.coords[i]<<std::endl;
-  }*/
   apf::setCoords(mesh, m.coords, m.localNumVerts, outMap);
   delete [] m.coords;
   if( isMatched ) {

From b46f2c88ea7fc52bfd1a950d3804ea307c27d70e Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 13 Jan 2023 15:07:50 -0500
Subject: [PATCH 505/555] remove debug print

---
 test/matchedNodeElmReader.cc | 2 --
 1 file changed, 2 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 90022a4b1..e803419aa 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -675,8 +675,6 @@ bool seekPart(std::ifstream& f, const std::string& marker) {
   std::string line;
   while (std::getline(f, line)) {
     if (std::regex_match(line,partId)) {
-      std::cerr << "string object matched " << line << "\n";
-      std::cerr << "found marker " << marker << "!\n";
       return true;
     }
   }

From cacacacec80ebc037ca63cc6893dd9f5c13c94c1 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 20 Jan 2023 10:35:58 -0500
Subject: [PATCH 506/555] forgot to push back elm array

---
 test/matchedNodeElmReader.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index e803419aa..762e907a3 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -836,6 +836,7 @@ void readMesh(const char* meshfilename,
     apf::Gid* elements = new apf::Gid[b.numElms*b.vtxPerElm];
     readElements(meshConnStream, b.numElms, b.vtxPerElm, elements);
     mesh.elementType.push_back(getElmType(mesh.dim, b.vtxPerElm));
+    mesh.elements.push_back(elements);
   }
   meshConnStream.close();
 }

From 4d24dc05d7b31078abedc28996eb9566479197c3 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 20 Jan 2023 10:50:21 -0500
Subject: [PATCH 507/555] move vtk before smb

---
 test/matchedNodeElmReader.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 762e907a3..6ee18364b 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -941,9 +941,9 @@ int main(int argc, char** argv)
 
   outMap.clear();
   gmi_write_dmg(model, argv[8]);
+  apf::writeVtkFiles("rendered",mesh);
   mesh->writeNative(argv[9]);
   if(noVerify != 0) mesh->verify();
-  apf::writeVtkFiles("rendered",mesh);
 
   mesh->destroyNative();
   apf::destroyMesh(mesh);

From a5227c7d9466f4eee540b5ca4a6141823389d01a Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 20 Jan 2023 15:18:19 -0500
Subject: [PATCH 508/555] remove unused vars

---
 test/matchedNodeElmReader.cc | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 6ee18364b..bd9a848dc 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -759,9 +759,8 @@ struct MeshInfo {
   std::vector<unsigned> elementType;
   apf::Gid numVerts;
   int localNumVerts;
-  std::vector<apf::Gid> numElms; //TODO does not appear to be used
-  std::vector<int> localNumElms; //TODO store per block 
-  std::vector<unsigned> numVtxPerElm; //TODO can be a pumi query?
+  std::vector<apf::Gid> numElms;
+  std::vector<unsigned> numVtxPerElm;
 };
 
 void readMesh(const char* meshfilename,

From a6b54ac2da80a7a3d511e47496dc0da08c506f13 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 20 Jan 2023 15:36:39 -0500
Subject: [PATCH 509/555] assert elm read

---
 test/matchedNodeElmReader.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index bd9a848dc..9128d4c50 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -745,6 +745,7 @@ void readElements(std::ifstream& f, apf::Gid numElms,
     }
     elmIdx++;
   }
+  PCU_ALWAYS_ASSERT(numElms==elmIdx);
   delete [] elmVtx;
 }
 

From 303c2fab8828535acde947cde81df35652bd3131 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 20 Jan 2023 16:07:51 -0500
Subject: [PATCH 510/555] check the elm type

---
 apf/apfConstruct.cc | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index a87128bd0..92e58ebe4 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -32,7 +32,6 @@ static void constructElements(
 {
   ModelEntity* interior = m->findModelEntity(m->getDimension(), 0);
   int nev = apf::Mesh::adjacentCount[etype][0];
-  unsigned etypeL=1;
   for (int i = 0; i < nelem; ++i) {
     Downward verts;
     int offset = i * nev;
@@ -49,10 +48,12 @@ static void constructElements(
       } 
     }
 
+    unsigned etypeL;
     if(uniqueVerts==4) etypeL=apf::Mesh::TET;
-    if(uniqueVerts==5) etypeL=apf::Mesh::PYRAMID;
-    if(uniqueVerts==6) etypeL=apf::Mesh::PRISM;
-    if(uniqueVerts==8) etypeL=apf::Mesh::HEX;
+    else if(uniqueVerts==5) etypeL=apf::Mesh::PYRAMID;
+    else if(uniqueVerts==6) etypeL=apf::Mesh::PRISM;
+    else if(uniqueVerts==8) etypeL=apf::Mesh::HEX;
+    else PCU_ALWAYS_ASSERT(false);
     buildElement(m, interior, etypeL, verts);
   }
 }

From d7256ae6f367fd4ca9d48a057bd4f695c6a76007 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Mon, 23 Jan 2023 12:19:53 -0500
Subject: [PATCH 511/555] fix noverify flag

---
 test/matchedNodeElmReader.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 9128d4c50..9997323f1 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -943,7 +943,7 @@ int main(int argc, char** argv)
   gmi_write_dmg(model, argv[8]);
   apf::writeVtkFiles("rendered",mesh);
   mesh->writeNative(argv[9]);
-  if(noVerify != 0) mesh->verify();
+  if(noVerify != 1) mesh->verify();
 
   mesh->destroyNative();
   apf::destroyMesh(mesh);

From be63271867accd5efdeef9f38b13ec82697d28fe Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Mon, 23 Jan 2023 12:21:45 -0500
Subject: [PATCH 512/555] pass matched flag to makeMdsBox

removed makeMdsBoxModel and its use.  An extra empty mesh was being created in mner.
---
 mds/apfBox.cc                | 18 +++++-------------
 mds/apfBox.h                 | 10 +++-------
 pumi-meshes                  |  2 +-
 test/matchedNodeElmReader.cc |  5 ++---
 4 files changed, 11 insertions(+), 24 deletions(-)

diff --git a/mds/apfBox.cc b/mds/apfBox.cc
index b6dec78c0..53cc40abc 100644
--- a/mds/apfBox.cc
+++ b/mds/apfBox.cc
@@ -67,7 +67,7 @@ int Grid::in(Indices is)
 
 BoxBuilder::BoxBuilder(int nx, int ny, int nz,
       double wx, double wy, double wz,
-      bool is):
+      bool is, bool matched):
   grid(nx + 1, ny + 1, nz + 1),
   mgrid(nx ? 3 : 1, ny ? 3 : 1, nz ? 3 : 1)
 {
@@ -78,7 +78,7 @@ BoxBuilder::BoxBuilder(int nx, int ny, int nz,
   is_simplex = is;
   formModelTable();
   gmi_model* gm = buildModel();
-  m = makeEmptyMdsMesh(gm, dim, false);
+  m = makeEmptyMdsMesh(gm, dim, matched);
   v.resize(grid.total());
   buildMeshAndModel();
 }
@@ -329,19 +329,11 @@ void BoxBuilder::buildMeshAndModel()
 
 Mesh2* makeMdsBox(
     int nex, int ney, int nez,
-    double wx, double wy, double wz, bool is)
+    double wx, double wy, double wz, bool is,
+    bool matched)
 {
-  BoxBuilder bb(nex, ney, nez, wx, wy, wz, is);
+  BoxBuilder bb(nex, ney, nez, wx, wy, wz, is, matched);
   return bb.m;
 }
 
-gmi_model* makeMdsBoxModel(
-    int nex, int ney, int nez,
-    double wx, double wy, double wz, bool is)
-{
-  BoxBuilder bb(nex, ney, nez, wx, wy, wz, is);
-  return bb.buildModel();
-}
-
-
 }
diff --git a/mds/apfBox.h b/mds/apfBox.h
index 035a29b78..b00e9b738 100644
--- a/mds/apfBox.h
+++ b/mds/apfBox.h
@@ -39,7 +39,7 @@ struct BoxBuilder
   std::vector<MeshEntity*> v;
   BoxBuilder(int nx, int ny, int nz,
       double wx, double wy, double wz,
-      bool is);
+      bool is, bool matched);
   void formModelTable();
   void addModelUse(gmi_base* gb, agm_bdry ab, Indices di);
   gmi_model* buildModel();
@@ -69,14 +69,10 @@ struct BoxBuilder
   \param wy y dimension width
   \param wz z dimension width
   \param is true = simplical mesh, false = quad/hex
+  \param matched true = matched mesh, false = otherwise
   \details set ny,nz=0 for a 1D mesh, set nz=0 for a 2D mesh */
 Mesh2* makeMdsBox(
-    int nx, int ny, int nz, double wx, double wy, double wz, bool is);
-
-/** \brief see makeMdsBox - only creates geometric model */
-gmi_model* makeMdsBoxModel(
-    int nx, int ny, int nz, double wx, double wy, double wz, bool is);
-
+    int nx, int ny, int nz, double wx, double wy, double wz, bool is, bool matched=false);
 
 }
 
diff --git a/pumi-meshes b/pumi-meshes
index 7733c3ee8..d18d87b67 160000
--- a/pumi-meshes
+++ b/pumi-meshes
@@ -1 +1 @@
-Subproject commit 7733c3ee82cc589ef378816c1202322e264c98c1
+Subproject commit d18d87b67df5339ab93c158569d03431ac75dc6e
diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 9997323f1..2b4edfe6f 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -880,9 +880,8 @@ int main(int argc, char** argv)
   if(!PCU_Comm_Self())
     fprintf(stderr, "isMatched %d\n", isMatched);
 
-  //gmi_model* model = gmi_load(".null");
-  gmi_model* model = apf::makeMdsBoxModel(2,2,2,1,1,1,0);
-  apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, m.dim, isMatched);
+  apf::Mesh2* mesh = apf::makeMdsBox(2,2,2,1,1,1,0,isMatched);
+  gmi_model* model = mesh->getModel();
   apf::GlobalToVert outMap;
   PCU_Debug_Open();
   for( size_t i=0; i< m.elements.size(); i++) {

From 9922fba24414b4c54313ebb9ac68dac44d7e3b60 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Mon, 23 Jan 2023 14:27:38 -0500
Subject: [PATCH 513/555] debug missing tags

---
 test/matchedNodeElmReader.cc | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 2b4edfe6f..41c02f416 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -900,6 +900,17 @@ int main(int argc, char** argv)
   }
   apf::MeshTag* tc = setMappedTag(mesh, "classification", m.classification, 1,
       m.localNumVerts, outMap);
+  { //debug - some verts don't have the classification tag...
+    apf::MeshEntity* v;
+    apf::MeshIterator* verts = mesh->begin(0);
+    int i=0;
+    while ((v = mesh->iterate(verts))) {
+      if(!mesh->hasTag(v,tc)) {
+        PCU_Debug_Print("%d missing tag\n", i);
+      }
+      i++;
+    }
+  }
   setClassification(model,mesh,tc);
   apf::removeTagFromDimension(mesh, tc, 0);
   mesh->destroyTag(tc);

From fbd7c3cb6bdb498b03fea2cc1fc4c05a90dceffa Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Tue, 24 Jan 2023 11:47:13 -0500
Subject: [PATCH 514/555] fix rewind bug for multitopo

---
 test/matchedNodeElmReader.cc | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 41c02f416..c07fd6b87 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -732,8 +732,8 @@ std::vector<BlockInfo> readHeader(std::ifstream& fh) {
   the order listed in the section of the header file for that part
 **/
 void readElements(std::ifstream& f, apf::Gid numElms,
-    unsigned numVtxPerElm, apf::Gid* elements) {
-  rewindStream(f);
+    unsigned numVtxPerElm, apf::Gid* elements, bool rewind) {
+  if(rewind) rewindStream(f);
   unsigned elmIdx = 0;
   apf::Gid* elmVtx = new apf::Gid[numVtxPerElm];
   for (int i = 0; i < numElms; i++) {
@@ -830,11 +830,13 @@ void readMesh(const char* meshfilename,
   PCU_ALWAYS_ASSERT(connHeadStream.is_open());
   auto blockInfo = readHeader(connHeadStream);
   connHeadStream.close();
+  bool rewind = true;
   for(auto b : blockInfo) {
     mesh.numElms.push_back(b.numElms);
     mesh.numVtxPerElm.push_back(b.vtxPerElm);
     apf::Gid* elements = new apf::Gid[b.numElms*b.vtxPerElm];
-    readElements(meshConnStream, b.numElms, b.vtxPerElm, elements);
+    readElements(meshConnStream, b.numElms, b.vtxPerElm, elements,rewind);
+    rewind=false;
     mesh.elementType.push_back(getElmType(mesh.dim, b.vtxPerElm));
     mesh.elements.push_back(elements);
   }

From 6c9b56d3e88df0060f5091fea47f8213a0e56444 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Tue, 24 Jan 2023 11:48:10 -0500
Subject: [PATCH 515/555] Revert "pass matched flag to makeMdsBox"

This reverts commit be63271867accd5efdeef9f38b13ec82697d28fe.

Fails in verify for the serial test case with a quad on the leading edge
of the airfoil:

APF FAILED: apf::Verify: quad with 1 adjacent regions
centroid: (0.534062, 4.98461e-05, 0.0833333)
based on the following:
 - quad is classified on a model region
we would expect the adjacent region count to be exactly 2
---
 mds/apfBox.cc                | 18 +++++++++++++-----
 mds/apfBox.h                 | 10 +++++++---
 pumi-meshes                  |  2 +-
 test/matchedNodeElmReader.cc |  5 +++--
 4 files changed, 24 insertions(+), 11 deletions(-)

diff --git a/mds/apfBox.cc b/mds/apfBox.cc
index 53cc40abc..b6dec78c0 100644
--- a/mds/apfBox.cc
+++ b/mds/apfBox.cc
@@ -67,7 +67,7 @@ int Grid::in(Indices is)
 
 BoxBuilder::BoxBuilder(int nx, int ny, int nz,
       double wx, double wy, double wz,
-      bool is, bool matched):
+      bool is):
   grid(nx + 1, ny + 1, nz + 1),
   mgrid(nx ? 3 : 1, ny ? 3 : 1, nz ? 3 : 1)
 {
@@ -78,7 +78,7 @@ BoxBuilder::BoxBuilder(int nx, int ny, int nz,
   is_simplex = is;
   formModelTable();
   gmi_model* gm = buildModel();
-  m = makeEmptyMdsMesh(gm, dim, matched);
+  m = makeEmptyMdsMesh(gm, dim, false);
   v.resize(grid.total());
   buildMeshAndModel();
 }
@@ -329,11 +329,19 @@ void BoxBuilder::buildMeshAndModel()
 
 Mesh2* makeMdsBox(
     int nex, int ney, int nez,
-    double wx, double wy, double wz, bool is,
-    bool matched)
+    double wx, double wy, double wz, bool is)
 {
-  BoxBuilder bb(nex, ney, nez, wx, wy, wz, is, matched);
+  BoxBuilder bb(nex, ney, nez, wx, wy, wz, is);
   return bb.m;
 }
 
+gmi_model* makeMdsBoxModel(
+    int nex, int ney, int nez,
+    double wx, double wy, double wz, bool is)
+{
+  BoxBuilder bb(nex, ney, nez, wx, wy, wz, is);
+  return bb.buildModel();
+}
+
+
 }
diff --git a/mds/apfBox.h b/mds/apfBox.h
index b00e9b738..035a29b78 100644
--- a/mds/apfBox.h
+++ b/mds/apfBox.h
@@ -39,7 +39,7 @@ struct BoxBuilder
   std::vector<MeshEntity*> v;
   BoxBuilder(int nx, int ny, int nz,
       double wx, double wy, double wz,
-      bool is, bool matched);
+      bool is);
   void formModelTable();
   void addModelUse(gmi_base* gb, agm_bdry ab, Indices di);
   gmi_model* buildModel();
@@ -69,10 +69,14 @@ struct BoxBuilder
   \param wy y dimension width
   \param wz z dimension width
   \param is true = simplical mesh, false = quad/hex
-  \param matched true = matched mesh, false = otherwise
   \details set ny,nz=0 for a 1D mesh, set nz=0 for a 2D mesh */
 Mesh2* makeMdsBox(
-    int nx, int ny, int nz, double wx, double wy, double wz, bool is, bool matched=false);
+    int nx, int ny, int nz, double wx, double wy, double wz, bool is);
+
+/** \brief see makeMdsBox - only creates geometric model */
+gmi_model* makeMdsBoxModel(
+    int nx, int ny, int nz, double wx, double wy, double wz, bool is);
+
 
 }
 
diff --git a/pumi-meshes b/pumi-meshes
index d18d87b67..7733c3ee8 160000
--- a/pumi-meshes
+++ b/pumi-meshes
@@ -1 +1 @@
-Subproject commit d18d87b67df5339ab93c158569d03431ac75dc6e
+Subproject commit 7733c3ee82cc589ef378816c1202322e264c98c1
diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index c07fd6b87..8caba2c25 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -882,8 +882,9 @@ int main(int argc, char** argv)
   if(!PCU_Comm_Self())
     fprintf(stderr, "isMatched %d\n", isMatched);
 
-  apf::Mesh2* mesh = apf::makeMdsBox(2,2,2,1,1,1,0,isMatched);
-  gmi_model* model = mesh->getModel();
+  //gmi_model* model = gmi_load(".null");
+  gmi_model* model = apf::makeMdsBoxModel(2,2,2,1,1,1,0);
+  apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, m.dim, isMatched);
   apf::GlobalToVert outMap;
   PCU_Debug_Open();
   for( size_t i=0; i< m.elements.size(); i++) {

From 7cbbd0c80ecf146dfe4c801f50aaccf5338d99de Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Tue, 24 Jan 2023 16:10:59 -0700
Subject: [PATCH 516/555] merged in ExtrusionTagsOffDev branch work in
 matchedNodeElmReader.cc as it reflects better promotion of vertex
 classification to edge,face, and region classification.

---
 pumi-meshes                  |   2 +-
 test/matchedNodeElmReader.cc | 560 +++++++++++++----------------------
 2 files changed, 204 insertions(+), 358 deletions(-)

diff --git a/pumi-meshes b/pumi-meshes
index 7733c3ee8..635fff268 160000
--- a/pumi-meshes
+++ b/pumi-meshes
@@ -1 +1 @@
-Subproject commit 7733c3ee82cc589ef378816c1202322e264c98c1
+Subproject commit 635fff268201e528628595595fd320ab5aa5b8e5
diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 8caba2c25..5b0eaf01e 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -37,7 +37,8 @@
 #define VERTEX_LAST 38
 
 /* model entity ids */
-#define INTERIOR_REGION 0
+//#define INTERIOR_REGION 0
+int INTERIOR_REGION=0; // initialized but will be checked from read input
 
 apf::ModelEntity* getMdlRgn(gmi_model* model) {
   apf::ModelEntity* rgn = reinterpret_cast<apf::ModelEntity*>(
@@ -80,69 +81,23 @@ void setVtxClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtxC
     //std::cout<<"Returned tag is c= "<<c<<std::endl;
     //std::cout<<" "<<std::endl;
     //count++;
-    if (c == INTERIORTAG) {
+//
+// New approach:  mesher will add 0 to mdlConvert's dmg tag number for vertices,
+//                          1000000 to mdlConvert's dmg tag number for edges,
+//                          2000000 to mdlConvert's dmg tag number for faces,
+//                          3000000 to mdlConvert's dmg tag number for regions
+    if (c >= 3000000 ) {
+       INTERIOR_REGION=c-3000000;
        mesh->setModelEntity(v,getMdlRgn(model));
        //cint++;
-    } else if (c >= FACE && c <= FACE_LAST) {
-       if (c == 1) { //face tag 1 corresponds to model face 0
-          mesh->setModelEntity(v,getMdlFace(mesh,0));
-       } else if (c == 2) { //face tag 2 corresponds to model face 1
-          mesh->setModelEntity(v,getMdlFace(mesh,1));
-       } else if (c == 3) { //face tag 3 corresponds to model face 3
-          mesh->setModelEntity(v,getMdlFace(mesh,3));
-       } else if (c == 4) { //face tag 4 corresponds to model face 4
-          mesh->setModelEntity(v,getMdlFace(mesh,4));
-       } else if (c == 5) { //face tag 5 corresponds to model face 2
-          mesh->setModelEntity(v,getMdlFace(mesh,2));
-       } else if (c == 6) { //face tag 6 corresponds to model face 5
-          mesh->setModelEntity(v,getMdlFace(mesh,5));
-       }
+    } else if (c >= 2000000) {
+       mesh->setModelEntity(v,getMdlFace(mesh,c-2000000));
        //cface++;
-    } else if (c >= EDGE && c <= EDGE_LAST) {
-       if (c == 11) { //edge tag 11 corresponds to model edge 0
-          mesh->setModelEntity(v,getMdlEdge(mesh,0));
-       } else if (c == 12) {
-          mesh->setModelEntity(v,getMdlEdge(mesh,2));
-       } else if (c == 13) {
-          mesh->setModelEntity(v,getMdlEdge(mesh,3));
-       } else if (c == 14) {
-          mesh->setModelEntity(v,getMdlEdge(mesh,1));
-       } else if (c == 15) {
-          mesh->setModelEntity(v,getMdlEdge(mesh,4));
-       } else if (c == 16) {
-          mesh->setModelEntity(v,getMdlEdge(mesh,5));
-       } else if (c == 17) {
-          mesh->setModelEntity(v,getMdlEdge(mesh,7));
-       } else if (c == 18) {
-          mesh->setModelEntity(v,getMdlEdge(mesh,6));
-       } else if (c == 19) {
-          mesh->setModelEntity(v,getMdlEdge(mesh,8));
-       } else if (c == 20) {
-          mesh->setModelEntity(v,getMdlEdge(mesh,10));
-       } else if (c == 21) {
-          mesh->setModelEntity(v,getMdlEdge(mesh,11));
-       } else if (c == 22) {
-          mesh->setModelEntity(v,getMdlEdge(mesh,9));
-       }
+    } else if (c >= 1000000) {
+       mesh->setModelEntity(v,getMdlEdge(mesh,c-1000000));
        //cedge++;
-    } else if (c >= VERTEX && c <= VERTEX_LAST) {
-       if (c == 31) { //vertex tag 31 corresponds to model vertex 0
-          mesh->setModelEntity(v,getMdlVtx(mesh,0));
-       } else if (c == 32) { //vertex tag 32 corresponds to model vertex 1
-          mesh->setModelEntity(v,getMdlVtx(mesh,1));
-       } else if (c == 33) { //vertex tag 33 corresponds to model vertex 3
-          mesh->setModelEntity(v,getMdlVtx(mesh,3));
-       } else if (c == 34) { //vertex tag 34 corresponds to model vertex 2
-          mesh->setModelEntity(v,getMdlVtx(mesh,2));
-       } else if (c == 35) { //vertex tag 35 corresponds to model vertex 4
-          mesh->setModelEntity(v,getMdlVtx(mesh,4));
-       } else if (c == 36) { //vertex tag 36 corresponds to model vertex 5
-          mesh->setModelEntity(v,getMdlVtx(mesh,5));
-       } else if (c == 37) { //vertex tag 37 corresponds to model vertex 7
-          mesh->setModelEntity(v,getMdlVtx(mesh,7));
-       } else if (c == 38) { //vertex tag 38 corresponds to model vertex 6
-          mesh->setModelEntity(v,getMdlVtx(mesh,6));
-       }
+    } else {
+       mesh->setModelEntity(v,getMdlVtx(mesh,c));
        //cvtx++;
     }
   }
@@ -160,82 +115,112 @@ void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh,apf::MeshTag* vtxC
   int c;
   //int count=0;
   apf::Adjacent verts;
+  int k,ff;
+  double distFromDebug1;
+//  apf::Vector3 xd1(-0.54864, 7.44015e-06, 0.0397148 );
+  apf::Vector3 xd1(-2.63644, -0.953687, 0.00762);
+  apf::Vector3 xd2(0.914478, 0.0145401, 0.04 );
+  apf::Vector3 dx1;
+  apf::Vector3 dx2;
+  apf::Vector3 tmp;
+  apf::Vector3 Centroid;
   while( (e = mesh->iterate(it)) ) {
     //std::cout<<"Edge number "<<count++<<" with nodes"<<std::endl;
     mesh->getAdjacent(e, 0, verts);
-    int cmin=100;
-    for(size_t i=0; i<verts.size(); i++) {
+
+    int nverts = verts.size();
+    if(1==0) {
+    Centroid=apf::getLinearCentroid(mesh,e);
+    dx1=xd1-Centroid;
+ //   dx2=xd2-Centroid;
+    distFromDebug1=dx1[0]*dx1[0]
+                  +dx1[1]*dx1[1]
+                  +dx1[2]*dx1[2];
+/*    distFromDebug2=dx2[0]*dx2[0]
+                  +dx2[1]*dx2[1]
+                  +dx2[2]*dx2[2];
+*/
+    }
+    int cmax=-100;
+    int cmin=100000000;
+    for(int i=0; i<(int)verts.size(); i++) {
       mesh->getIntTag(verts[i],vtxClass,&c);
-      //mesh->getPoint(verts[i], 0, vCoord);
-      //std::cout<<vCoord[0]<<" "<<vCoord[1]<<" "<<vCoord[2]<<std::endl;
+      cmax=std::max(cmax,c);
       cmin=std::min(cmin,c);
     }
-    //std::cout<<"has classification "<<cmin<<std::endl;
-    //std::cout<<" "<<std::endl;
-    if (cmin == INTERIORTAG) {
+    if(1==0 && distFromDebug1 < 1e-3) {
+         fprintf(stderr, "%d %d %.15e %.15E %.15E \n", cmin, cmax, Centroid[0], Centroid[1], Centroid[2]);
+         for (int i=0; i < nverts; i++) {
+            mesh->getPoint(verts[i],0,tmp);
+//            fprintf(stderr, "%d %.15e %.15E %.15E \n", i , tmp[0], tmp[1], tmp[2]);
+         }
+    }
+    int dimMin=cmin/1000000;
+    int dimMax=cmax/1000000;
+    int tagMin=cmin-dimMin*1000000;
+    int tagMax=cmax-dimMax*1000000;
+    if (cmax >= 3000000) {
+       INTERIOR_REGION=cmax-3000000;
        mesh->setModelEntity(e,getMdlRgn(model));
        //cint++;
-    } else if (cmin >= FACE && cmin <= FACE_LAST) {
-       if (cmin == 1) { //face tag 1 corresponds to model face 0
-          mesh->setModelEntity(e,getMdlFace(mesh,0));
-       } else if (cmin == 2) { //face tag 2 corresponds to model face 1
-          mesh->setModelEntity(e,getMdlFace(mesh,1));
-       } else if (cmin == 3) { //face tag 3 corresponds to model face 3
-          mesh->setModelEntity(e,getMdlFace(mesh,3));
-       } else if (cmin == 4) { //face tag 4 corresponds to model face 4
-          mesh->setModelEntity(e,getMdlFace(mesh,4));
-       } else if (cmin == 5) { //face tag 5 corresponds to model face 2
-          mesh->setModelEntity(e,getMdlFace(mesh,2));
-       } else if (cmin == 6) { //face tag 6 corresponds to model face 5
-          mesh->setModelEntity(e,getMdlFace(mesh,5));
-       }
-       //cface++;
-    } else if (cmin >= EDGE && cmin <= EDGE_LAST) {
-       if (cmin == 11) { //edge tag 11 corresponds to model edge 0
-          mesh->setModelEntity(e,getMdlEdge(mesh,0));
-       } else if (cmin == 12) {
-          mesh->setModelEntity(e,getMdlEdge(mesh,2));
-       } else if (cmin == 13) {
-          mesh->setModelEntity(e,getMdlEdge(mesh,3));
-       } else if (cmin == 14) {
-          mesh->setModelEntity(e,getMdlEdge(mesh,1));
-       } else if (cmin == 15) {
-          mesh->setModelEntity(e,getMdlEdge(mesh,4));
-       } else if (cmin == 16) {
-          mesh->setModelEntity(e,getMdlEdge(mesh,5));
-       } else if (cmin == 17) {
-          mesh->setModelEntity(e,getMdlEdge(mesh,7));
-       } else if (cmin == 18) {
-          mesh->setModelEntity(e,getMdlEdge(mesh,6));
-       } else if (cmin == 19) {
-          mesh->setModelEntity(e,getMdlEdge(mesh,8));
-       } else if (cmin == 20) {
-          mesh->setModelEntity(e,getMdlEdge(mesh,10));
-       } else if (cmin == 21) {
-          mesh->setModelEntity(e,getMdlEdge(mesh,11));
-       } else if (cmin == 22) {
-          mesh->setModelEntity(e,getMdlEdge(mesh,9));
-       }
-       //cedge++;
-    } else if (cmin >= VERTEX && cmin <= VERTEX_LAST) {
-       if (cmin == 31) { //vertex tag 31 corresponds to model vertex 0
-          mesh->setModelEntity(e,getMdlVtx(mesh,0));
-       } else if (cmin == 32) { //vertex tag 32 corresponds to model vertex 1
-          mesh->setModelEntity(e,getMdlVtx(mesh,1));
-       } else if (cmin == 33) { //vertex tag 33 corresponds to model vertex 3
-          mesh->setModelEntity(e,getMdlVtx(mesh,3));
-       } else if (cmin == 34) { //vertex tag 34 corresponds to model vertex 2
-          mesh->setModelEntity(e,getMdlVtx(mesh,2));
-       } else if (cmin == 35) { //vertex tag 35 corresponds to model vertex 4
-          mesh->setModelEntity(e,getMdlVtx(mesh,4));
-       } else if (cmin == 36) { //vertex tag 36 corresponds to model vertex 5
-          mesh->setModelEntity(e,getMdlVtx(mesh,5));
-       } else if (cmin == 37) { //vertex tag 37 corresponds to model vertex 7
-          mesh->setModelEntity(e,getMdlVtx(mesh,7));
-       } else if (cmin == 38) { //vertex tag 38 corresponds to model vertex 6
-          mesh->setModelEntity(e,getMdlVtx(mesh,6));
+    } else if (cmax >= 2000000) {  //max is a face
+       if(cmin==cmax) { //min is same face -> correct face to cls
+          mesh->setModelEntity(e,getMdlFace(mesh,cmax-2000000));
+          //cface++;
+       } else if (cmin >= 2000000) { // min is a DIFFERENT face -> interior
+          mesh->setModelEntity(e,getMdlRgn(model));
+       } else { 
+//FAILS  ROLL OUR OWN         int res = gmi_is_in_closure_of(model,gmi_find(model,dimMin,tagMin), gmi_find(model,dimMax,tagMax));
+          ff=-1;
+          gmi_ent* ge=gmi_find(model,dimMin,tagMin);
+          gmi_ent* gf =gmi_find(model,2,tagMax); // get the model face that goes with max
+          gmi_set* Edges = gmi_adjacent(model,gf,1);
+          k=0;
+          while(k<((Edges->n)) && ff==-1){ // check all edges until one found
+            if(dimMin==1) {
+              if(ge==Edges->e[k]) ff=k;  // edges must be checked. 
+            } else { // Verts probably can't fail but we still check
+              gmi_set* Verts = gmi_adjacent(model,Edges->e[k],0);
+              for (int j = 0; j < Verts->n; j++)  
+                if(ge==Verts->e[j]) ff=j;
+            }
+            k++;
+          }  
+          if( ff!=-1 ) { // is it in cls
+            mesh->setModelEntity(e,getMdlFace(mesh,cmax-2000000));
+          } else  // edge not in closure so interior
+          mesh->setModelEntity(e,getMdlRgn(model));
        }
-       //cvtx++;
+    } else if (cmax >= 1000000) { // max is an edge
+       if (cmin == cmax)  // cls on same edge 
+          mesh->setModelEntity(e,getMdlEdge(mesh,cmax-1000000));
+       else if (cmin >= 1000000) { // min is a different edge  and there is a face they must be in the closure of but which face is it 
+         gmi_set* maxFaces = gmi_adjacent(model,gmi_find(model,1,tagMax),2);  
+         gmi_set* minFaces = gmi_adjacent(model,gmi_find(model,1,tagMin),2);  
+         for (int i = 0; i < maxFaces->n; i++) {
+           for (int j = 0; j < minFaces->n; j++) {
+             if(minFaces->e[i]==maxFaces->e[j]){
+               int fftag=gmi_tag(model,maxFaces->e[j]);
+               mesh->setModelEntity(e,getMdlFace(mesh,fftag));
+             }
+           }
+         }
+       } else mesh->setModelEntity(e,getMdlEdge(mesh,cmax-1000000)); // min is vtx thus max is correct edge to classify
+    
+    } else if (cmax < 1000000) { // two model verts so this is a 1 elm in z mesh
+         gmi_set* maxEdges = gmi_adjacent(model,gmi_find(model,1,tagMax),1);  
+         gmi_set* minEdges = gmi_adjacent(model,gmi_find(model,1,tagMin),1);  
+         for (int i = 0; i < maxEdges->n; i++) {
+           for (int j = 0; j < minEdges->n; j++) {
+             if(minEdges->e[i]==maxEdges->e[j]){
+               int fftag=gmi_tag(model,maxEdges->e[j]);
+               mesh->setModelEntity(e,getMdlEdge(mesh,fftag));
+             }
+           }
+         }
+      
+    } else { // should never get here since cmax < 10000000 is a vtx
+            fprintf(stderr, "edge classification of these vert failed %d  %d \n", cmin, cmax);
     }
   }
   mesh->end(it);
@@ -252,90 +237,11 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
   apf::MeshIterator* it = mesh->begin(2);
   apf::MeshEntity* f;
   int c;
-  int TofM[6] = {0, 1, 3, 4, 2, 5}; // tags of discrete model are NOT same as model faces
-  // int EofF[6][4];
-  // int VofE[12][2];
-  int FofE[12][2];
-  FofE[0][0]=0;
-  FofE[0][1]=1;
-  FofE[1][0]=0;
-  FofE[1][1]=2;
-  FofE[2][0]=0;
-  FofE[2][1]=3;
-  FofE[3][0]=0;
-  FofE[3][1]=4;
-  FofE[4][0]=1;
-  FofE[4][1]=4;
-  FofE[5][0]=1;
-  FofE[5][1]=2;
-  FofE[6][0]=2;
-  FofE[6][1]=3;
-  FofE[7][0]=3;
-  FofE[7][1]=4;
-  FofE[8][0]=1;
-  FofE[8][1]=5;
-  FofE[9][0]=2;
-  FofE[9][1]=5;
-  FofE[10][0]=3;
-  FofE[10][1]=5;
-  FofE[11][0]=4;
-  FofE[11][1]=5;
-/*
-  VofE[0][0]=0;
-  VofE[0][1]=1;
-  VofE[1][0]=1;
-  VofE[1][1]=2;
-  VofE[2][0]=2;
-  VofE[2][1]=3;
-  VofE[3][0]=3;
-  VofE[3][1]=0;
-  VofE[4][0]=0;
-  VofE[4][1]=4;
-  VofE[5][0]=1;
-  VofE[5][1]=5;
-  VofE[6][0]=2;
-  VofE[6][1]=6;
-  VofE[7][0]=3;
-  VofE[7][1]=7;
-  VofE[8][0]=4;
-  VofE[8][1]=5;
-  VofE[9][0]=5;
-  VofE[9][1]=6;
-  VofE[10][0]=6;
-  VofE[10][1]=7;
-  VofE[11][0]=7;
-  VofE[11][1]=4;
-  EofF[0][0]=0;
-  EofF[0][1]=1;
-  EofF[0][2]=2;
-  EofF[0][3]=3;
-  EofF[1][0]=4;
-  EofF[1][1]=8;
-  EofF[1][2]=5;
-  EofF[1][3]=0;
-  EofF[2][0]=1;
-  EofF[2][1]=6;
-  EofF[2][2]=9;
-  EofF[2][3]=5;
-  EofF[3][0]=2;
-  EofF[3][1]=6;
-  EofF[3][2]=10;
-  EofF[3][3]=7;
-  EofF[4][0]=3;
-  EofF[4][1]=7;
-  EofF[4][2]=11;
-  EofF[4][3]=4;
-  EofF[5][0]=8;
-  EofF[5][1]=11;
-  EofF[5][2]=10;
-  EofF[5][3]=9;
-*/
-
   apf::Adjacent verts;
 
-  double distFromDebug1, distFromDebug2;
+  double distFromDebug1;
 //  apf::Vector3 xd1(-0.54864, 7.44015e-06, 0.0397148 );
-  apf::Vector3 xd1(-0.306845, 0.443585, 0.0291);
+  apf::Vector3 xd1(-2.63644, -0.953687, 0.00762);
   apf::Vector3 xd2(0.914478, 0.0145401, 0.04 );
   apf::Vector3 dx1;
   apf::Vector3 dx2;
@@ -344,174 +250,90 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
 
   while( (f = mesh->iterate(it)) ) {
     mesh->getAdjacent(f, 0, verts);
-    size_t nverts = verts.size();
- 
+    int nverts = verts.size();
+    if(1==0) {
     Centroid=apf::getLinearCentroid(mesh,f);
     dx1=xd1-Centroid;
-    dx2=xd2-Centroid;
+ //   dx2=xd2-Centroid;
     distFromDebug1=dx1[0]*dx1[0]
                   +dx1[1]*dx1[1]
                   +dx1[2]*dx1[2];
-    distFromDebug2=dx2[0]*dx2[0]
+/*    distFromDebug2=dx2[0]*dx2[0]
                   +dx2[1]*dx2[1]
                   +dx2[2]*dx2[2];
-
-    int cmin=100;
+*/
+    }
+    int cmin=100000000;
     int cmax=-100;
-    int cmid=-100;
     int ctri[4];  // up to 4 points on a face
-    int f1, f2, f1x, f2x, f1d, f2d;
-    int emax,emin,F0max,F0min,F1max,F1min;
-    for(size_t i=0; i<verts.size(); i++) {
+    for(int i=0; i<nverts; i++) {
       mesh->getIntTag(verts[i],vtxClass,&c);
       cmin=std::min(cmin,c);
       cmax=std::max(cmax,c);
       ctri[i]=c;
     }
-    int imax=0;  
-    int imin=0;  
-    for(size_t i=0; i<nverts; i++) {
-      if(ctri[i]==cmax) imax++;
-      if(ctri[i]==cmin) imin++;
-    }
-    if(cmax==cmin) { 
-       cmid= cmin;  // all nverts on same face
-    } else if (imax>=2) { // >=2 but not all with cmax
-       cmid=cmax;
-    } else if (imin>=2 ) { // >=2 but not all with min
-       cmid=cmin;
-    } else { // not 2 of either so mid is distinct
-      for(size_t i=0; i<nverts; i++) { // find it
-        if((ctri[i] != cmin) && (ctri[i] != cmax)) cmid=std::max(cmid,ctri[i]);  // max is to catch lowest dim/highest code for quads....actually not necessary since either of the other two will follow switches as noted below.
-      }
-    }
-
-    if(std::min(distFromDebug1,distFromDebug2) < 1e-12) {
-         fprintf(stderr, "%d %d %d %.15e %.15E %.15E \n", cmin, cmid, cmax, Centroid[0], Centroid[1], Centroid[2]);
-         for (size_t i=0; i < nverts; i++) {
+//    if(std::min(distFromDebug1,distFromDebug2) < 1e-12) {
+    if(1==0 && distFromDebug1 < 1e-3) {
+         fprintf(stderr, "%d %d %.15e %.15E %.15E \n", cmin, cmax, Centroid[0], Centroid[1], Centroid[2]);
+         for (int i=0; i < nverts; i++) {
             mesh->getPoint(verts[i],0,tmp);
 //            fprintf(stderr, "%d %.15e %.15E %.15E \n", i , tmp[0], tmp[1], tmp[2]);
          }
     }
-     
-    if (cmin == INTERIORTAG) {  // no brainer since a point on the interior always classifies interior
+    if (cmax >= 3000000) { // at least one vertex is interior -> cls interior
+       INTERIOR_REGION=cmax-3000000;
        mesh->setModelEntity(f,getMdlRgn(model));
        //cint++;
-    } else  if(cmax <= FACE_LAST && cmin >= FACE) { // all nodes on  model face(s?)
-         if(cmax != cmin) { // all on faces but not all on same so classified on interior
-             mesh->setModelEntity(f,getMdlRgn(model));
-         } else { // all on same face so classify on that one
-          mesh->setModelEntity(f,getMdlFace(mesh,TofM[cmin-1]));
-         }
-    } else if (cmid <= FACE_LAST )   {  // two points on face(s?)
-         if( cmin==cmid) {   // happens if 2 on SAME face
-             if( cmax > EDGE_LAST) {   // happens if third is a model vert which is ALWAYS in the same plane
-               mesh->setModelEntity(f,getMdlFace(mesh,TofM[cmin-1])); // classified on cmin face either way
-             } else {  // third on a model edge but might not be in closure of the face
-               f1=1+FofE[(cmax-11)][0];
-               f2=1+FofE[(cmax-11)][1];
-               if(f1==cmin || f2==cmin) { // if either face usage of this edge is cmin (or cmid) then class on face
-                   mesh->setModelEntity(f,getMdlFace(mesh,TofM[cmin-1]));  
-               } else mesh->setModelEntity(f,getMdlRgn(model));  // edge not in face closure so class on region
-             }
-          } else mesh->setModelEntity(f,getMdlRgn(model)); // cmin != cmid means on different faces so clas on region
-    } else if (cmin <=FACE_LAST) { // since 3, and 2 face class handled already this is only 1 pt class on face
-         if (cmax > EDGE_LAST ){ // never 2 on verts so  1face, 1edge and 1 vert  check to see if edge in closure of face
-               f1=1+FofE[(cmid-11)][0];  // get two uses of the edge which is cmid
-               f2=1+FofE[(cmid-11)][1];
-               if(f1==cmin || f2==cmin) { // one of the usages is on cmin face and vertex is always so class on face
-                   mesh->setModelEntity(f,getMdlFace(mesh,TofM[cmin-1]));  
-               } else mesh->setModelEntity(f,getMdlRgn(model));  // edge not in closor so classify on region
-         } else { // getting here only if 1face and 2 edges...most complicated case only face if  both edges in closure
-               f1x=1+FofE[(cmax-11)][0]; // first two find 2 face usages of max edge
-               f2x=1+FofE[(cmax-11)][1];
-               f1d=1+FofE[(cmid-11)][0]; // next two find 2 face usages of mid edge
-               f2d=1+FofE[(cmid-11)][1];
-               if(   (f1x==cmin || f2x==cmin)   // cmax edge is in cmin closure
-                  && (f1d==cmin || f2d==cmin) ) { // cmid edge is in cmin closure -- must have both to class on face
-                     mesh->setModelEntity(f,getMdlFace(mesh,TofM[cmin-1]));  
-                  } else mesh->setModelEntity(f,getMdlRgn(model)); // at least one of the edges not in face closure so clssify on region
-         }
-    } else if (cmax <= EDGE_LAST){ 
-         if( cmax==cmid || cmin== cmid) { // two on same edge one on other so this is on a face--  find which one
-                emax=cmax-11;
-                emin=cmin-11;
-   		F0max=FofE[emax][0];
- 		F0min=FofE[emin][0];
-  		F1max=FofE[emax][1];
-  		F1min=FofE[emin][1];
-                if(F0max==F0min) mesh->setModelEntity(f,getMdlFace(mesh,F0max)); // don't need map TofM[f1d-1]));
-                if(F0max==F1min) mesh->setModelEntity(f,getMdlFace(mesh,F0max)); 
-                if(F1max==F0min) mesh->setModelEntity(f,getMdlFace(mesh,F1max)); 
-                if(F1max==F1min) mesh->setModelEntity(f,getMdlFace(mesh,F1max)); 
-         } else  {// catch all now correct?? was not before
- // since none on face (3, 2, and 1 done above) and none on verts, 3 edges-> interior 
-               mesh->setModelEntity(f,getMdlRgn(model));
-         }
-    } else { // getting here only if none of the face verts are interior or on model face so must be 2 e and 1 v but
-        f1x=1+FofE[(cmin-11)][0]; // first two find 2 face usages of min edge
-        f2x=1+FofE[(cmin-11)][1];
-        f1d=1+FofE[(cmid-11)][0]; // next two find 2 face usages of mid edge
-        f2d=1+FofE[(cmid-11)][1];
-        if (f1x==f1d || f2x==f1d) {  // these two check if ether face using cmin match first face using cmid
-              mesh->setModelEntity(f,getMdlFace(mesh,TofM[f1d-1]));  
-        } else if ( f1x==f2d || f2x==f2d) {  // these two check if ether face using cmin match second face using cmid
-              mesh->setModelEntity(f,getMdlFace(mesh,TofM[f2d-1]));  
-        } else {
-            fprintf(stderr, "face classification of these vert classification failed %d %d  %d \n", cmin, cmid, cmax);
+    } else if(cmin >= 2000000) { // all nodes on  model face(s?)
+        if(cmax != cmin) { // all on faces but not all on same so classified on interior
+          mesh->setModelEntity(f,getMdlRgn(model));
+        } else { // all on same face so classify on that one
+          mesh->setModelEntity(f,getMdlFace(mesh,cmax-2000000));
         }
-        if (0) {   
-// above is cleaner but below was first idea
-// none of these nodal classifications know directly the face to classify this face on.  HARD CODE for now to our case
-         
-         int iface=0;
-         if(cmax==31) {
-            if( cmin== 11) {
-              if( cmid==14) iface=1; else iface=2;
-           }
-           else iface=5;
-         } else if(cmax==32) {
-            if( cmin== 11) {
-              if( cmid==12) iface=1; else iface=2;
-            }
-            else iface=3;
-         } else if(cmax==33) {
-            if( cmin== 12) {
-              if( cmid==13) iface=1; else iface=3;
-            }
-            else iface=4;
-         } else if(cmax==34) {
-            if( cmin== 13) {
-              if( cmid==14) iface=1; else iface=4;
-            }
-            else iface=5;
-         } else if(cmax==35) {
-            if( cmin== 15) {
-              if( cmid==19) iface=2; else iface=5;
-            }
-            else iface=6;
-         } else if(cmax==36) {
-            if( cmin== 16) {
-              if( cmid==19) iface=2; else iface=3;
-            }
-            else iface=6;
-         } else if(cmax==37) {
-            if( cmin== 17) {
-              if( cmid==20) iface=3; else iface=4;
-            }
-            else iface=6;
-         } else if(cmax==38) {
-            if( cmin== 18) {
-              if( cmid==21) iface=4; else iface=5;
-            }
-            else iface=6;
-         }
-         if(iface==0){
-            fprintf(stderr, "%d %d  %d \n", cmin, cmid, cmax);
-         }
-         PCU_ALWAYS_ASSERT(iface != 0);
-         mesh->setModelEntity(f,getMdlFace(mesh,TofM[iface-1])); // classified on chosen face 
+    } else { // faces can ONLY be classified on model faces or interior but their vertices can be classified  on model faces, edge, or vertices (regions caught in if).  Consequently, the simplest logic is to loop over faces  and check if any face has all of this mesh face's verts model classification in its closure
+      gmi_iter* gi=gmi_begin(model,2); // iterator over ALL the models faces
+      gmi_ent* gf;
+      gmi_ent* gt;
+      int i,dimi,ff,tagi,k;
+      int faceFound=0;
+      int ifaceS=0;
+      while ( (gf=gmi_next(model,gi)) && faceFound != nverts ) {
+        faceFound=0;
+        i=0;
+        while(i<nverts && faceFound==i){ // check all verts but one not found == not in face
+          ff=-1;
+          dimi=ctri[i]/1000000;
+          tagi=ctri[i]-dimi*1000000;
+          gt=gmi_find(model,dimi,tagi); // model entity that vertex i is classified on
+          // is_in_closure only works for simmetrix models so  roll our own
+          if(dimi ==2 && gf==gt) ff=ifaceS;
+          else if(dimi < 2) { // check this face's edges and those edges vertices
+            gmi_ent* ge=gmi_find(model,dimi,tagi);
+            gmi_set* Edges = gmi_adjacent(model,gf,1);
+            k=0;
+            while(k<((Edges->n)) && ff==-1){ // check all edges until one found
+              if(dimi==1) {
+                if(ge==Edges->e[k]) ff=k;  // edges must be checked. 
+              } else { // Verts probably can't fail but we still check
+                gmi_set* Verts = gmi_adjacent(model,Edges->e[k],0);
+                for (int j = 0; j < Verts->n; j++)  
+                   if(ge==Verts->e[j]) ff=j;
+              }
+              k++;
+            }  
+          }  
+          if( ff!=-1 ) faceFound++;
+          i++;
         }
+        if(faceFound==nverts ) {
+          int fftag=gmi_tag(model,gf);
+          mesh->setModelEntity(f,getMdlFace(mesh,fftag));
+        }
+        ifaceS++;
+      }
+      gmi_end(model,gi);
+      if(faceFound != nverts ) // none of the model face's closure held all verts classificaton  so interior
+        mesh->setModelEntity(f,getMdlRgn(model));
     }
   }
   mesh->end(it);
@@ -533,7 +355,7 @@ void setClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* t) {
   setRgnClassification(model,mesh);
   setFaceClassification(model,mesh,t);
   setEdgeClassification(model,mesh,t);
-  setVtxClassification(model,mesh,t);
+  setVtxClassification(model,mesh,t); 
   mesh->acceptChanges();
 }
 
@@ -604,12 +426,26 @@ void readClassification(FILE* f, int localNumVtx, int** classification) {
   *classification = new int[localNumVtx];
   rewind(f);
   int mdlId;
+  int maxmdlId=0;
   for(int i=0; i<localNumVtx; i++) {
     gmi_fscanf(f, 1, "%d",  &mdlId);
     (*classification)[i] = mdlId;
+    maxmdlId=std::max(maxmdlId,mdlId); 
   }
+  INTERIOR_REGION=maxmdlId-3000000; // correct if any vtx in region but will be negative if maxmdlId is on a face which is the case for 1 element in z... fix after reading model
 }
 
+void readFathers(FILE* f, int localNumVtx, int** fathers) {
+  *fathers = new int[localNumVtx];
+  rewind(f);
+  int mdlId;
+  for(int i=0; i<localNumVtx; i++) {
+    gmi_fscanf(f, 1, "%d",  &mdlId);
+    (*fathers)[i] = mdlId;
+  }
+}
+
+
 void readCoords(FILE* f, int& localnumvtx, double** coordinates) {
   *coordinates = new double[localnumvtx*3];
   for(int i=0; i<localnumvtx; i++) {
@@ -810,7 +646,7 @@ void readMesh(const char* meshfilename,
     sprintf(filename, "%s.%d",fathers2Dfilename,self);
     FILE* fff = fopen(filename, "r");
     PCU_ALWAYS_ASSERT(fff);
-    readClassification(fff, mesh.localNumVerts, &(mesh.fathers2D)); // note we re-use classification reader
+    readFathers(fff, mesh.localNumVerts, &(mesh.fathers2D)); // note we re-use classification reader
     fclose(fff);
   }
 
@@ -883,7 +719,17 @@ int main(int argc, char** argv)
     fprintf(stderr, "isMatched %d\n", isMatched);
 
   //gmi_model* model = gmi_load(".null");
-  gmi_model* model = apf::makeMdsBoxModel(2,2,2,1,1,1,0);
+  gmi_model* model = gmi_load("outModelr.dmg");
+//  gmi_model* model = apf::makeMdsBox(2,2,2,1,1,1,0);
+ //  1 element in z has no vertices to tell us the region ID so we have to find it from the tag number of the region that is adjacent to a face whose tag number will be the max when there is no region.  We know we have this problem when INTERIOR_REGION is negative
+  if(INTERIOR_REGION < 0) {
+    int maxmdlId=0;
+    maxmdlId=1000000+INTERIOR_REGION;  // get back to maxmdlId found 
+    gmi_set* Rgns=gmi_adjacent(model,gmi_find(model,2,maxmdlId),3);
+    for(int i=0; i<((Rgns->n)); i++) 
+      INTERIOR_REGION=gmi_tag(model,Rgns->e[i]);
+  }
+  printf("INTERIOR_REGION=%d\n",INTERIOR_REGION);
   apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, m.dim, isMatched);
   apf::GlobalToVert outMap;
   PCU_Debug_Open();

From f8768cd586da0cdfc4a499cc755bc48b1cb9ea01 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Tue, 24 Jan 2023 16:41:45 -0700
Subject: [PATCH 517/555] runs through on the CTS case at
 /nobackup/uncompressed/Models/GustWing/2dCTS/SlicedGeometry/FreshConvert/MGEN1/mner
 but NOTE this is a major change to how classification is promoted from
 vertices (as read) to higher dimensional entities. Note further, these
 changes read an outModelr.dmg file coming from mdlConvert (sic?).

---
 test/matchedNodeElmReader.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 5b0eaf01e..2616707fe 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -432,7 +432,9 @@ void readClassification(FILE* f, int localNumVtx, int** classification) {
     (*classification)[i] = mdlId;
     maxmdlId=std::max(maxmdlId,mdlId); 
   }
+  printf("maxmdlId,INTERIOR_REGION=%d %d\n",maxmdlId,INTERIOR_REGION);
   INTERIOR_REGION=maxmdlId-3000000; // correct if any vtx in region but will be negative if maxmdlId is on a face which is the case for 1 element in z... fix after reading model
+  printf("maxmdlId,INTERIOR_REGION=%d %d\n",maxmdlId,INTERIOR_REGION);
 }
 
 void readFathers(FILE* f, int localNumVtx, int** fathers) {
@@ -724,7 +726,9 @@ int main(int argc, char** argv)
  //  1 element in z has no vertices to tell us the region ID so we have to find it from the tag number of the region that is adjacent to a face whose tag number will be the max when there is no region.  We know we have this problem when INTERIOR_REGION is negative
   if(INTERIOR_REGION < 0) {
     int maxmdlId=0;
+    printf("maxmdlId,INTERIOR_REGION=%d %d\n",maxmdlId,INTERIOR_REGION);
     maxmdlId=1000000+INTERIOR_REGION;  // get back to maxmdlId found 
+    printf("maxmdlId,INTERIOR_REGION=%d %d\n",maxmdlId,INTERIOR_REGION);
     gmi_set* Rgns=gmi_adjacent(model,gmi_find(model,2,maxmdlId),3);
     for(int i=0; i<((Rgns->n)); i++) 
       INTERIOR_REGION=gmi_tag(model,Rgns->e[i]);

From 85f3c113fe5ee636ecc6b81a066ff575893834aa Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Wed, 25 Jan 2023 18:21:31 -0700
Subject: [PATCH 518/555] compiles with extruded classification for root
 vertices

---
 test/convert.cc | 95 +++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 93 insertions(+), 2 deletions(-)

diff --git a/test/convert.cc b/test/convert.cc
index 707a8d672..fd33b7bb1 100644
--- a/test/convert.cc
+++ b/test/convert.cc
@@ -159,13 +159,15 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
   // create a tag on vertices fathers
   pMeshDataId myFather = MD_newMeshDataId( "fathers2D");
   
-  pPList listV,listVn,regions,faces;
+  pPList listV,listVn,faces,regions;
   pFace face;
   pRegion region;
   pVertex vrts[4];
   int dir, err;
   int count2D=0;
+  pGRegion gregion;
   pGFace gface;
+  pGEdge gedge;
   pGFace ExtruRootFace=NULL;
   pVertex entV;
   pMesh meshP= PM_mesh	(sim_mesh, 0 );	
@@ -207,13 +209,101 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
       for(i=0; i< nvert ; i++) {
         int* markedData;
         if(!EN_getDataPtr((pEntity)vrts[i],myFather,(void**)&markedData)){  // not sure about marked yet
+          gType  vClassDim;
+          pGEntity vConG;
+          int foundESTag = 0;
+          int foundETag = 0;
+          int foundEETag = 0;
+          pPList gRegions,gFaces,gEdges;
+          double coordGVSelf[3];
+          double coordGVOther[3];
+          double dx,dy;
+          vClassDim=V_whatInType(vrts[i]);
+          vConG=V_whatIn(vrts[i]);
+          int echeck;
+          switch(vClassDim) {
+            case 0:   // classified on vert so vert->edge->vert
+              foundESTag = GEN_tag( (pGVertex) vConG ); // found Extrusion Start Tag
+              gEdges = GV_edges( (pGVertex) vConG ); // pPList of model edges adjacent to root model vertex
+              for(int j = 0; j < PList_size( gEdges ); j++ ){
+                pGEdge gEdge = (pGEdge) PList_item( gEdges , j ); // candidate edge
+                pGVertex gVert0 = GE_vertex( gEdge , 0 );
+                pGVertex gVert1 = GE_vertex( gEdge , 1 );
+                if( gVert0 == (pGVertex) vConG) { //1 is at other end of edge
+                   GV_point( gVert1 , coordGVOther );
+                   GV_point( gVert0 , coordGVSelf );
+                   dx = coordGVOther[0] - coordGVSelf[0];
+                   dy = coordGVOther[1] - coordGVSelf[1];
+                   if( dx*dx + dy*dy < 1e-12 ) {
+                      foundETag = GEN_tag( gEdge );
+                      foundEETag = GEN_tag( gVert1 );
+                   }
+                } else { // 0 is at the other edge
+                   GV_point( gVert0 , coordGVOther );
+                   GV_point( gVert1 , coordGVSelf );
+                   dx = coordGVOther[0] - coordGVSelf[0];
+                   dy = coordGVOther[1] - coordGVSelf[1];
+                   if( dx*dx + dy*dy < 1e-12) {
+                      foundETag = GEN_tag(gEdge);
+                      foundEETag = GEN_tag(gVert0);
+                   }
+                }
+              } 
+              PList_delete(gEdges);
+              break; 
+            case 1:   // classified on edge so edge->face->edge
+              foundESTag = GEN_tag( (pGEdge) vConG ); // found Extrusion Start Tag
+              GE_point( (pGEdge) vConG , 0.5, coordGVSelf ); 
+              gFaces = GE_faces( (pGEdge) vConG); // pPList of model faces adjacent to root model edge
+              for(int j = 0; j < PList_size( gFaces ); j++ ){
+                pGFace gFace = (pGFace) PList_item( gFaces , j ); // candidate face
+                gEdges = GF_edges( gFace ); // pPList of model edges of jth adjacent face
+                for(int k = 0; j < PList_size( gEdges ); k++ ){ // loop over that pPlist
+                  pGEdge gEdge = (pGEdge)  PList_item( gEdges , k ); // candidate edge on candidate face
+                  if( gEdge != (pGEdge) vConG ) { // exclude root classified edge
+                    echeck = GE_point( gEdge , 0.5, coordGVOther ); 
+                    dx = coordGVOther[0] - coordGVSelf[0];
+                    dy = coordGVOther[1] - coordGVSelf[1];
+                    if( dx*dx + dy*dy < 1e-12) {
+                       foundETag = GEN_tag( gFace ); // found Extruded Tag
+                       foundEETag = GEN_tag( gEdge );   // found Extrusion End Tag
+                    }
+                  }
+                }
+                PList_delete(gEdges);
+              }
+              PList_delete(gFaces);
+              break; 
+            case 2:   // classified on face so face->region->face
+              foundESTag = GEN_tag( (pGFace) vConG ); // found Extrusion Start Tag
+              double parFace[2] { 0.5 , 0.5 }; 
+              echeck = GF_point( (pGFace) vConG, parFace , coordGVSelf );
+              gRegions = GF_regions( (pGFace) vConG ); //pPList of model regions adjacent to root model Fac
+              pGRegion gRegion = (pGRegion) PList_item( gRegions , 0 ); // there can be only one for extrusions
+              gFaces = GR_faces( gRegion );
+              for( int j = 0; j < PList_size( gFaces ); j++ ){
+                pGFace gFace = (pGFace) PList_item( gFaces , j );
+                if( gFace != (pGFace) vConG ) { // exclude root classified face
+                  echeck = GF_point( (pGFace) vConG , parFace , coordGVOther );
+                  dx = coordGVOther[0] - coordGVSelf[0];
+                  dy = coordGVOther[1] - coordGVSelf[1];
+                  if( dx*dx + dy*dy < 1e-12) {
+                    foundETag = GEN_tag( gRegion ); // found Extruded Tag
+                    foundEETag = GEN_tag( gFace );   // found Extrusion End Tag
+                  }
+                }
+              }
+              PList_delete( gFaces );
+              break; 
+          } 
+          assert(foundEETag != 0);
           count2D++;
           int* vtxData = new int[1];
           vtxData[0] = count2D;
           EN_attachDataPtr((pEntity)vrts[i],myFather,(void*)vtxData);
           V_coord(vrts[i],coordNewPt[i]);
 
-          fprintf ( fcr, "%.15E %.15E %d \n", coordNewPt[i][0],coordNewPt[i][1], V_whatInType(vrts[i]));
+          fprintf ( fcr, "%.15E %.15E %d %d %d %d  \n", coordNewPt[i][0],coordNewPt[i][1], vClassDim, foundESTag, foundETag, foundEETag );
         }
       }
 
@@ -240,6 +330,7 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
       regions=PList_new();
       faces=PList_new();
       err = Extrusion_3DRegionsAndLayerFaces(region, regions, faces, 1);
+      PList_delete(regions); // not used so delete
       if(err!=1 && !PCU_Comm_Self())
         fprintf(stderr, "Extrusion_3DRegionsAndLayerFaces returned %d for err \n", err);
 

From dafc66e258bf86a61107c093817548e534b110e6 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Thu, 26 Jan 2023 10:29:59 -0700
Subject: [PATCH 519/555] commented out edge extru so that vertex happens
 before crash

---
 test/convert.cc | 61 +++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 57 insertions(+), 4 deletions(-)

diff --git a/test/convert.cc b/test/convert.cc
index fd33b7bb1..5f8b1eb49 100644
--- a/test/convert.cc
+++ b/test/convert.cc
@@ -217,10 +217,15 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
           pPList gRegions,gFaces,gEdges;
           double coordGVSelf[3];
           double coordGVOther[3];
+          double coordGVOther1[3];
           double dx,dy;
           vClassDim=V_whatInType(vrts[i]);
           vConG=V_whatIn(vrts[i]);
           int echeck;
+          double parFace[2]; 
+          double aLow[2];
+          double aHigh[2];
+          double pLow, pHigh;
           switch(vClassDim) {
             case 0:   // classified on vert so vert->edge->vert
               foundESTag = GEN_tag( (pGVertex) vConG ); // found Extrusion Start Tag
@@ -252,8 +257,46 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
               PList_delete(gEdges);
               break; 
             case 1:   // classified on edge so edge->face->edge
+/*               foundESTag=GEN_tag( (pGEdge) vConG); // found Extrusion Start Tag
+               pGVertex gVert0;
+               pGVertex gVert1;
+               gVert0=GE_vertex( (pGEdge) vConG,0);
+               gVert1=GE_vertex( (pGEdge) vConG,1);
+               GV_point(gVert0,coordGVOther); // reusing imperfectly named
+               GV_point(gVert1,coordGVSelf);  // reusing imperfectly named
+               coordGVSelf[0]=(coordGVOther[0]+coordGVSelf[0])/2; // avg x for edge centroid x
+               coordGVSelf[1]=(coordGVOther[1]+coordGVSelf[1])/2; // avg y for edge centroid y
+               gFaces=GE_faces((pGEdge)vConG); // pPList of model faces adjacent to root model edge
+               for(int j = 0; j < PList_size( gFaces ); j++){
+                 pGFace gFace = (pGFace)PList_item( gFaces , j ); // candidate face
+                 gEdges=GF_edges( gFace ); // pPList of model edges of jth adjacent face
+                 for(int k = 0; j < PList_size( gEdges ); k++){ // loop over that pPlist
+                   pGEdge gEdge=(pGEdge) PList_item(gEdges,k); // candidate edge on candidate face
+                   if(gEdge != (pGEdge) vConG) { // exclude root classified edge
+                     gVert0=GE_vertex( gEdge,0);
+                     gVert1=GE_vertex( gEdge,1);
+                     GV_point(gVert0,coordGVOther); // reusing imperfectly named
+                     GV_point(gVert0,coordGVOther1); 
+                     coordGVOther[0]=(coordGVOther[0]+coordGVOther1[0])/2; // avg x for edge centroid x
+                     coordGVOther[1]=(coordGVOther[1]+coordGVOther1[1])/2; // avg y for edge centroid y
+                     dx=(coordGVOther[0]-coordGVSelf[0]);
+                     dy=(coordGVOther[1]-coordGVSelf[1]);
+                     if(dx*dx+dy*dy < 1e-12) {
+                        foundETag=GEN_tag(gFace); // found Extruded Tag
+                        foundEETag=GEN_tag(gEdge);   // found Extrusion End Tag
+                     }
+                   }
+                 }
+                 PList_delete( gEdges );
+               }
+*/
+/*
+               PList_delete( gFaces );
               foundESTag = GEN_tag( (pGEdge) vConG ); // found Extrusion Start Tag
-              GE_point( (pGEdge) vConG , 0.5, coordGVSelf ); 
+              GE_parRange ( (pGEdge) vConG, &pLow, &pHigh);
+//              GE_parRange ( (pGEdge) vConG, aLow, aHigh);
+              parFace[0] = ( pLow + pHigh ) * 0.5;
+              echeck = GE_point( (pGEdge) vConG , parFace[0], coordGVSelf ); 
               gFaces = GE_faces( (pGEdge) vConG); // pPList of model faces adjacent to root model edge
               for(int j = 0; j < PList_size( gFaces ); j++ ){
                 pGFace gFace = (pGFace) PList_item( gFaces , j ); // candidate face
@@ -261,7 +304,9 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
                 for(int k = 0; j < PList_size( gEdges ); k++ ){ // loop over that pPlist
                   pGEdge gEdge = (pGEdge)  PList_item( gEdges , k ); // candidate edge on candidate face
                   if( gEdge != (pGEdge) vConG ) { // exclude root classified edge
-                    echeck = GE_point( gEdge , 0.5, coordGVOther ); 
+                    GE_parRange ( gEdge, &pLow, &pHigh);
+                    parFace[0] = ( pLow + pHigh ) * 0.5;
+                    echeck = GE_point( gEdge , parFace[0], coordGVOther ); 
                     dx = coordGVOther[0] - coordGVSelf[0];
                     dy = coordGVOther[1] - coordGVSelf[1];
                     if( dx*dx + dy*dy < 1e-12) {
@@ -273,10 +318,14 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
                 PList_delete(gEdges);
               }
               PList_delete(gFaces);
+*/
               break; 
             case 2:   // classified on face so face->region->face
               foundESTag = GEN_tag( (pGFace) vConG ); // found Extrusion Start Tag
-              double parFace[2] { 0.5 , 0.5 }; 
+              GF_parRange ( (pGFace) vConG, 0, &pLow, &pHigh);
+              parFace[0] = ( pLow + pHigh ) * 0.5;
+              GF_parRange ( (pGFace) vConG, 1, &pLow, &pHigh);
+              parFace[1] = ( pLow + pHigh ) * 0.5;
               echeck = GF_point( (pGFace) vConG, parFace , coordGVSelf );
               gRegions = GF_regions( (pGFace) vConG ); //pPList of model regions adjacent to root model Fac
               pGRegion gRegion = (pGRegion) PList_item( gRegions , 0 ); // there can be only one for extrusions
@@ -284,6 +333,10 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
               for( int j = 0; j < PList_size( gFaces ); j++ ){
                 pGFace gFace = (pGFace) PList_item( gFaces , j );
                 if( gFace != (pGFace) vConG ) { // exclude root classified face
+                  GF_parRange ( gFace, 0, &pLow, &pHigh);
+                  parFace[0] = ( pLow + pHigh ) * 0.5;
+                  GF_parRange ( gFace, 1, &pLow, &pHigh);
+                  parFace[1] = ( pLow + pHigh ) * 0.5;
                   echeck = GF_point( (pGFace) vConG , parFace , coordGVOther );
                   dx = coordGVOther[0] - coordGVSelf[0];
                   dy = coordGVOther[1] - coordGVSelf[1];
@@ -296,7 +349,7 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
               PList_delete( gFaces );
               break; 
           } 
-          assert(foundEETag != 0);
+//          assert(foundEETag != 0);
           count2D++;
           int* vtxData = new int[1];
           vtxData[0] = count2D;

From 3a8c6221783e1b75e2eb13feb68fde51b01114dd Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Thu, 26 Jan 2023 18:17:12 -0700
Subject: [PATCH 520/555] dim 0 and dim 1 passing.....close

---
 test/convert.cc | 40 +++++++++++++++++++++++++++++++++-------
 1 file changed, 33 insertions(+), 7 deletions(-)

diff --git a/test/convert.cc b/test/convert.cc
index 5f8b1eb49..a1ebb4f4b 100644
--- a/test/convert.cc
+++ b/test/convert.cc
@@ -2,6 +2,7 @@
 #include <lionPrint.h>
 #include <MeshSim.h>
 #include <SimPartitionedMesh.h>
+#include "SimParasolidKrnl.h"
 #include <SimAdvMeshing.h>
 #include <SimUtil.h>
 #include <apfSIM.h>
@@ -20,9 +21,14 @@
 #include <iostream>
 #include <cassert>
 #include <getopt.h>
-
+#include <string.h>
 #include <stdio.h>
 
+using namespace std;
+
+
+
+
 apf::Field* convert_my_tag(apf::Mesh* m, apf::MeshTag* t) {
   apf::MeshEntity* vtx;
   apf::MeshIterator* it = m->begin(0);
@@ -133,7 +139,8 @@ void getConfig(int argc, char** argv) {
   gmi_path = argv[i++];
   sms_path = argv[i++];
   smb_path = argv[i++];
-
+//  gmi_native_path= "/nobackup/uncompressed/Models/GustWing/2dCTS/SlicedGeomtryV12/sms2mds/geom.xmt_txt";
+  gmi_native_path= "geom.xmt_txt";
   if (!PCU_Comm_Self()) {
     printf ("fix_pyramids %d attach_order %d enable_log %d extruRootPath %s\n",
             should_fix_pyramids, should_attach_order, should_log, extruRootPath);
@@ -289,9 +296,9 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
                  }
                  PList_delete( gEdges );
                }
-*/
-/*
                PList_delete( gFaces );
+*/
+
               foundESTag = GEN_tag( (pGEdge) vConG ); // found Extrusion Start Tag
               GE_parRange ( (pGEdge) vConG, &pLow, &pHigh);
 //              GE_parRange ( (pGEdge) vConG, aLow, aHigh);
@@ -301,7 +308,7 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
               for(int j = 0; j < PList_size( gFaces ); j++ ){
                 pGFace gFace = (pGFace) PList_item( gFaces , j ); // candidate face
                 gEdges = GF_edges( gFace ); // pPList of model edges of jth adjacent face
-                for(int k = 0; j < PList_size( gEdges ); k++ ){ // loop over that pPlist
+                for(int k = 0; k < PList_size( gEdges ); k++ ){ // loop over that pPlist
                   pGEdge gEdge = (pGEdge)  PList_item( gEdges , k ); // candidate edge on candidate face
                   if( gEdge != (pGEdge) vConG ) { // exclude root classified edge
                     GE_parRange ( gEdge, &pLow, &pHigh);
@@ -318,7 +325,7 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
                 PList_delete(gEdges);
               }
               PList_delete(gFaces);
-*/
+
               break; 
             case 2:   // classified on face so face->region->face
               foundESTag = GEN_tag( (pGFace) vConG ); // found Extrusion Start Tag
@@ -349,7 +356,7 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
               PList_delete( gFaces );
               break; 
           } 
-//          assert(foundEETag != 0);
+          assert(foundEETag != 0);
           count2D++;
           int* vtxData = new int[1];
           vtxData[0] = count2D;
@@ -462,7 +469,26 @@ int main(int argc, char** argv)
     mdl = gmi_sim_load(gmi_native_path,gmi_path);
   else
     mdl = gmi_load(gmi_path);
+
   pGModel simModel = gmi_export_sim(mdl);
+/*
+  pParasolidNativeModel nModel = ParasolidNM_createFromFile(gmi_native_path,0);
+  pGModel    Amodel = GAM_createFromNativeModel(nModel,progress); 
+*/
+/* leaving this litle model tester in for a while
+  pGModel    Amodel = simModel;
+  double coordGVOther[3];
+  pGVertex gvertex;
+  GVIter gvIter=GM_vertexIter(Amodel);
+  while ( (gvertex=GVIter_next(gvIter))) {
+    int id = GEN_tag(gvertex);
+    GV_point( gvertex , coordGVOther );
+    cout<< id << " tag and coords " << coordGVOther[0] << " "  << coordGVOther[1]<< " " << coordGVOther[2] <<endl; 
+  }
+  GVIter_delete(gvIter);
+  assert(false);
+*/
+
   double t0 = PCU_Time();
   pParMesh sim_mesh = PM_load(sms_path, simModel, progress);
   double t1 = PCU_Time();

From a7f2e487f2e87119d565bd7a48584ac1c94dd96f Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Thu, 26 Jan 2023 21:10:03 -0500
Subject: [PATCH 521/555] better warning for unrecognized options

---
 pumi-meshes     |  2 +-
 test/convert.cc | 11 ++++++-----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/pumi-meshes b/pumi-meshes
index 635fff268..cc07751ec 160000
--- a/pumi-meshes
+++ b/pumi-meshes
@@ -1 +1 @@
-Subproject commit 635fff268201e528628595595fd320ab5aa5b8e5
+Subproject commit cc07751ec833240b9a8e09bd470779d4102e3e92
diff --git a/test/convert.cc b/test/convert.cc
index a1ebb4f4b..1f5d58daa 100644
--- a/test/convert.cc
+++ b/test/convert.cc
@@ -121,7 +121,7 @@ void getConfig(int argc, char** argv) {
         break;
       case '?':
         if (!PCU_Comm_Self())
-          printf ("warning: skipping unrecognized option\n");
+          printf ("warning: skipping unrecognized option \'%s\'\n", argv[optind-1]);
         break;
       default:
         if (!PCU_Comm_Self())
@@ -139,8 +139,6 @@ void getConfig(int argc, char** argv) {
   gmi_path = argv[i++];
   sms_path = argv[i++];
   smb_path = argv[i++];
-//  gmi_native_path= "/nobackup/uncompressed/Models/GustWing/2dCTS/SlicedGeomtryV12/sms2mds/geom.xmt_txt";
-  gmi_native_path= "geom.xmt_txt";
   if (!PCU_Comm_Self()) {
     printf ("fix_pyramids %d attach_order %d enable_log %d extruRootPath %s\n",
             should_fix_pyramids, should_attach_order, should_log, extruRootPath);
@@ -465,10 +463,13 @@ int main(int argc, char** argv)
   Progress_setDefaultCallback(progress);
 
   gmi_model* mdl;
-  if( gmi_native_path )
+  if( gmi_native_path ) {
+    if (!PCU_Comm_Self())
+      fprintf(stderr, "loading native model %s\n", gmi_native_path);
     mdl = gmi_sim_load(gmi_native_path,gmi_path);
-  else
+  } else {
     mdl = gmi_load(gmi_path);
+  }
 
   pGModel simModel = gmi_export_sim(mdl);
 /*

From 15d6806101f1c500492b0ca87b9ba046890985c3 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Thu, 26 Jan 2023 19:50:42 -0700
Subject: [PATCH 522/555] works

---
 test/convert.cc | 51 ++++++++++---------------------------------------
 1 file changed, 10 insertions(+), 41 deletions(-)

diff --git a/test/convert.cc b/test/convert.cc
index a1ebb4f4b..60b3f1b33 100644
--- a/test/convert.cc
+++ b/test/convert.cc
@@ -188,6 +188,9 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
 
   FILE* fid = fopen(extrusionFaceFile, "r"); // helper file that contains all faces with extrusions
   assert(fid);
+  double VdisTol=1e-12;
+  double EdisTol=1e-7;
+  double FdisTol=1e-7;
   while(1 == fscanf(fid,"%d",&ExtruRootId)) {
     fprintf(stderr,"ExtruRootId= %d \n",ExtruRootId);
     //find the root face of the extrusion
@@ -246,7 +249,7 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
                    GV_point( gVert0 , coordGVSelf );
                    dx = coordGVOther[0] - coordGVSelf[0];
                    dy = coordGVOther[1] - coordGVSelf[1];
-                   if( dx*dx + dy*dy < 1e-12 ) {
+                   if( dx*dx + dy*dy < VdisTol ) {
                       foundETag = GEN_tag( gEdge );
                       foundEETag = GEN_tag( gVert1 );
                    }
@@ -255,7 +258,7 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
                    GV_point( gVert1 , coordGVSelf );
                    dx = coordGVOther[0] - coordGVSelf[0];
                    dy = coordGVOther[1] - coordGVSelf[1];
-                   if( dx*dx + dy*dy < 1e-12) {
+                   if( dx*dx + dy*dy < VdisTol) {
                       foundETag = GEN_tag(gEdge);
                       foundEETag = GEN_tag(gVert0);
                    }
@@ -264,44 +267,8 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
               PList_delete(gEdges);
               break; 
             case 1:   // classified on edge so edge->face->edge
-/*               foundESTag=GEN_tag( (pGEdge) vConG); // found Extrusion Start Tag
-               pGVertex gVert0;
-               pGVertex gVert1;
-               gVert0=GE_vertex( (pGEdge) vConG,0);
-               gVert1=GE_vertex( (pGEdge) vConG,1);
-               GV_point(gVert0,coordGVOther); // reusing imperfectly named
-               GV_point(gVert1,coordGVSelf);  // reusing imperfectly named
-               coordGVSelf[0]=(coordGVOther[0]+coordGVSelf[0])/2; // avg x for edge centroid x
-               coordGVSelf[1]=(coordGVOther[1]+coordGVSelf[1])/2; // avg y for edge centroid y
-               gFaces=GE_faces((pGEdge)vConG); // pPList of model faces adjacent to root model edge
-               for(int j = 0; j < PList_size( gFaces ); j++){
-                 pGFace gFace = (pGFace)PList_item( gFaces , j ); // candidate face
-                 gEdges=GF_edges( gFace ); // pPList of model edges of jth adjacent face
-                 for(int k = 0; j < PList_size( gEdges ); k++){ // loop over that pPlist
-                   pGEdge gEdge=(pGEdge) PList_item(gEdges,k); // candidate edge on candidate face
-                   if(gEdge != (pGEdge) vConG) { // exclude root classified edge
-                     gVert0=GE_vertex( gEdge,0);
-                     gVert1=GE_vertex( gEdge,1);
-                     GV_point(gVert0,coordGVOther); // reusing imperfectly named
-                     GV_point(gVert0,coordGVOther1); 
-                     coordGVOther[0]=(coordGVOther[0]+coordGVOther1[0])/2; // avg x for edge centroid x
-                     coordGVOther[1]=(coordGVOther[1]+coordGVOther1[1])/2; // avg y for edge centroid y
-                     dx=(coordGVOther[0]-coordGVSelf[0]);
-                     dy=(coordGVOther[1]-coordGVSelf[1]);
-                     if(dx*dx+dy*dy < 1e-12) {
-                        foundETag=GEN_tag(gFace); // found Extruded Tag
-                        foundEETag=GEN_tag(gEdge);   // found Extrusion End Tag
-                     }
-                   }
-                 }
-                 PList_delete( gEdges );
-               }
-               PList_delete( gFaces );
-*/
-
               foundESTag = GEN_tag( (pGEdge) vConG ); // found Extrusion Start Tag
               GE_parRange ( (pGEdge) vConG, &pLow, &pHigh);
-//              GE_parRange ( (pGEdge) vConG, aLow, aHigh);
               parFace[0] = ( pLow + pHigh ) * 0.5;
               echeck = GE_point( (pGEdge) vConG , parFace[0], coordGVSelf ); 
               gFaces = GE_faces( (pGEdge) vConG); // pPList of model faces adjacent to root model edge
@@ -316,7 +283,7 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
                     echeck = GE_point( gEdge , parFace[0], coordGVOther ); 
                     dx = coordGVOther[0] - coordGVSelf[0];
                     dy = coordGVOther[1] - coordGVSelf[1];
-                    if( dx*dx + dy*dy < 1e-12) {
+                    if( dx*dx + dy*dy < EdisTol) {
                        foundETag = GEN_tag( gFace ); // found Extruded Tag
                        foundEETag = GEN_tag( gEdge );   // found Extrusion End Tag
                     }
@@ -336,18 +303,20 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
               echeck = GF_point( (pGFace) vConG, parFace , coordGVSelf );
               gRegions = GF_regions( (pGFace) vConG ); //pPList of model regions adjacent to root model Fac
               pGRegion gRegion = (pGRegion) PList_item( gRegions , 0 ); // there can be only one for extrusions
+              int  currentTagRegion = GEN_tag( gRegion ); // candidate region's tag (debugging only)
               gFaces = GR_faces( gRegion );
               for( int j = 0; j < PList_size( gFaces ); j++ ){
                 pGFace gFace = (pGFace) PList_item( gFaces , j );
+                int  currentTag = GEN_tag( gFace ); // candidate face's tag (debugging only)
                 if( gFace != (pGFace) vConG ) { // exclude root classified face
                   GF_parRange ( gFace, 0, &pLow, &pHigh);
                   parFace[0] = ( pLow + pHigh ) * 0.5;
                   GF_parRange ( gFace, 1, &pLow, &pHigh);
                   parFace[1] = ( pLow + pHigh ) * 0.5;
-                  echeck = GF_point( (pGFace) vConG , parFace , coordGVOther );
+                  echeck = GF_point( (pGFace) gFace , parFace , coordGVOther );
                   dx = coordGVOther[0] - coordGVSelf[0];
                   dy = coordGVOther[1] - coordGVSelf[1];
-                  if( dx*dx + dy*dy < 1e-12) {
+                  if( dx*dx + dy*dy < FdisTol) {
                     foundETag = GEN_tag( gRegion ); // found Extruded Tag
                     foundEETag = GEN_tag( gFace );   // found Extrusion End Tag
                   }

From 67696835e79a367f9b3bcadf44752df23d204f39 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Fri, 27 Jan 2023 08:07:12 -0700
Subject: [PATCH 523/555]  comments for future strnger versions

---
 pumi-meshes     | 2 +-
 test/convert.cc | 9 +++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/pumi-meshes b/pumi-meshes
index cc07751ec..635fff268 160000
--- a/pumi-meshes
+++ b/pumi-meshes
@@ -1 +1 @@
-Subproject commit cc07751ec833240b9a8e09bd470779d4102e3e92
+Subproject commit 635fff268201e528628595595fd320ab5aa5b8e5
diff --git a/test/convert.cc b/test/convert.cc
index d16f5edb0..61d393433 100644
--- a/test/convert.cc
+++ b/test/convert.cc
@@ -198,6 +198,15 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
       if(id==ExtruRootId) ExtruRootFace=gface;
     }
     assert(ExtruRootFace != NULL);
+    // notes for next version 
+    // we could get the region, get a list of vertices for the region, loop over those vertices to find the extrusion length
+    // then instead of checking matching x,y of opposite ends of the extrusion, abs(zroot - zOppositeRoot) matching the extrusion 
+    // length gives the match
+    // Note if we want to handle extrusions not purely in z, the normal to ExtruRootFace provides an axis to project a vector between     // the centers of the root and candidate oppositeRoot entity on to.
+
+    // all of the above assumes translation extrusion.  Rotation extrusion (sweeping extruded entiy over an arc of some angle about 
+    // a given axis) is useful but this would require some code change.  The principle is the same.  Every root entity has another 
+    // oppositeRoot entity whose position obeys a fixed angle rotation about a fixed axis.
 
     FIter fIter = M_classifiedFaceIter( meshP, ExtruRootFace, 0 ); // 0 says I don't want closure
     while ((face = FIter_next(fIter))) {

From b5ca7f3f6b358d2facf528d9f1155d0616271db8 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Fri, 27 Jan 2023 08:25:25 -0700
Subject: [PATCH 524/555] silence warnings

---
 test/convert.cc | 35 ++++++++++++-----------------------
 1 file changed, 12 insertions(+), 23 deletions(-)

diff --git a/test/convert.cc b/test/convert.cc
index 61d393433..63c28a2cd 100644
--- a/test/convert.cc
+++ b/test/convert.cc
@@ -170,9 +170,7 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
   pVertex vrts[4];
   int dir, err;
   int count2D=0;
-  pGRegion gregion;
   pGFace gface;
-  pGEdge gedge;
   pGFace ExtruRootFace=NULL;
   pVertex entV;
   pMesh meshP= PM_mesh	(sim_mesh, 0 );	
@@ -234,17 +232,12 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
           pPList gRegions,gFaces,gEdges;
           double coordGVSelf[3];
           double coordGVOther[3];
-          double coordGVOther1[3];
           double dx,dy;
           vClassDim=V_whatInType(vrts[i]);
           vConG=V_whatIn(vrts[i]);
-          int echeck;
           double parFace[2]; 
-          double aLow[2];
-          double aHigh[2];
           double pLow, pHigh;
-          switch(vClassDim) {
-            case 0:   // classified on vert so vert->edge->vert
+          if(vClassDim == 0) {// classified on vert so vert->edge->vert
               foundESTag = GEN_tag( (pGVertex) vConG ); // found Extrusion Start Tag
               gEdges = GV_edges( (pGVertex) vConG ); // pPList of model edges adjacent to root model vertex
               for(int j = 0; j < PList_size( gEdges ); j++ ){
@@ -272,12 +265,11 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
                 }
               } 
               PList_delete(gEdges);
-              break; 
-            case 1:   // classified on edge so edge->face->edge
+	  } else if(vClassDim == 1) {   // classified on edge so edge->face->edge
               foundESTag = GEN_tag( (pGEdge) vConG ); // found Extrusion Start Tag
               GE_parRange ( (pGEdge) vConG, &pLow, &pHigh);
               parFace[0] = ( pLow + pHigh ) * 0.5;
-              echeck = GE_point( (pGEdge) vConG , parFace[0], coordGVSelf ); 
+              GE_point( (pGEdge) vConG , parFace[0], coordGVSelf ); 
               gFaces = GE_faces( (pGEdge) vConG); // pPList of model faces adjacent to root model edge
               for(int j = 0; j < PList_size( gFaces ); j++ ){
                 pGFace gFace = (pGFace) PList_item( gFaces , j ); // candidate face
@@ -287,7 +279,7 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
                   if( gEdge != (pGEdge) vConG ) { // exclude root classified edge
                     GE_parRange ( gEdge, &pLow, &pHigh);
                     parFace[0] = ( pLow + pHigh ) * 0.5;
-                    echeck = GE_point( gEdge , parFace[0], coordGVOther ); 
+                    GE_point( gEdge , parFace[0], coordGVOther ); 
                     dx = coordGVOther[0] - coordGVSelf[0];
                     dy = coordGVOther[1] - coordGVSelf[1];
                     if( dx*dx + dy*dy < EdisTol) {
@@ -299,28 +291,24 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
                 PList_delete(gEdges);
               }
               PList_delete(gFaces);
-
-              break; 
-            case 2:   // classified on face so face->region->face
+	   } else if(vClassDim == 2) {   // classified on face so face->region->face
               foundESTag = GEN_tag( (pGFace) vConG ); // found Extrusion Start Tag
               GF_parRange ( (pGFace) vConG, 0, &pLow, &pHigh);
               parFace[0] = ( pLow + pHigh ) * 0.5;
               GF_parRange ( (pGFace) vConG, 1, &pLow, &pHigh);
               parFace[1] = ( pLow + pHigh ) * 0.5;
-              echeck = GF_point( (pGFace) vConG, parFace , coordGVSelf );
+              GF_point( (pGFace) vConG, parFace , coordGVSelf );
               gRegions = GF_regions( (pGFace) vConG ); //pPList of model regions adjacent to root model Fac
               pGRegion gRegion = (pGRegion) PList_item( gRegions , 0 ); // there can be only one for extrusions
-              int  currentTagRegion = GEN_tag( gRegion ); // candidate region's tag (debugging only)
               gFaces = GR_faces( gRegion );
               for( int j = 0; j < PList_size( gFaces ); j++ ){
                 pGFace gFace = (pGFace) PList_item( gFaces , j );
-                int  currentTag = GEN_tag( gFace ); // candidate face's tag (debugging only)
                 if( gFace != (pGFace) vConG ) { // exclude root classified face
                   GF_parRange ( gFace, 0, &pLow, &pHigh);
                   parFace[0] = ( pLow + pHigh ) * 0.5;
                   GF_parRange ( gFace, 1, &pLow, &pHigh);
                   parFace[1] = ( pLow + pHigh ) * 0.5;
-                  echeck = GF_point( (pGFace) gFace , parFace , coordGVOther );
+                  GF_point( (pGFace) gFace , parFace , coordGVOther );
                   dx = coordGVOther[0] - coordGVSelf[0];
                   dy = coordGVOther[1] - coordGVSelf[1];
                   if( dx*dx + dy*dy < FdisTol) {
@@ -329,10 +317,11 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
                   }
                 }
               }
-              PList_delete( gFaces );
-              break; 
-          } 
-          assert(foundEETag != 0);
+              PList_delete( gFaces ); 
+	   } else {
+             PCU_ALWAYS_ASSERT(false);
+	   }
+          PCU_ALWAYS_ASSERT(foundEETag != 0);
           count2D++;
           int* vtxData = new int[1];
           vtxData[0] = count2D;

From d2adee540075a0fd1b53f4ca71a9ed616dcb20a6 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Fri, 27 Jan 2023 08:34:24 -0700
Subject: [PATCH 525/555] update convert extrusion test case

---
 pumi-meshes | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pumi-meshes b/pumi-meshes
index 635fff268..73284454c 160000
--- a/pumi-meshes
+++ b/pumi-meshes
@@ -1 +1 @@
-Subproject commit 635fff268201e528628595595fd320ab5aa5b8e5
+Subproject commit 73284454c38572039dac887b9d4b8b792cb423df

From 7ca72b68c7050cf58b46e42389c721b184e61f51 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Sat, 28 Jan 2023 22:01:00 -0700
Subject: [PATCH 526/555] non-manifold passess verify...committing with
 alternate source paths befoe cleanup

---
 test/convert.cc              |  79 ++++++++---
 test/matchedNodeElmReader.cc | 260 ++++++++++++++++++++++++++++-------
 2 files changed, 274 insertions(+), 65 deletions(-)

diff --git a/test/convert.cc b/test/convert.cc
index 63c28a2cd..b35b37919 100644
--- a/test/convert.cc
+++ b/test/convert.cc
@@ -171,7 +171,6 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
   int dir, err;
   int count2D=0;
   pGFace gface;
-  pGFace ExtruRootFace=NULL;
   pVertex entV;
   pMesh meshP= PM_mesh	(sim_mesh, 0 );	
 
@@ -188,6 +187,7 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
   double EdisTol=1e-7;
   double FdisTol=1e-7;
   while(1 == fscanf(fid,"%d",&ExtruRootId)) {
+    pGFace ExtruRootFace=NULL;
     fprintf(stderr,"ExtruRootId= %d \n",ExtruRootId);
     //find the root face of the extrusion
     GFIter gfIter=GM_faceIter(simModel);
@@ -205,6 +205,40 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
     // all of the above assumes translation extrusion.  Rotation extrusion (sweeping extruded entiy over an arc of some angle about 
     // a given axis) is useful but this would require some code change.  The principle is the same.  Every root entity has another 
     // oppositeRoot entity whose position obeys a fixed angle rotation about a fixed axis.
+    pPList gRegions,gFaces,gEdges,gVertices;
+    double parFace[2]; 
+    double normal[3]; 
+    double pLow, pHigh;
+    double coordGVSelf[3];
+    double coordGVOther[3];
+    double xmin[3] = {1.0e8, 1.0e8, 1.0e8};
+    double xmax[3] = {-1.0e8, -1.0e8, -1.0e8};
+    GF_parRange ( ExtruRootFace, 0, &pLow, &pHigh);
+    parFace[0] = ( pLow + pHigh ) * 0.5;
+    GF_parRange ( ExtruRootFace, 1, &pLow, &pHigh);
+    parFace[1] = ( pLow + pHigh ) * 0.5;
+    GF_normal(ExtruRootFace,parFace,normal);
+    gRegions=GF_regions(ExtruRootFace); //pPList of model regions adjacent to root model Fac
+    pGRegion gRegion = (pGRegion) PList_item( gRegions , 0 ); // there can be only one for extrusions
+    gVertices = GR_vertices(gRegion);
+    for( int j = 0; j < PList_size( gVertices ); j++ ){
+      pGVertex gVertex = (pGVertex) PList_item( gVertices , j );
+      GV_point( gVertex , coordGVOther );
+      for( int i = 0; i < 3; i++) {
+        xmin[i]=std::min(xmin[i],coordGVOther[i]);
+        xmax[i]=std::max(xmax[i],coordGVOther[i]);
+      }
+    }
+    // just in case normal is not a unit vector
+    double nLength=(normal[0]*normal[0]+normal[1]*normal[1]+normal[2]*normal[2]);
+    for( int i = 0; i < 3; i++) normal[i]=normal[i]/nLength;
+
+    double sepVec[3];
+    for( int i = 0; i < 3; i++) sepVec[i]=xmax[i]-xmin[i];  
+    double ExtruDistance=abs(sepVec[0]*normal[0]+sepVec[1]*normal[1]+sepVec[2]*normal[2]);
+    PList_delete(gRegions);
+    PList_delete(gVertices);
+   
 
     FIter fIter = M_classifiedFaceIter( meshP, ExtruRootFace, 0 ); // 0 says I don't want closure
     while ((face = FIter_next(fIter))) {
@@ -229,14 +263,9 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
           int foundESTag = 0;
           int foundETag = 0;
           int foundEETag = 0;
-          pPList gRegions,gFaces,gEdges;
-          double coordGVSelf[3];
-          double coordGVOther[3];
-          double dx,dy;
+          double de; //dx,dy;
           vClassDim=V_whatInType(vrts[i]);
           vConG=V_whatIn(vrts[i]);
-          double parFace[2]; 
-          double pLow, pHigh;
           if(vClassDim == 0) {// classified on vert so vert->edge->vert
               foundESTag = GEN_tag( (pGVertex) vConG ); // found Extrusion Start Tag
               gEdges = GV_edges( (pGVertex) vConG ); // pPList of model edges adjacent to root model vertex
@@ -247,18 +276,24 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
                 if( gVert0 == (pGVertex) vConG) { //1 is at other end of edge
                    GV_point( gVert1 , coordGVOther );
                    GV_point( gVert0 , coordGVSelf );
-                   dx = coordGVOther[0] - coordGVSelf[0];
-                   dy = coordGVOther[1] - coordGVSelf[1];
-                   if( dx*dx + dy*dy < VdisTol ) {
+//                   dx = coordGVOther[0] - coordGVSelf[0];
+//                   dy = coordGVOther[1] - coordGVSelf[1];
+//                   if( dx*dx + dy*dy < VdisTol ) {
+                   for( int i = 0; i < 3; i++) sepVec[i]=coordGVOther[i]-coordGVSelf[i];  
+                   de=abs(sepVec[0]*normal[0]+sepVec[1]*normal[1]+sepVec[2]*normal[2]);
+                   if( abs(de-ExtruDistance) < VdisTol ) {
                       foundETag = GEN_tag( gEdge );
                       foundEETag = GEN_tag( gVert1 );
                    }
                 } else { // 0 is at the other edge
                    GV_point( gVert0 , coordGVOther );
                    GV_point( gVert1 , coordGVSelf );
-                   dx = coordGVOther[0] - coordGVSelf[0];
-                   dy = coordGVOther[1] - coordGVSelf[1];
-                   if( dx*dx + dy*dy < VdisTol) {
+//                   dx = coordGVOther[0] - coordGVSelf[0];
+//                   dy = coordGVOther[1] - coordGVSelf[1];
+//                   if( dx*dx + dy*dy < VdisTol) {
+                   for( int i = 0; i < 3; i++) sepVec[i]=coordGVOther[i]-coordGVSelf[i];  
+                   de=abs(sepVec[0]*normal[0]+sepVec[1]*normal[1]+sepVec[2]*normal[2]);
+                   if( abs(de-ExtruDistance) < VdisTol ) {
                       foundETag = GEN_tag(gEdge);
                       foundEETag = GEN_tag(gVert0);
                    }
@@ -280,9 +315,12 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
                     GE_parRange ( gEdge, &pLow, &pHigh);
                     parFace[0] = ( pLow + pHigh ) * 0.5;
                     GE_point( gEdge , parFace[0], coordGVOther ); 
-                    dx = coordGVOther[0] - coordGVSelf[0];
-                    dy = coordGVOther[1] - coordGVSelf[1];
-                    if( dx*dx + dy*dy < EdisTol) {
+//                    dx = coordGVOther[0] - coordGVSelf[0];
+//                    dy = coordGVOther[1] - coordGVSelf[1];
+//                    if( dx*dx + dy*dy < EdisTol) {
+                    for( int i = 0; i < 3; i++) sepVec[i]=coordGVOther[i]-coordGVSelf[i];  
+                    de=abs(sepVec[0]*normal[0]+sepVec[1]*normal[1]+sepVec[2]*normal[2]);
+                    if( abs(de-ExtruDistance) < VdisTol ) {
                        foundETag = GEN_tag( gFace ); // found Extruded Tag
                        foundEETag = GEN_tag( gEdge );   // found Extrusion End Tag
                     }
@@ -309,9 +347,12 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
                   GF_parRange ( gFace, 1, &pLow, &pHigh);
                   parFace[1] = ( pLow + pHigh ) * 0.5;
                   GF_point( (pGFace) gFace , parFace , coordGVOther );
-                  dx = coordGVOther[0] - coordGVSelf[0];
-                  dy = coordGVOther[1] - coordGVSelf[1];
-                  if( dx*dx + dy*dy < FdisTol) {
+//                  dx = coordGVOther[0] - coordGVSelf[0];
+//                  dy = coordGVOther[1] - coordGVSelf[1];
+//                  if( dx*dx + dy*dy < FdisTol) {
+                  for( int i = 0; i < 3; i++) sepVec[i]=coordGVOther[i]-coordGVSelf[i];  
+                  de=abs(sepVec[0]*normal[0]+sepVec[1]*normal[1]+sepVec[2]*normal[2]);
+                  if( abs(de-ExtruDistance) < VdisTol ) {
                     foundETag = GEN_tag( gRegion ); // found Extruded Tag
                     foundEETag = GEN_tag( gFace );   // found Extrusion End Tag
                   }
diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 2616707fe..fd8e8cf94 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -40,11 +40,18 @@
 //#define INTERIOR_REGION 0
 int INTERIOR_REGION=0; // initialized but will be checked from read input
 
-apf::ModelEntity* getMdlRgn(gmi_model* model) {
-  apf::ModelEntity* rgn = reinterpret_cast<apf::ModelEntity*>(
-      gmi_find(model, 3, INTERIOR_REGION));
-  PCU_ALWAYS_ASSERT(rgn);
-  return rgn;
+//Manifold single region apf::ModelEntity* getMdlRgn(gmi_model* model) {
+//Manifold single region   apf::ModelEntity* rgn = reinterpret_cast<apf::ModelEntity*>(
+//Manifold single region       gmi_find(model, 3, INTERIOR_REGION));
+//Manifold single region   PCU_ALWAYS_ASSERT(rgn);
+//Manifold single region   return rgn;
+//Manifold single region }
+
+
+apf::ModelEntity* getMdlRegion(apf::Mesh2* mesh, int tag) {
+  apf::ModelEntity* region = mesh->findModelEntity(3,tag);
+  PCU_ALWAYS_ASSERT(region);
+  return region;
 }
 
 apf::ModelEntity* getMdlFace(apf::Mesh2* mesh, int tag) {
@@ -65,6 +72,45 @@ apf::ModelEntity* getMdlVtx(apf::Mesh2* mesh, int tag) {
   return vertex;
 }
 
+int findRegionTag2Face(gmi_model* model, int* cAll, int nverts) {
+  int cOrdered[nverts];
+  int rtag, cmax,jmax;
+  int cLocal[nverts];
+  for(int i=0; i<nverts; i++) cLocal[i]=cAll[i];
+  for(int i=0; i<nverts; i++) {
+    cmax=-100;
+    for(int j=0; j<nverts; j++) { 
+      cmax=std::max(cmax,cLocal[j]);
+      if(cmax==cLocal[j]) jmax=j;
+    }
+    cOrdered[i]=cmax;
+    cLocal[jmax]=-100;
+  }   
+  gmi_ent* maxFace = gmi_find(model,2,cOrdered[0]-2000000);  
+  int inc=1;
+  int ifound=0;
+  while (inc<nverts && ifound==0) {
+    if(cOrdered[inc] < 2000000) {
+      ifound=-1; // won't happen for wedge but might for tet handle later assert(false); 
+    } else if ( cOrdered[inc]==cOrdered[0]) {  // same as max...keep looking
+      inc++;
+    } else {
+      ifound=1;
+      gmi_ent* minFace = gmi_find(model,2,cOrdered[inc]-2000000);  
+      gmi_set* RegionsAdjMax = gmi_adjacent(model,maxFace,3);
+      gmi_set* RegionsAdjMin = gmi_adjacent(model,minFace,3);
+      for (int i = 0; i < RegionsAdjMax->n; i++) {
+        for (int j = 0; j < RegionsAdjMin->n; j++) {
+          if(RegionsAdjMax->e[i]==RegionsAdjMin->e[j]){
+            rtag=gmi_tag(model,RegionsAdjMax->e[j]);
+            return rtag;
+          }
+        }
+      }
+    }
+  }
+}
+
 void setVtxClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtxClass) {
   (void)model;
   (void)mesh;
@@ -87,8 +133,9 @@ void setVtxClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtxC
 //                          2000000 to mdlConvert's dmg tag number for faces,
 //                          3000000 to mdlConvert's dmg tag number for regions
     if (c >= 3000000 ) {
-       INTERIOR_REGION=c-3000000;
-       mesh->setModelEntity(v,getMdlRgn(model));
+//       INTERIOR_REGION=c-3000000;
+//       mesh->setModelEntity(v,getMdlRgn(model));
+       mesh->setModelEntity(v,getMdlRegion(mesh,c-3000000));
        //cint++;
     } else if (c >= 2000000) {
        mesh->setModelEntity(v,getMdlFace(mesh,c-2000000));
@@ -118,7 +165,8 @@ void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh,apf::MeshTag* vtxC
   int k,ff;
   double distFromDebug1;
 //  apf::Vector3 xd1(-0.54864, 7.44015e-06, 0.0397148 );
-  apf::Vector3 xd1(-2.63644, -0.953687, 0.00762);
+//  apf::Vector3 xd1(-2.63644, -0.953687, 0.00762);
+  apf::Vector3 xd1(-0.0047625, -0.477012, 0.1);
   apf::Vector3 xd2(0.914478, 0.0145401, 0.04 );
   apf::Vector3 dx1;
   apf::Vector3 dx2;
@@ -129,7 +177,7 @@ void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh,apf::MeshTag* vtxC
     mesh->getAdjacent(e, 0, verts);
 
     int nverts = verts.size();
-    if(1==0) {
+    if(1==1) {
     Centroid=apf::getLinearCentroid(mesh,e);
     dx1=xd1-Centroid;
  //   dx2=xd2-Centroid;
@@ -148,11 +196,11 @@ void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh,apf::MeshTag* vtxC
       cmax=std::max(cmax,c);
       cmin=std::min(cmin,c);
     }
-    if(1==0 && distFromDebug1 < 1e-3) {
+    if(1==1 && distFromDebug1 < 1e-8) {
          fprintf(stderr, "%d %d %.15e %.15E %.15E \n", cmin, cmax, Centroid[0], Centroid[1], Centroid[2]);
          for (int i=0; i < nverts; i++) {
             mesh->getPoint(verts[i],0,tmp);
-//            fprintf(stderr, "%d %.15e %.15E %.15E \n", i , tmp[0], tmp[1], tmp[2]);
+            fprintf(stderr, "%d %.15e %.15E %.15E \n", i , tmp[0], tmp[1], tmp[2]);
          }
     }
     int dimMin=cmin/1000000;
@@ -160,15 +208,38 @@ void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh,apf::MeshTag* vtxC
     int tagMin=cmin-dimMin*1000000;
     int tagMax=cmax-dimMax*1000000;
     if (cmax >= 3000000) {
-       INTERIOR_REGION=cmax-3000000;
-       mesh->setModelEntity(e,getMdlRgn(model));
+//       INTERIOR_REGION=cmax-3000000;
+//       mesh->setModelEntity(e,getMdlRgn(model));
+       mesh->setModelEntity(e,getMdlRegion(mesh,cmax-3000000));
        //cint++;
     } else if (cmax >= 2000000) {  //max is a face
        if(cmin==cmax) { //min is same face -> correct face to cls
           mesh->setModelEntity(e,getMdlFace(mesh,cmax-2000000));
           //cface++;
        } else if (cmin >= 2000000) { // min is a DIFFERENT face -> interior
-          mesh->setModelEntity(e,getMdlRgn(model));
+          nverts=2;
+          int ctri[2]={cmax,cmin};
+          int rtag = findRegionTag2Face(model, ctri, nverts);
+          mesh->setModelEntity(e,getMdlRegion(mesh,rtag));
+//         assert(false);
+/*
+         gmi_ent* maxFace = gmi_find(model,2,cmax-2000000);  
+         gmi_ent* minFace = gmi_find(model,2,cmin-2000000);  
+         gmi_set* RegionsAdjMax = gmi_adjacent(model,maxFace,3);
+         gmi_set* RegionsAdjMin = gmi_adjacent(model,minFace,3);
+         for (int i = 0; i < RegionsAdjMax->n; i++) {
+           for (int j = 0; j < RegionsAdjMin->n; j++) {
+             if(RegionsAdjMax->e[i]==RegionsAdjMin->e[j]){
+               int rtag=gmi_tag(model,RegionsAdjMax->e[j]);
+               mesh->setModelEntity(e,getMdlRegion(mesh,rtag));
+             }
+           }
+         }
+//  HARD  I need to ask what region(s) each face is part of the closure/shell_of_faces of
+//   I then need to intersect those two region list to find the right region 
+//  NOT THIS EASY         mesh->setModelEntity(e,getMdlRegion(mesh,c-3000000));
+//          mesh->setModelEntity(e,getMdlRgn(model));
+*/
        } else { 
 //FAILS  ROLL OUR OWN         int res = gmi_is_in_closure_of(model,gmi_find(model,dimMin,tagMin), gmi_find(model,dimMax,tagMax));
           ff=-1;
@@ -189,7 +260,9 @@ void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh,apf::MeshTag* vtxC
           if( ff!=-1 ) { // is it in cls
             mesh->setModelEntity(e,getMdlFace(mesh,cmax-2000000));
           } else  // edge not in closure so interior
-          mesh->setModelEntity(e,getMdlRgn(model));
+            assert(false); // hoping we don't get here
+//            mesh->setModelEntity(e,getMdlRegion(mesh,c-3000000));
+//          mesh->setModelEntity(e,getMdlRgn(model));
        }
     } else if (cmax >= 1000000) { // max is an edge
        if (cmin == cmax)  // cls on same edge 
@@ -199,7 +272,7 @@ void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh,apf::MeshTag* vtxC
          gmi_set* minFaces = gmi_adjacent(model,gmi_find(model,1,tagMin),2);  
          for (int i = 0; i < maxFaces->n; i++) {
            for (int j = 0; j < minFaces->n; j++) {
-             if(minFaces->e[i]==maxFaces->e[j]){
+             if(minFaces->e[j]==maxFaces->e[i]){
                int fftag=gmi_tag(model,maxFaces->e[j]);
                mesh->setModelEntity(e,getMdlFace(mesh,fftag));
              }
@@ -208,12 +281,16 @@ void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh,apf::MeshTag* vtxC
        } else mesh->setModelEntity(e,getMdlEdge(mesh,cmax-1000000)); // min is vtx thus max is correct edge to classify
     
     } else if (cmax < 1000000) { // two model verts so this is a 1 elm in z mesh
-         gmi_set* maxEdges = gmi_adjacent(model,gmi_find(model,1,tagMax),1);  
-         gmi_set* minEdges = gmi_adjacent(model,gmi_find(model,1,tagMin),1);  
+         gmi_set* maxEdges = gmi_adjacent(model,gmi_find(model,0,tagMax),1);  
+         gmi_set* minEdges = gmi_adjacent(model,gmi_find(model,0,tagMin),1);  
+         int nMaxEdges=maxEdges->n;
+         int nMinEdges=minEdges->n;
          for (int i = 0; i < maxEdges->n; i++) {
+           int tEmax=gmi_tag(model,maxEdges->e[i]);
            for (int j = 0; j < minEdges->n; j++) {
-             if(minEdges->e[i]==maxEdges->e[j]){
-               int fftag=gmi_tag(model,maxEdges->e[j]);
+             int tEmin=gmi_tag(model,minEdges->e[j]);
+             if(minEdges->e[j]==maxEdges->e[i]){
+               int fftag=gmi_tag(model,maxEdges->e[i]);
                mesh->setModelEntity(e,getMdlEdge(mesh,fftag));
              }
            }
@@ -236,32 +313,24 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
   (void)vtxClass;
   apf::MeshIterator* it = mesh->begin(2);
   apf::MeshEntity* f;
-  int c;
+  int c,rtag;
   apf::Adjacent verts;
-
   double distFromDebug1;
-//  apf::Vector3 xd1(-0.54864, 7.44015e-06, 0.0397148 );
   apf::Vector3 xd1(-2.63644, -0.953687, 0.00762);
   apf::Vector3 xd2(0.914478, 0.0145401, 0.04 );
   apf::Vector3 dx1;
   apf::Vector3 dx2;
   apf::Vector3 tmp;
   apf::Vector3 Centroid;
-
   while( (f = mesh->iterate(it)) ) {
     mesh->getAdjacent(f, 0, verts);
     int nverts = verts.size();
     if(1==0) {
     Centroid=apf::getLinearCentroid(mesh,f);
     dx1=xd1-Centroid;
- //   dx2=xd2-Centroid;
     distFromDebug1=dx1[0]*dx1[0]
                   +dx1[1]*dx1[1]
                   +dx1[2]*dx1[2];
-/*    distFromDebug2=dx2[0]*dx2[0]
-                  +dx2[1]*dx2[1]
-                  +dx2[2]*dx2[2];
-*/
     }
     int cmin=100000000;
     int cmax=-100;
@@ -272,21 +341,20 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
       cmax=std::max(cmax,c);
       ctri[i]=c;
     }
-//    if(std::min(distFromDebug1,distFromDebug2) < 1e-12) {
     if(1==0 && distFromDebug1 < 1e-3) {
          fprintf(stderr, "%d %d %.15e %.15E %.15E \n", cmin, cmax, Centroid[0], Centroid[1], Centroid[2]);
          for (int i=0; i < nverts; i++) {
-            mesh->getPoint(verts[i],0,tmp);
-//            fprintf(stderr, "%d %.15e %.15E %.15E \n", i , tmp[0], tmp[1], tmp[2]);
+            mesh->getPoint(verts[i],0,tmp); //            fprintf(stderr, "%d %.15e %.15E %.15E \n", i , tmp[0], tmp[1], tmp[2]);
          }
     }
     if (cmax >= 3000000) { // at least one vertex is interior -> cls interior
        INTERIOR_REGION=cmax-3000000;
-       mesh->setModelEntity(f,getMdlRgn(model));
-       //cint++;
+       mesh->setModelEntity(f,getMdlRegion(mesh,cmax-3000000)); //       mesh->setModelEntity(f,getMdlRgn(model)); //cint++;
     } else if(cmin >= 2000000) { // all nodes on  model face(s?)
         if(cmax != cmin) { // all on faces but not all on same so classified on interior
-          mesh->setModelEntity(f,getMdlRgn(model));
+          rtag = findRegionTag2Face(model, ctri, nverts);
+          mesh->setModelEntity(f,getMdlRegion(mesh,rtag));
+//          assert(false); // should not happen with extrustions but will for tets //          mesh->setModelEntity(f,getMdlRegion(mesh,c-3000000)); //          mesh->setModelEntity(f,getMdlRgn(model));
         } else { // all on same face so classify on that one
           mesh->setModelEntity(f,getMdlFace(mesh,cmax-2000000));
         }
@@ -298,8 +366,7 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
       int faceFound=0;
       int ifaceS=0;
       while ( (gf=gmi_next(model,gi)) && faceFound != nverts ) {
-        faceFound=0;
-        i=0;
+        faceFound=0; i=0;
         while(i<nverts && faceFound==i){ // check all verts but one not found == not in face
           ff=-1;
           dimi=ctri[i]/1000000;
@@ -332,8 +399,47 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
         ifaceS++;
       }
       gmi_end(model,gi);
-      if(faceFound != nverts ) // none of the model face's closure held all verts classificaton  so interior
-        mesh->setModelEntity(f,getMdlRgn(model));
+      if(faceFound != nverts ) { // none of the model face's closure held all verts classificaton  so interior wedges can sit in corner with a face with 1 vertcex on each of two faces and other one on the edge that intersects the two faces (or a model vertex) so the above fails in this LITERAL conner case)
+        rtag = findRegionTag2Face(model, ctri, nverts);
+        mesh->setModelEntity(f,getMdlRegion(mesh,rtag));
+/*
+        int cOrdered[nverts];
+        int jmax;
+        apf::MeshEntity* rgn;
+        for(int i=0; i<nverts; i++) {
+          cmax=-100;
+          for(int j=0; j<nverts; j++) { 
+            cmax=std::max(cmax,ctri[j]);
+            if(cmax==ctri[j]) jmax=j;
+          }
+          cOrdered[i]=cmax;
+          ctri[jmax]=-100;
+        }   
+        gmi_ent* maxFace = gmi_find(model,2,cOrdered[0]-2000000);  
+        int inc=1;
+        int ifound=0;
+        while (inc<nverts && ifound==0) {
+          if(cOrdered[inc] < 2000000) {
+            ifound=-1; // won't happen for wedge but might for tet handle later assert(false); 
+          } else if ( cOrdered[inc]==cOrdered[0]) {  // same as max...keep looking
+               inc++;
+          } else {
+            ifound=1;
+            gmi_ent* minFace = gmi_find(model,2,cOrdered[inc]-2000000);  
+            gmi_set* RegionsAdjMax = gmi_adjacent(model,maxFace,3);
+            gmi_set* RegionsAdjMin = gmi_adjacent(model,minFace,3);
+            for (int i = 0; i < RegionsAdjMax->n; i++) {
+              for (int j = 0; j < RegionsAdjMin->n; j++) {
+                if(RegionsAdjMax->e[i]==RegionsAdjMin->e[j]){
+                  int rtag=gmi_tag(model,RegionsAdjMax->e[j]);
+                  mesh->setModelEntity(rgn,getMdlRegion(mesh,rtag));
+                }
+              }
+            }
+          }
+        }
+*/
+      }
     }
   }
   mesh->end(it);
@@ -342,20 +448,82 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
 /** \brief set the mesh region classification
   \details hacked to set the classification to the same geometric model region
 */
-void setRgnClassification(gmi_model* model, apf::Mesh2* mesh) {
-  apf::ModelEntity* mdlRgn = getMdlRgn(model);
+//manifold void setRgnClassification(gmi_model* model, apf::Mesh2* mesh) {
+void setRgnClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtxClass) {
+  (void)model;
+  (void)mesh;
+  (void)vtxClass;
+  int c;
+  apf::Adjacent verts;
+//manifold  apf::ModelEntity* mdlRgn = getMdlRgn(model);
   apf::MeshIterator* it = mesh->begin(3);
   apf::MeshEntity* rgn;
-  while( (rgn = mesh->iterate(it)) )
-    mesh->setModelEntity(rgn,mdlRgn);
+//  while( (rgn = mesh->iterate(it)) ) 
+//manifold    mesh->setModelEntity(rgn,mdlRgn);
+  while( (rgn = mesh->iterate(it)) ) {
+// plan    for this mesh region get all its vertices, find the max c from a loop over those vertices and call the next line
+    mesh->getAdjacent(rgn, 0, verts);
+    int nverts = verts.size();
+    int cmax=-100;
+    int jmax,rtag;
+    int cAll[nverts];
+    int cOrdered[nverts];
+    for(int i=0; i<nverts; i++) {
+      mesh->getIntTag(verts[i],vtxClass,&c);
+      cmax=std::max(cmax,c);
+      cAll[i]=c;
+    }
+    if(cmax > 3000000) {
+      mesh->setModelEntity(rgn,getMdlRegion(mesh,cmax-3000000));
+    } else {
+      rtag = findRegionTag2Face(model, cAll, nverts);
+      mesh->setModelEntity(rgn,getMdlRegion(mesh,rtag));
+     // wedges can sit in corner with 2 vertices on the model each of two faces and other two on the edge that intersects the two faces (or a model vertex) so the above fails in this LITERAL conner case)
+/*
+      for(int i=0; i<nverts; i++) {
+        cmax=-100;
+        for(int j=0; j<nverts; j++) { 
+          cmax=std::max(cmax,cAll[j]);
+          if(cmax==cAll[j]) jmax=j;
+        }
+        cOrdered[i]=cmax;
+        cAll[jmax]=-100;
+      }   
+      gmi_ent* maxFace = gmi_find(model,2,cOrdered[0]-2000000);  
+      int inc=1;
+      int ifound=0;
+      while (inc<8 && ifound==0) {
+        if(cOrdered[inc] < 2000000) {
+          ifound=-1; // there were not two unique faces (should not happen)
+          assert(false); 
+        } else if ( cOrdered[inc]==cOrdered[0]) {  // same as max...keep looking
+             inc++;
+        } else {
+          ifound=1;
+          gmi_ent* minFace = gmi_find(model,2,cOrdered[inc]-2000000);  
+          gmi_set* RegionsAdjMax = gmi_adjacent(model,maxFace,3);
+          gmi_set* RegionsAdjMin = gmi_adjacent(model,minFace,3);
+          for (int i = 0; i < RegionsAdjMax->n; i++) {
+            for (int j = 0; j < RegionsAdjMin->n; j++) {
+              if(RegionsAdjMax->e[i]==RegionsAdjMin->e[j]){
+                int rtag=gmi_tag(model,RegionsAdjMax->e[j]);
+                mesh->setModelEntity(rgn,getMdlRegion(mesh,rtag));
+              }
+            }
+          }
+        }
+      }
+*/
+    }
+  }
   mesh->end(it);
 }
 
 void setClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* t) {
-  setRgnClassification(model,mesh);
-  setFaceClassification(model,mesh,t);
-  setEdgeClassification(model,mesh,t);
   setVtxClassification(model,mesh,t); 
+  setEdgeClassification(model,mesh,t);
+  setFaceClassification(model,mesh,t);
+  setRgnClassification(model,mesh,t);
   mesh->acceptChanges();
 }
 

From ea7ca3659c979e97ebb009d5458134dd08495f81 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Sat, 28 Jan 2023 23:30:46 -0700
Subject: [PATCH 527/555] cleaned up  commented code and debug variables

---
 test/matchedNodeElmReader.cc | 172 +++--------------------------------
 1 file changed, 13 insertions(+), 159 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index fd8e8cf94..b5ab98139 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -38,7 +38,7 @@
 
 /* model entity ids */
 //#define INTERIOR_REGION 0
-int INTERIOR_REGION=0; // initialized but will be checked from read input
+//int INTERIOR_REGION=0; // initialized but will be checked from read input
 
 //Manifold single region apf::ModelEntity* getMdlRgn(gmi_model* model) {
 //Manifold single region   apf::ModelEntity* rgn = reinterpret_cast<apf::ModelEntity*>(
@@ -74,7 +74,8 @@ apf::ModelEntity* getMdlVtx(apf::Mesh2* mesh, int tag) {
 
 int findRegionTag2Face(gmi_model* model, int* cAll, int nverts) {
   int cOrdered[nverts];
-  int rtag, cmax,jmax;
+  int jmax=0;
+  int rtag, cmax;
   int cLocal[nverts];
   for(int i=0; i<nverts; i++) cLocal[i]=cAll[i];
   for(int i=0; i<nverts; i++) {
@@ -91,7 +92,8 @@ int findRegionTag2Face(gmi_model* model, int* cAll, int nverts) {
   int ifound=0;
   while (inc<nverts && ifound==0) {
     if(cOrdered[inc] < 2000000) {
-      ifound=-1; // won't happen for wedge but might for tet handle later assert(false); 
+      ifound=-1; // won't happen for wedge but might for tet handle later 
+      assert(false); 
     } else if ( cOrdered[inc]==cOrdered[0]) {  // same as max...keep looking
       inc++;
     } else {
@@ -109,6 +111,7 @@ int findRegionTag2Face(gmi_model* model, int* cAll, int nverts) {
       }
     }
   }
+  return 0;
 }
 
 void setVtxClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtxClass) {
@@ -117,38 +120,24 @@ void setVtxClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtxC
   (void)vtxClass;
   apf::MeshIterator* it = mesh->begin(0);
   apf::MeshEntity* v;
-  //apf::Vector3 vCoord;
   int c;
-  //int count=0,cint=0,cface=0,cedge=0,cvtx=0;
   while( (v = mesh->iterate(it)) ) {
-    //mesh->getPoint(v, 0, vCoord);
-    //std::cout<<"Coordinates: "<<vCoord[0]<<" "<<vCoord[1]<<" "<<vCoord[2]<<std::endl;
     mesh->getIntTag(v,vtxClass,&c);
-    //std::cout<<"Returned tag is c= "<<c<<std::endl;
-    //std::cout<<" "<<std::endl;
-    //count++;
 //
 // New approach:  mesher will add 0 to mdlConvert's dmg tag number for vertices,
 //                          1000000 to mdlConvert's dmg tag number for edges,
 //                          2000000 to mdlConvert's dmg tag number for faces,
 //                          3000000 to mdlConvert's dmg tag number for regions
     if (c >= 3000000 ) {
-//       INTERIOR_REGION=c-3000000;
-//       mesh->setModelEntity(v,getMdlRgn(model));
        mesh->setModelEntity(v,getMdlRegion(mesh,c-3000000));
-       //cint++;
     } else if (c >= 2000000) {
        mesh->setModelEntity(v,getMdlFace(mesh,c-2000000));
-       //cface++;
     } else if (c >= 1000000) {
        mesh->setModelEntity(v,getMdlEdge(mesh,c-1000000));
-       //cedge++;
     } else {
        mesh->setModelEntity(v,getMdlVtx(mesh,c));
-       //cvtx++;
     }
   }
-  //std::cout<<"count is "<<cvtx<<std::endl;
   mesh->end(it);
 }
 
@@ -158,36 +147,24 @@ void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh,apf::MeshTag* vtxC
   (void)vtxClass;
   apf::MeshIterator* it = mesh->begin(1);
   apf::MeshEntity* e;
-  //apf::Vector3 vCoord;
   int c;
-  //int count=0;
   apf::Adjacent verts;
   int k,ff;
   double distFromDebug1;
-//  apf::Vector3 xd1(-0.54864, 7.44015e-06, 0.0397148 );
-//  apf::Vector3 xd1(-2.63644, -0.953687, 0.00762);
   apf::Vector3 xd1(-0.0047625, -0.477012, 0.1);
-  apf::Vector3 xd2(0.914478, 0.0145401, 0.04 );
   apf::Vector3 dx1;
   apf::Vector3 dx2;
   apf::Vector3 tmp;
   apf::Vector3 Centroid;
   while( (e = mesh->iterate(it)) ) {
-    //std::cout<<"Edge number "<<count++<<" with nodes"<<std::endl;
     mesh->getAdjacent(e, 0, verts);
-
     int nverts = verts.size();
-    if(1==1) {
+    if(1==0) {
     Centroid=apf::getLinearCentroid(mesh,e);
     dx1=xd1-Centroid;
- //   dx2=xd2-Centroid;
     distFromDebug1=dx1[0]*dx1[0]
                   +dx1[1]*dx1[1]
                   +dx1[2]*dx1[2];
-/*    distFromDebug2=dx2[0]*dx2[0]
-                  +dx2[1]*dx2[1]
-                  +dx2[2]*dx2[2];
-*/
     }
     int cmax=-100;
     int cmin=100000000;
@@ -196,7 +173,7 @@ void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh,apf::MeshTag* vtxC
       cmax=std::max(cmax,c);
       cmin=std::min(cmin,c);
     }
-    if(1==1 && distFromDebug1 < 1e-8) {
+    if(1==0 && distFromDebug1 < 1e-8) {
          fprintf(stderr, "%d %d %.15e %.15E %.15E \n", cmin, cmax, Centroid[0], Centroid[1], Centroid[2]);
          for (int i=0; i < nverts; i++) {
             mesh->getPoint(verts[i],0,tmp);
@@ -208,38 +185,16 @@ void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh,apf::MeshTag* vtxC
     int tagMin=cmin-dimMin*1000000;
     int tagMax=cmax-dimMax*1000000;
     if (cmax >= 3000000) {
-//       INTERIOR_REGION=cmax-3000000;
-//       mesh->setModelEntity(e,getMdlRgn(model));
        mesh->setModelEntity(e,getMdlRegion(mesh,cmax-3000000));
-       //cint++;
     } else if (cmax >= 2000000) {  //max is a face
        if(cmin==cmax) { //min is same face -> correct face to cls
           mesh->setModelEntity(e,getMdlFace(mesh,cmax-2000000));
-          //cface++;
        } else if (cmin >= 2000000) { // min is a DIFFERENT face -> interior
           nverts=2;
           int ctri[2]={cmax,cmin};
           int rtag = findRegionTag2Face(model, ctri, nverts);
+          assert(rtag!=0);
           mesh->setModelEntity(e,getMdlRegion(mesh,rtag));
-//         assert(false);
-/*
-         gmi_ent* maxFace = gmi_find(model,2,cmax-2000000);  
-         gmi_ent* minFace = gmi_find(model,2,cmin-2000000);  
-         gmi_set* RegionsAdjMax = gmi_adjacent(model,maxFace,3);
-         gmi_set* RegionsAdjMin = gmi_adjacent(model,minFace,3);
-         for (int i = 0; i < RegionsAdjMax->n; i++) {
-           for (int j = 0; j < RegionsAdjMin->n; j++) {
-             if(RegionsAdjMax->e[i]==RegionsAdjMin->e[j]){
-               int rtag=gmi_tag(model,RegionsAdjMax->e[j]);
-               mesh->setModelEntity(e,getMdlRegion(mesh,rtag));
-             }
-           }
-         }
-//  HARD  I need to ask what region(s) each face is part of the closure/shell_of_faces of
-//   I then need to intersect those two region list to find the right region 
-//  NOT THIS EASY         mesh->setModelEntity(e,getMdlRegion(mesh,c-3000000));
-//          mesh->setModelEntity(e,getMdlRgn(model));
-*/
        } else { 
 //FAILS  ROLL OUR OWN         int res = gmi_is_in_closure_of(model,gmi_find(model,dimMin,tagMin), gmi_find(model,dimMax,tagMax));
           ff=-1;
@@ -261,8 +216,6 @@ void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh,apf::MeshTag* vtxC
             mesh->setModelEntity(e,getMdlFace(mesh,cmax-2000000));
           } else  // edge not in closure so interior
             assert(false); // hoping we don't get here
-//            mesh->setModelEntity(e,getMdlRegion(mesh,c-3000000));
-//          mesh->setModelEntity(e,getMdlRgn(model));
        }
     } else if (cmax >= 1000000) { // max is an edge
        if (cmin == cmax)  // cls on same edge 
@@ -283,12 +236,8 @@ void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh,apf::MeshTag* vtxC
     } else if (cmax < 1000000) { // two model verts so this is a 1 elm in z mesh
          gmi_set* maxEdges = gmi_adjacent(model,gmi_find(model,0,tagMax),1);  
          gmi_set* minEdges = gmi_adjacent(model,gmi_find(model,0,tagMin),1);  
-         int nMaxEdges=maxEdges->n;
-         int nMinEdges=minEdges->n;
          for (int i = 0; i < maxEdges->n; i++) {
-           int tEmax=gmi_tag(model,maxEdges->e[i]);
            for (int j = 0; j < minEdges->n; j++) {
-             int tEmin=gmi_tag(model,minEdges->e[j]);
              if(minEdges->e[j]==maxEdges->e[i]){
                int fftag=gmi_tag(model,maxEdges->e[i]);
                mesh->setModelEntity(e,getMdlEdge(mesh,fftag));
@@ -317,7 +266,6 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
   apf::Adjacent verts;
   double distFromDebug1;
   apf::Vector3 xd1(-2.63644, -0.953687, 0.00762);
-  apf::Vector3 xd2(0.914478, 0.0145401, 0.04 );
   apf::Vector3 dx1;
   apf::Vector3 dx2;
   apf::Vector3 tmp;
@@ -348,13 +296,12 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
          }
     }
     if (cmax >= 3000000) { // at least one vertex is interior -> cls interior
-       INTERIOR_REGION=cmax-3000000;
        mesh->setModelEntity(f,getMdlRegion(mesh,cmax-3000000)); //       mesh->setModelEntity(f,getMdlRgn(model)); //cint++;
     } else if(cmin >= 2000000) { // all nodes on  model face(s?)
         if(cmax != cmin) { // all on faces but not all on same so classified on interior
           rtag = findRegionTag2Face(model, ctri, nverts);
+          assert(rtag!=0);
           mesh->setModelEntity(f,getMdlRegion(mesh,rtag));
-//          assert(false); // should not happen with extrustions but will for tets //          mesh->setModelEntity(f,getMdlRegion(mesh,c-3000000)); //          mesh->setModelEntity(f,getMdlRgn(model));
         } else { // all on same face so classify on that one
           mesh->setModelEntity(f,getMdlFace(mesh,cmax-2000000));
         }
@@ -401,44 +348,8 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
       gmi_end(model,gi);
       if(faceFound != nverts ) { // none of the model face's closure held all verts classificaton  so interior wedges can sit in corner with a face with 1 vertcex on each of two faces and other one on the edge that intersects the two faces (or a model vertex) so the above fails in this LITERAL conner case)
         rtag = findRegionTag2Face(model, ctri, nverts);
+        assert(rtag!=0);
         mesh->setModelEntity(f,getMdlRegion(mesh,rtag));
-/*
-        int cOrdered[nverts];
-        int jmax;
-        apf::MeshEntity* rgn;
-        for(int i=0; i<nverts; i++) {
-          cmax=-100;
-          for(int j=0; j<nverts; j++) { 
-            cmax=std::max(cmax,ctri[j]);
-            if(cmax==ctri[j]) jmax=j;
-          }
-          cOrdered[i]=cmax;
-          ctri[jmax]=-100;
-        }   
-        gmi_ent* maxFace = gmi_find(model,2,cOrdered[0]-2000000);  
-        int inc=1;
-        int ifound=0;
-        while (inc<nverts && ifound==0) {
-          if(cOrdered[inc] < 2000000) {
-            ifound=-1; // won't happen for wedge but might for tet handle later assert(false); 
-          } else if ( cOrdered[inc]==cOrdered[0]) {  // same as max...keep looking
-               inc++;
-          } else {
-            ifound=1;
-            gmi_ent* minFace = gmi_find(model,2,cOrdered[inc]-2000000);  
-            gmi_set* RegionsAdjMax = gmi_adjacent(model,maxFace,3);
-            gmi_set* RegionsAdjMin = gmi_adjacent(model,minFace,3);
-            for (int i = 0; i < RegionsAdjMax->n; i++) {
-              for (int j = 0; j < RegionsAdjMin->n; j++) {
-                if(RegionsAdjMax->e[i]==RegionsAdjMin->e[j]){
-                  int rtag=gmi_tag(model,RegionsAdjMax->e[j]);
-                  mesh->setModelEntity(rgn,getMdlRegion(mesh,rtag));
-                }
-              }
-            }
-          }
-        }
-*/
       }
     }
   }
@@ -455,19 +366,13 @@ void setRgnClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtxC
   (void)vtxClass;
   int c;
   apf::Adjacent verts;
-//manifold  apf::ModelEntity* mdlRgn = getMdlRgn(model);
   apf::MeshIterator* it = mesh->begin(3);
   apf::MeshEntity* rgn;
-//  while( (rgn = mesh->iterate(it)) ) 
-//manifold    mesh->setModelEntity(rgn,mdlRgn);
   while( (rgn = mesh->iterate(it)) ) {
-// plan    for this mesh region get all its vertices, find the max c from a loop over those vertices and call the next line
     mesh->getAdjacent(rgn, 0, verts);
     int nverts = verts.size();
     int cmax=-100;
-    int jmax,rtag;
     int cAll[nverts];
-    int cOrdered[nverts];
     for(int i=0; i<nverts; i++) {
       mesh->getIntTag(verts[i],vtxClass,&c);
       cmax=std::max(cmax,c);
@@ -476,44 +381,9 @@ void setRgnClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtxC
     if(cmax > 3000000) {
       mesh->setModelEntity(rgn,getMdlRegion(mesh,cmax-3000000));
     } else {
-      rtag = findRegionTag2Face(model, cAll, nverts);
+      int rtag = findRegionTag2Face(model, cAll, nverts);
+      assert(rtag!=0);
       mesh->setModelEntity(rgn,getMdlRegion(mesh,rtag));
-     // wedges can sit in corner with 2 vertices on the model each of two faces and other two on the edge that intersects the two faces (or a model vertex) so the above fails in this LITERAL conner case)
-/*
-      for(int i=0; i<nverts; i++) {
-        cmax=-100;
-        for(int j=0; j<nverts; j++) { 
-          cmax=std::max(cmax,cAll[j]);
-          if(cmax==cAll[j]) jmax=j;
-        }
-        cOrdered[i]=cmax;
-        cAll[jmax]=-100;
-      }   
-      gmi_ent* maxFace = gmi_find(model,2,cOrdered[0]-2000000);  
-      int inc=1;
-      int ifound=0;
-      while (inc<8 && ifound==0) {
-        if(cOrdered[inc] < 2000000) {
-          ifound=-1; // there were not two unique faces (should not happen)
-          assert(false); 
-        } else if ( cOrdered[inc]==cOrdered[0]) {  // same as max...keep looking
-             inc++;
-        } else {
-          ifound=1;
-          gmi_ent* minFace = gmi_find(model,2,cOrdered[inc]-2000000);  
-          gmi_set* RegionsAdjMax = gmi_adjacent(model,maxFace,3);
-          gmi_set* RegionsAdjMin = gmi_adjacent(model,minFace,3);
-          for (int i = 0; i < RegionsAdjMax->n; i++) {
-            for (int j = 0; j < RegionsAdjMin->n; j++) {
-              if(RegionsAdjMax->e[i]==RegionsAdjMin->e[j]){
-                int rtag=gmi_tag(model,RegionsAdjMax->e[j]);
-                mesh->setModelEntity(rgn,getMdlRegion(mesh,rtag));
-              }
-            }
-          }
-        }
-      }
-*/
     }
   }
   mesh->end(it);
@@ -594,15 +464,10 @@ void readClassification(FILE* f, int localNumVtx, int** classification) {
   *classification = new int[localNumVtx];
   rewind(f);
   int mdlId;
-  int maxmdlId=0;
   for(int i=0; i<localNumVtx; i++) {
     gmi_fscanf(f, 1, "%d",  &mdlId);
     (*classification)[i] = mdlId;
-    maxmdlId=std::max(maxmdlId,mdlId); 
   }
-  printf("maxmdlId,INTERIOR_REGION=%d %d\n",maxmdlId,INTERIOR_REGION);
-  INTERIOR_REGION=maxmdlId-3000000; // correct if any vtx in region but will be negative if maxmdlId is on a face which is the case for 1 element in z... fix after reading model
-  printf("maxmdlId,INTERIOR_REGION=%d %d\n",maxmdlId,INTERIOR_REGION);
 }
 
 void readFathers(FILE* f, int localNumVtx, int** fathers) {
@@ -891,17 +756,6 @@ int main(int argc, char** argv)
   //gmi_model* model = gmi_load(".null");
   gmi_model* model = gmi_load("outModelr.dmg");
 //  gmi_model* model = apf::makeMdsBox(2,2,2,1,1,1,0);
- //  1 element in z has no vertices to tell us the region ID so we have to find it from the tag number of the region that is adjacent to a face whose tag number will be the max when there is no region.  We know we have this problem when INTERIOR_REGION is negative
-  if(INTERIOR_REGION < 0) {
-    int maxmdlId=0;
-    printf("maxmdlId,INTERIOR_REGION=%d %d\n",maxmdlId,INTERIOR_REGION);
-    maxmdlId=1000000+INTERIOR_REGION;  // get back to maxmdlId found 
-    printf("maxmdlId,INTERIOR_REGION=%d %d\n",maxmdlId,INTERIOR_REGION);
-    gmi_set* Rgns=gmi_adjacent(model,gmi_find(model,2,maxmdlId),3);
-    for(int i=0; i<((Rgns->n)); i++) 
-      INTERIOR_REGION=gmi_tag(model,Rgns->e[i]);
-  }
-  printf("INTERIOR_REGION=%d\n",INTERIOR_REGION);
   apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, m.dim, isMatched);
   apf::GlobalToVert outMap;
   PCU_Debug_Open();

From 6a3cb3295260f13fe15f310073d127d0c8c3192a Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Sat, 28 Jan 2023 23:34:31 -0700
Subject: [PATCH 528/555] cleaned up convert

---
 test/convert.cc | 22 +---------------------
 1 file changed, 1 insertion(+), 21 deletions(-)

diff --git a/test/convert.cc b/test/convert.cc
index b35b37919..904c0797a 100644
--- a/test/convert.cc
+++ b/test/convert.cc
@@ -184,8 +184,6 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
   FILE* fid = fopen(extrusionFaceFile, "r"); // helper file that contains all faces with extrusions
   assert(fid);
   double VdisTol=1e-12;
-  double EdisTol=1e-7;
-  double FdisTol=1e-7;
   while(1 == fscanf(fid,"%d",&ExtruRootId)) {
     pGFace ExtruRootFace=NULL;
     fprintf(stderr,"ExtruRootId= %d \n",ExtruRootId);
@@ -196,13 +194,7 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
       if(id==ExtruRootId) ExtruRootFace=gface;
     }
     assert(ExtruRootFace != NULL);
-    // notes for next version 
-    // we could get the region, get a list of vertices for the region, loop over those vertices to find the extrusion length
-    // then instead of checking matching x,y of opposite ends of the extrusion, abs(zroot - zOppositeRoot) matching the extrusion 
-    // length gives the match
-    // Note if we want to handle extrusions not purely in z, the normal to ExtruRootFace provides an axis to project a vector between     // the centers of the root and candidate oppositeRoot entity on to.
-
-    // all of the above assumes translation extrusion.  Rotation extrusion (sweeping extruded entiy over an arc of some angle about 
+    // all of the work so far assumes translation extrusion.  Rotation extrusion (sweeping extruded entiy over an arc of some angle about 
     // a given axis) is useful but this would require some code change.  The principle is the same.  Every root entity has another 
     // oppositeRoot entity whose position obeys a fixed angle rotation about a fixed axis.
     pPList gRegions,gFaces,gEdges,gVertices;
@@ -276,9 +268,6 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
                 if( gVert0 == (pGVertex) vConG) { //1 is at other end of edge
                    GV_point( gVert1 , coordGVOther );
                    GV_point( gVert0 , coordGVSelf );
-//                   dx = coordGVOther[0] - coordGVSelf[0];
-//                   dy = coordGVOther[1] - coordGVSelf[1];
-//                   if( dx*dx + dy*dy < VdisTol ) {
                    for( int i = 0; i < 3; i++) sepVec[i]=coordGVOther[i]-coordGVSelf[i];  
                    de=abs(sepVec[0]*normal[0]+sepVec[1]*normal[1]+sepVec[2]*normal[2]);
                    if( abs(de-ExtruDistance) < VdisTol ) {
@@ -288,9 +277,6 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
                 } else { // 0 is at the other edge
                    GV_point( gVert0 , coordGVOther );
                    GV_point( gVert1 , coordGVSelf );
-//                   dx = coordGVOther[0] - coordGVSelf[0];
-//                   dy = coordGVOther[1] - coordGVSelf[1];
-//                   if( dx*dx + dy*dy < VdisTol) {
                    for( int i = 0; i < 3; i++) sepVec[i]=coordGVOther[i]-coordGVSelf[i];  
                    de=abs(sepVec[0]*normal[0]+sepVec[1]*normal[1]+sepVec[2]*normal[2]);
                    if( abs(de-ExtruDistance) < VdisTol ) {
@@ -315,9 +301,6 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
                     GE_parRange ( gEdge, &pLow, &pHigh);
                     parFace[0] = ( pLow + pHigh ) * 0.5;
                     GE_point( gEdge , parFace[0], coordGVOther ); 
-//                    dx = coordGVOther[0] - coordGVSelf[0];
-//                    dy = coordGVOther[1] - coordGVSelf[1];
-//                    if( dx*dx + dy*dy < EdisTol) {
                     for( int i = 0; i < 3; i++) sepVec[i]=coordGVOther[i]-coordGVSelf[i];  
                     de=abs(sepVec[0]*normal[0]+sepVec[1]*normal[1]+sepVec[2]*normal[2]);
                     if( abs(de-ExtruDistance) < VdisTol ) {
@@ -347,9 +330,6 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c
                   GF_parRange ( gFace, 1, &pLow, &pHigh);
                   parFace[1] = ( pLow + pHigh ) * 0.5;
                   GF_point( (pGFace) gFace , parFace , coordGVOther );
-//                  dx = coordGVOther[0] - coordGVSelf[0];
-//                  dy = coordGVOther[1] - coordGVSelf[1];
-//                  if( dx*dx + dy*dy < FdisTol) {
                   for( int i = 0; i < 3; i++) sepVec[i]=coordGVOther[i]-coordGVSelf[i];  
                   de=abs(sepVec[0]*normal[0]+sepVec[1]*normal[1]+sepVec[2]*normal[2]);
                   if( abs(de-ExtruDistance) < VdisTol ) {

From 0fa8f87ad0082ffd5c4e1495312ca1c2fbec3bbd Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Sun, 29 Jan 2023 11:49:39 -0700
Subject: [PATCH 529/555] cleaned up convert

---
 test/matchedNodeElmReader.cc | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index b5ab98139..80199be70 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -92,8 +92,7 @@ int findRegionTag2Face(gmi_model* model, int* cAll, int nverts) {
   int ifound=0;
   while (inc<nverts && ifound==0) {
     if(cOrdered[inc] < 2000000) {
-      ifound=-1; // won't happen for wedge but might for tet handle later 
-      assert(false); 
+      return 0 //  there were not two distinct faces in the provided list
     } else if ( cOrdered[inc]==cOrdered[0]) {  // same as max...keep looking
       inc++;
     } else {
@@ -193,7 +192,7 @@ void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh,apf::MeshTag* vtxC
           nverts=2;
           int ctri[2]={cmax,cmin};
           int rtag = findRegionTag2Face(model, ctri, nverts);
-          assert(rtag!=0);
+          assert(rtag!=0); // bad input list of ctri (e.g., not two distinct faces)
           mesh->setModelEntity(e,getMdlRegion(mesh,rtag));
        } else { 
 //FAILS  ROLL OUR OWN         int res = gmi_is_in_closure_of(model,gmi_find(model,dimMin,tagMin), gmi_find(model,dimMax,tagMax));
@@ -300,7 +299,7 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
     } else if(cmin >= 2000000) { // all nodes on  model face(s?)
         if(cmax != cmin) { // all on faces but not all on same so classified on interior
           rtag = findRegionTag2Face(model, ctri, nverts);
-          assert(rtag!=0);
+          assert(rtag!=0); // bad input list of ctri (e.g., not two distinct faces)
           mesh->setModelEntity(f,getMdlRegion(mesh,rtag));
         } else { // all on same face so classify on that one
           mesh->setModelEntity(f,getMdlFace(mesh,cmax-2000000));
@@ -346,9 +345,9 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
         ifaceS++;
       }
       gmi_end(model,gi);
-      if(faceFound != nverts ) { // none of the model face's closure held all verts classificaton  so interior wedges can sit in corner with a face with 1 vertcex on each of two faces and other one on the edge that intersects the two faces (or a model vertex) so the above fails in this LITERAL conner case)
+      if(faceFound != nverts ) { // none of the model face's closure held all verts classificaton  so interior wedges can sit in corner with a face with 1 vertex on each of two faces and other one on the edge that intersects the two faces (or a model vertex) so the above fails in this LITERAL conner case)
         rtag = findRegionTag2Face(model, ctri, nverts);
-        assert(rtag!=0);
+        assert(rtag!=0); //bad input list of ctri (e.g., not two distinct faces)  POSSIBLE yet unseen/unhandled case
         mesh->setModelEntity(f,getMdlRegion(mesh,rtag));
       }
     }
@@ -382,7 +381,7 @@ void setRgnClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtxC
       mesh->setModelEntity(rgn,getMdlRegion(mesh,cmax-3000000));
     } else {
       int rtag = findRegionTag2Face(model, cAll, nverts);
-      assert(rtag!=0);
+      assert(rtag!=0); // bad input list of ctri (e.g., not two distinct faces)  POSSIBLE yet unseen/unhandled case
       mesh->setModelEntity(rgn,getMdlRegion(mesh,rtag));
     }
   }

From 81600e910ffa54ce59d7184a7ebce8ada80fcf86 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Sun, 29 Jan 2023 11:56:54 -0700
Subject: [PATCH 530/555] cleanup created a bug

---
 test/matchedNodeElmReader.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 80199be70..8211d6452 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -92,7 +92,7 @@ int findRegionTag2Face(gmi_model* model, int* cAll, int nverts) {
   int ifound=0;
   while (inc<nverts && ifound==0) {
     if(cOrdered[inc] < 2000000) {
-      return 0 //  there were not two distinct faces in the provided list
+      return 0; //  there were not two distinct faces in the provided list
     } else if ( cOrdered[inc]==cOrdered[0]) {  // same as max...keep looking
       inc++;
     } else {

From a147fe7f2f6b969087eb20b37ef41688e8074b31 Mon Sep 17 00:00:00 2001
From: "Kenneth E. Jansen" <Kenneth.Jansen@colorado.edu>
Date: Sun, 29 Jan 2023 22:01:20 -0700
Subject: [PATCH 531/555] These changes allowed the mixed wedge-tet BL (like
 the bump) tests to pass.  Note the bigger changes were in getting the bump
 version of MGEN to write in the new (modelDim)*1M+dmg_model_dim approach. 
 Some retooling was necessary since the box model has 0 values for tags of all
 4 dimensions so error flags had to move to -1.

---
 test/matchedNodeElmReader.cc | 34 +++++++++++++++++++++++-----------
 1 file changed, 23 insertions(+), 11 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 8211d6452..6e1869e59 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -110,7 +110,17 @@ int findRegionTag2Face(gmi_model* model, int* cAll, int nverts) {
       }
     }
   }
-  return 0;
+  return -1;
+}
+
+int findRegionTag1Face(gmi_model* model, int cmax) {
+  int rtag,dimE, cnd;
+  dimE=cmax/1000000;
+  cnd=cmax-dimE*1000000;
+  gmi_ent* maxE = gmi_find(model,dimE,cnd);  
+  gmi_set* RegionsAdjMax = gmi_adjacent(model,maxE,3);
+  rtag=gmi_tag(model,RegionsAdjMax->e[0]);
+  return rtag;
 }
 
 void setVtxClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtxClass) {
@@ -150,7 +160,7 @@ void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh,apf::MeshTag* vtxC
   apf::Adjacent verts;
   int k,ff;
   double distFromDebug1;
-  apf::Vector3 xd1(-0.0047625, -0.477012, 0.1);
+  apf::Vector3 xd1(0.000509276, 0, 0.0797419);
   apf::Vector3 dx1;
   apf::Vector3 dx2;
   apf::Vector3 tmp;
@@ -192,7 +202,7 @@ void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh,apf::MeshTag* vtxC
           nverts=2;
           int ctri[2]={cmax,cmin};
           int rtag = findRegionTag2Face(model, ctri, nverts);
-          assert(rtag!=0); // bad input list of ctri (e.g., not two distinct faces)
+          assert(rtag != -1); // bad input list of ctri (e.g., not two distinct faces)
           mesh->setModelEntity(e,getMdlRegion(mesh,rtag));
        } else { 
 //FAILS  ROLL OUR OWN         int res = gmi_is_in_closure_of(model,gmi_find(model,dimMin,tagMin), gmi_find(model,dimMax,tagMax));
@@ -213,8 +223,10 @@ void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh,apf::MeshTag* vtxC
           }  
           if( ff!=-1 ) { // is it in cls
             mesh->setModelEntity(e,getMdlFace(mesh,cmax-2000000));
-          } else  // edge not in closure so interior
-            assert(false); // hoping we don't get here
+          } else { // edge not in closure so interior
+            int rtag = findRegionTag1Face(model, cmax) ;
+            mesh->setModelEntity(e,getMdlRegion(mesh,rtag));
+          }
        }
     } else if (cmax >= 1000000) { // max is an edge
        if (cmin == cmax)  // cls on same edge 
@@ -264,7 +276,7 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
   int c,rtag;
   apf::Adjacent verts;
   double distFromDebug1;
-  apf::Vector3 xd1(-2.63644, -0.953687, 0.00762);
+  apf::Vector3 xd1(-0.597998, 0.41004, 0.08);
   apf::Vector3 dx1;
   apf::Vector3 dx2;
   apf::Vector3 tmp;
@@ -288,7 +300,7 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
       cmax=std::max(cmax,c);
       ctri[i]=c;
     }
-    if(1==0 && distFromDebug1 < 1e-3) {
+    if(0==1 && distFromDebug1 < 1e-11) {
          fprintf(stderr, "%d %d %.15e %.15E %.15E \n", cmin, cmax, Centroid[0], Centroid[1], Centroid[2]);
          for (int i=0; i < nverts; i++) {
             mesh->getPoint(verts[i],0,tmp); //            fprintf(stderr, "%d %.15e %.15E %.15E \n", i , tmp[0], tmp[1], tmp[2]);
@@ -299,7 +311,7 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
     } else if(cmin >= 2000000) { // all nodes on  model face(s?)
         if(cmax != cmin) { // all on faces but not all on same so classified on interior
           rtag = findRegionTag2Face(model, ctri, nverts);
-          assert(rtag!=0); // bad input list of ctri (e.g., not two distinct faces)
+          assert(rtag!=-1); // bad input list of ctri (e.g., not two distinct faces)
           mesh->setModelEntity(f,getMdlRegion(mesh,rtag));
         } else { // all on same face so classify on that one
           mesh->setModelEntity(f,getMdlFace(mesh,cmax-2000000));
@@ -347,7 +359,7 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
       gmi_end(model,gi);
       if(faceFound != nverts ) { // none of the model face's closure held all verts classificaton  so interior wedges can sit in corner with a face with 1 vertex on each of two faces and other one on the edge that intersects the two faces (or a model vertex) so the above fails in this LITERAL conner case)
         rtag = findRegionTag2Face(model, ctri, nverts);
-        assert(rtag!=0); //bad input list of ctri (e.g., not two distinct faces)  POSSIBLE yet unseen/unhandled case
+        assert(rtag!=-1); //bad input list of ctri (e.g., not two distinct faces)  POSSIBLE yet unseen/unhandled case
         mesh->setModelEntity(f,getMdlRegion(mesh,rtag));
       }
     }
@@ -377,11 +389,11 @@ void setRgnClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtxC
       cmax=std::max(cmax,c);
       cAll[i]=c;
     }
-    if(cmax > 3000000) {
+    if(cmax >= 3000000) {
       mesh->setModelEntity(rgn,getMdlRegion(mesh,cmax-3000000));
     } else {
       int rtag = findRegionTag2Face(model, cAll, nverts);
-      assert(rtag!=0); // bad input list of ctri (e.g., not two distinct faces)  POSSIBLE yet unseen/unhandled case
+      assert(rtag!=-1); // bad input list of ctri (e.g., not two distinct faces)  POSSIBLE yet unseen/unhandled case
       mesh->setModelEntity(rgn,getMdlRegion(mesh,rtag));
     }
   }

From 700941e757e42be934676beb6425b48263fce2c9 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 1 Feb 2023 13:49:04 -0500
Subject: [PATCH 532/555] update pumi-meshes

---
 pumi-meshes | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pumi-meshes b/pumi-meshes
index 73284454c..14e53b821 160000
--- a/pumi-meshes
+++ b/pumi-meshes
@@ -1 +1 @@
-Subproject commit 73284454c38572039dac887b9d4b8b792cb423df
+Subproject commit 14e53b82141b9fff475245e4e87824c17ab9b6a6

From ea1f25f87723adfa07173278fc48fc77553d781b Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 1 Feb 2023 14:25:56 -0500
Subject: [PATCH 533/555] update pumi-meshes

---
 pumi-meshes | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pumi-meshes b/pumi-meshes
index 14e53b821..4d1a3df43 160000
--- a/pumi-meshes
+++ b/pumi-meshes
@@ -1 +1 @@
-Subproject commit 14e53b82141b9fff475245e4e87824c17ab9b6a6
+Subproject commit 4d1a3df43eb9fcef65e2c8fef2a43e969d8fb7b0

From ef72e440ab9eb6280dd5916344603c57b81ab594 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 1 Feb 2023 14:30:56 -0500
Subject: [PATCH 534/555] gmi: uncomment call to reparam_path

not sure why this was commented out
---
 gmi/gmi_analytic.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gmi/gmi_analytic.c b/gmi/gmi_analytic.c
index 8290a2de5..e093778e3 100644
--- a/gmi/gmi_analytic.c
+++ b/gmi/gmi_analytic.c
@@ -126,7 +126,7 @@ static void reparam_path(struct gmi_analytic* m, struct agm_use* path,
     return;
   }
   reparam_across(m, *path, from_p, tmp);
-//  reparam_path(m, (path + 1), (pathlen - 1), tmp, to_p);
+  reparam_path(m, path + 1, pathlen - 1, tmp, to_p);
 }
 
 static void reparam(struct gmi_model* m, struct gmi_ent* from,

From 65f661aafc3f847ecf6b1a11fcb714dfc5d4ed86 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 1 Feb 2023 15:35:13 -0500
Subject: [PATCH 535/555] constructElements: remove multitopo logic

---
 apf/apfConstruct.cc | 23 +++--------------------
 1 file changed, 3 insertions(+), 20 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index 92e58ebe4..0827b261d 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -35,26 +35,9 @@ static void constructElements(
   for (int i = 0; i < nelem; ++i) {
     Downward verts;
     int offset = i * nev;
-    Gid vCur=conn[offset];
-    int uniqueVerts=1;
-    verts[0]=globalToVert[vCur];
-    Gid vPrev=vCur;
-    for (int j = 1; j < nev; ++j) {
-      vCur=conn[j+offset];
-      if (vCur != vPrev) { // multi-topology determined by repeating last used vtx
-         verts[j] = globalToVert[vCur]; 
-         uniqueVerts++;
-         vPrev=vCur; // for next check
-      } 
-    }
-
-    unsigned etypeL;
-    if(uniqueVerts==4) etypeL=apf::Mesh::TET;
-    else if(uniqueVerts==5) etypeL=apf::Mesh::PYRAMID;
-    else if(uniqueVerts==6) etypeL=apf::Mesh::PRISM;
-    else if(uniqueVerts==8) etypeL=apf::Mesh::HEX;
-    else PCU_ALWAYS_ASSERT(false);
-    buildElement(m, interior, etypeL, verts);
+    for (int j = 0; j < nev; ++j)
+      verts[j] = globalToVert[conn[j + offset]];
+    buildElement(m, interior, etype, verts);
   }
 }
 

From da146de29887126e4e1817a87575c02d2ae50b56 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 1 Feb 2023 16:21:29 -0500
Subject: [PATCH 536/555] mner: specify model on command line

---
 test/matchedNodeElmReader.cc | 26 +++++++-------------------
 1 file changed, 7 insertions(+), 19 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 6e1869e59..c211c03fe 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -732,9 +732,10 @@ int main(int argc, char** argv)
   PCU_Comm_Init();
   lion_set_verbosity(1);
   int noVerify=0;    // maintain default of verifying if not explicitly requesting it off
-  if( argc < 10 ) {
+  if( argc < 11 ) {
     if( !PCU_Comm_Self() ) {
-      printf("Usage: %s <ascii mesh connectivity .cnn> "
+      printf("Usage: %s <input dmg model> "
+          "<ascii mesh connectivity .cnn> "
           "<ascii vertex coordinates .crd> "
           "<ascii vertex matching flag .match> "
           "<ascii vertex classification flag .class> "
@@ -755,7 +756,7 @@ int main(int argc, char** argv)
 
   double t0 = PCU_Time();
   MeshInfo m;
-  readMesh(argv[1],argv[2],argv[3],argv[4],argv[5],argv[6],argv[7],m);
+  readMesh(argv[2],argv[3],argv[4],argv[5],argv[6],argv[7],argv[8],m);
 
   bool isMatched = true;
   if( !strcmp(argv[3], "NULL") )
@@ -764,9 +765,7 @@ int main(int argc, char** argv)
   if(!PCU_Comm_Self())
     fprintf(stderr, "isMatched %d\n", isMatched);
 
-  //gmi_model* model = gmi_load(".null");
-  gmi_model* model = gmi_load("outModelr.dmg");
-//  gmi_model* model = apf::makeMdsBox(2,2,2,1,1,1,0);
+  gmi_model* model = gmi_load(argv[1]);
   apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, m.dim, isMatched);
   apf::GlobalToVert outMap;
   PCU_Debug_Open();
@@ -786,17 +785,6 @@ int main(int argc, char** argv)
   }
   apf::MeshTag* tc = setMappedTag(mesh, "classification", m.classification, 1,
       m.localNumVerts, outMap);
-  { //debug - some verts don't have the classification tag...
-    apf::MeshEntity* v;
-    apf::MeshIterator* verts = mesh->begin(0);
-    int i=0;
-    while ((v = mesh->iterate(verts))) {
-      if(!mesh->hasTag(v,tc)) {
-        PCU_Debug_Print("%d missing tag\n", i);
-      }
-      i++;
-    }
-  }
   setClassification(model,mesh,tc);
   apf::removeTagFromDimension(mesh, tc, 0);
   mesh->destroyTag(tc);
@@ -836,9 +824,9 @@ int main(int argc, char** argv)
   if(noVerify != 1) mesh->verify();
 
   outMap.clear();
-  gmi_write_dmg(model, argv[8]);
+  gmi_write_dmg(model, argv[9]);
   apf::writeVtkFiles("rendered",mesh);
-  mesh->writeNative(argv[9]);
+  mesh->writeNative(argv[10]);
   if(noVerify != 1) mesh->verify();
 
   mesh->destroyNative();

From 622d8ac7c3827e4da78fcc309bccaeacb60f580b Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 1 Feb 2023 16:27:33 -0500
Subject: [PATCH 537/555] mner: update tests

---
 pumi-meshes        |  2 +-
 test/testing.cmake | 25 +++++++++++++++++++------
 2 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/pumi-meshes b/pumi-meshes
index 4d1a3df43..b61b3f707 160000
--- a/pumi-meshes
+++ b/pumi-meshes
@@ -1 +1 @@
-Subproject commit 4d1a3df43eb9fcef65e2c8fef2a43e969d8fb7b0
+Subproject commit b61b3f707081e4b89fd72195f6a7972fea5288a7
diff --git a/test/testing.cmake b/test/testing.cmake
index 953e9cb6c..f5bf4e1a2 100644
--- a/test/testing.cmake
+++ b/test/testing.cmake
@@ -203,15 +203,28 @@ mpi_test(create_misSquare 1
   mis_test)
 
 set(MDIR ${MESHES}/matchedNodeElementReader)
+mpi_test(matchedNodeElementReader_p1 1
+  ./matchedNodeElmReader
+  "${MDIR}/model.dmg"
+  "${MDIR}/1part/geom3D.cnndt"
+  "${MDIR}/1part/geom3D.coord"
+  "${MDIR}/1part/geom3D.match"
+  "${MDIR}/1part/geom3D.class"
+  "${MDIR}/1part/geom3D.fathr"
+  "NULL"
+  "${MDIR}/1part/geom3DHead.cnn"
+  "geom.dmg" "geom.smb")
+
 mpi_test(matchedNodeElementReader_p4 4
   ./matchedNodeElmReader
-  "${MDIR}/geom3D.cnndt"
-  "${MDIR}/geom3D.coord"
-  "${MDIR}/geom3D.match"
-  "${MDIR}/geom3D.class"
-  "${MDIR}/geom3D.fathr"
+  "${MDIR}/model.dmg"
+  "${MDIR}/4part/geom3D.cnndt"
+  "${MDIR}/4part/geom3D.coord"
+  "${MDIR}/4part/geom3D.match"
+  "${MDIR}/4part/geom3D.class"
+  "${MDIR}/4part/geom3D.fathr"
   "NULL"
-  "${MDIR}/geom3DHead.cnn"
+  "${MDIR}/4part/geom3DHead.cnn"
   "geom.dmg" "geom.smb")
 
 set(MDIR ${MESHES}/gmsh)

From be4ad1eb7616ac57f7ffc69f1b617d8c101340b6 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Wed, 1 Feb 2023 16:28:22 -0500
Subject: [PATCH 538/555] mner: remove commented code

---
 test/matchedNodeElmReader.cc | 24 ++----------------------
 1 file changed, 2 insertions(+), 22 deletions(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index c211c03fe..283fb05de 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -768,7 +768,6 @@ int main(int argc, char** argv)
   gmi_model* model = gmi_load(argv[1]);
   apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, m.dim, isMatched);
   apf::GlobalToVert outMap;
-  PCU_Debug_Open();
   for( size_t i=0; i< m.elements.size(); i++) {
     apf::assemble(mesh, m.elements[i], m.numElms[i], m.elementType[i], outMap);
     delete [] m.elements[i];
@@ -789,36 +788,17 @@ int main(int argc, char** argv)
   apf::removeTagFromDimension(mesh, tc, 0);
   mesh->destroyTag(tc);
  
-  if( strcmp(argv[5], "NULL") ) {
-    apf::MeshTag* tf = setMappedTag(mesh, "fathers2D", m.fathers2D, 1,
-      m.localNumVerts, outMap);
-    (void) tf;
+  if( strcmp(argv[6], "NULL") ) {
+    setMappedTag(mesh, "fathers2D", m.fathers2D, 1, m.localNumVerts, outMap);
   } else if(!PCU_Comm_Self())
     fprintf(stderr, "fathers2D not requested \n");
 
-  //mesh->destroyTag(tf);
-
   if(0==1) {
   apf::MeshTag* ts = setMappedTag(mesh, "solution", m.solution, 5,
       m.localNumVerts, outMap);
   (void) ts;
   }
 
-
-  /* // Print the father2D tags
-  apf::MeshEntity* v;
-  apf::MeshIterator* it = mesh->begin(0);
-  apf::MeshTag* t = mesh->findTag("fathers2D");
-  if (t==NULL) {std::cout<<"Didn't find tag"<<std::endl;}
-  int tagNum; 
-  int count = 0;
-  while ((v = mesh->iterate(it))) { // loop over mesh vertices
-    mesh->getIntTag(v,t,&tagNum);
-    std::cout<<"Tag number "<<tagNum<<std::endl;
-    count++;
-  }
-  mesh->end(it);*/
-
   if(!PCU_Comm_Self())
     fprintf(stderr, "seconds to create mesh %.3f\n", PCU_Time()-t0);
   if(noVerify != 1) mesh->verify();

From 4eeeedc5ed96a23c44fdf0f7efa6832cb9fa3666 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 3 Feb 2023 10:06:37 -0500
Subject: [PATCH 539/555] construct: remove debug prints and barriers

---
 apf/apfConstruct.cc | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc
index 0827b261d..40d566640 100644
--- a/apf/apfConstruct.cc
+++ b/apf/apfConstruct.cc
@@ -65,9 +65,7 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
     }
     ifirst++;
   }
-  PCU_Barrier();
   Gid max = getMax(globalToVert);  // seems like we read this and know it already on every rank so why compute with global comm?
-  PCU_Barrier();
   ifirst=0;
   APF_ITERATE(GlobalToVert, globalToVert, it) {
     Gid gid = it->first;  
@@ -236,15 +234,6 @@ void setCoords(Mesh2* m, const double* coords, int nverts,
   tmpL=(to+1)*(long)quotient-start; 
   tmpInt=tmpL;
   int n = std::min(tmpInt, nverts);
-  if(n > 1000) {
-     Gid sizeToSend=n*3*sizeof(double);
-     lion_eprint(1, "setCoords int overflow of: self=%ld,mySize=%ld,total=%ld, n=%ld,to=%ld, quotient=%ld, remainder=%ld start=%ld, peers=%ld, sizeToSend=%ld, nverts=%u \n",self,mySize,total,n,to,quotient,remainder,start,peers,sizeToSend,nverts);
-//  Gid peersG = PCU_Comm_Peers();
-//    Gid quotientG = total / peersG;
-//  Gid remainderG = total % peersG;
-//     lion_eprint(1, "setCoords Gid0test: self=%d,mySize=%d,total=%ld, quotientG=%ld, remainderG=%ld,peers=%ld \n",self,mySize,total,quotientG,remainderG,peersG);
-}
-  PCU_Barrier();
 
   while (nverts > 0) {
     PCU_COMM_PACK(to, start);

From d1eb8bf5b357561296b6c42ea1a778bad6e66ba4 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 3 Feb 2023 10:15:33 -0500
Subject: [PATCH 540/555] mner: don't write the geometric model

---
 test/matchedNodeElmReader.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 283fb05de..ea3510e9d 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -804,7 +804,6 @@ int main(int argc, char** argv)
   if(noVerify != 1) mesh->verify();
 
   outMap.clear();
-  gmi_write_dmg(model, argv[9]);
   apf::writeVtkFiles("rendered",mesh);
   mesh->writeNative(argv[10]);
   if(noVerify != 1) mesh->verify();

From 8d9b08c723cf49ae806bdf318dc038b2ae8b0650 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 3 Feb 2023 10:21:17 -0500
Subject: [PATCH 541/555] mner: update mner test inputs

---
 pumi-meshes | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pumi-meshes b/pumi-meshes
index b61b3f707..b8973d0bd 160000
--- a/pumi-meshes
+++ b/pumi-meshes
@@ -1 +1 @@
-Subproject commit b61b3f707081e4b89fd72195f6a7972fea5288a7
+Subproject commit b8973d0bd907d73218a611f8b6f7efbe580acd09

From bb1171d4061ee835c09547f4098cc945a2e836ec Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 3 Feb 2023 11:05:47 -0500
Subject: [PATCH 542/555] mner: fix memory leak

---
 test/matchedNodeElmReader.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index ea3510e9d..d50fa5849 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -692,7 +692,7 @@ void readMesh(const char* meshfilename,
     sprintf(filename, "%s.%d",fathers2Dfilename,self);
     FILE* fff = fopen(filename, "r");
     PCU_ALWAYS_ASSERT(fff);
-    readFathers(fff, mesh.localNumVerts, &(mesh.fathers2D)); // note we re-use classification reader
+    readFathers(fff, mesh.localNumVerts, &(mesh.fathers2D));
     fclose(fff);
   }
 
@@ -784,12 +784,14 @@ int main(int argc, char** argv)
   }
   apf::MeshTag* tc = setMappedTag(mesh, "classification", m.classification, 1,
       m.localNumVerts, outMap);
+  delete [] m.classification;
   setClassification(model,mesh,tc);
   apf::removeTagFromDimension(mesh, tc, 0);
   mesh->destroyTag(tc);
  
   if( strcmp(argv[6], "NULL") ) {
     setMappedTag(mesh, "fathers2D", m.fathers2D, 1, m.localNumVerts, outMap);
+    delete [] m.fathers2D;
   } else if(!PCU_Comm_Self())
     fprintf(stderr, "fathers2D not requested \n");
 

From c57dc2b264114446884927073874c9869d6f6d1e Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 3 Feb 2023 11:55:20 -0500
Subject: [PATCH 543/555] mner: fix gmi ent set leaks

---
 test/matchedNodeElmReader.cc | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index d50fa5849..148c463d7 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -108,6 +108,8 @@ int findRegionTag2Face(gmi_model* model, int* cAll, int nverts) {
           }
         }
       }
+      free(RegionsAdjMax);
+      free(RegionsAdjMin);
     }
   }
   return -1;
@@ -120,6 +122,7 @@ int findRegionTag1Face(gmi_model* model, int cmax) {
   gmi_ent* maxE = gmi_find(model,dimE,cnd);  
   gmi_set* RegionsAdjMax = gmi_adjacent(model,maxE,3);
   rtag=gmi_tag(model,RegionsAdjMax->e[0]);
+  free(RegionsAdjMax);
   return rtag;
 }
 
@@ -218,9 +221,11 @@ void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh,apf::MeshTag* vtxC
               gmi_set* Verts = gmi_adjacent(model,Edges->e[k],0);
               for (int j = 0; j < Verts->n; j++)  
                 if(ge==Verts->e[j]) ff=j;
+              free(Verts);
             }
             k++;
           }  
+          free(Edges);
           if( ff!=-1 ) { // is it in cls
             mesh->setModelEntity(e,getMdlFace(mesh,cmax-2000000));
           } else { // edge not in closure so interior
@@ -242,6 +247,8 @@ void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh,apf::MeshTag* vtxC
              }
            }
          }
+         free(maxFaces);
+         free(minFaces);
        } else mesh->setModelEntity(e,getMdlEdge(mesh,cmax-1000000)); // min is vtx thus max is correct edge to classify
     
     } else if (cmax < 1000000) { // two model verts so this is a 1 elm in z mesh
@@ -255,7 +262,8 @@ void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh,apf::MeshTag* vtxC
              }
            }
          }
-      
+         free(maxEdges);
+         free(minEdges);
     } else { // should never get here since cmax < 10000000 is a vtx
             fprintf(stderr, "edge classification of these vert failed %d  %d \n", cmin, cmax);
     }
@@ -343,9 +351,11 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
                 gmi_set* Verts = gmi_adjacent(model,Edges->e[k],0);
                 for (int j = 0; j < Verts->n; j++)  
                    if(ge==Verts->e[j]) ff=j;
+                free(Verts);
               }
               k++;
             }  
+            free(Edges);
           }  
           if( ff!=-1 ) faceFound++;
           i++;

From 626eb698e736498775d98f68a2f6b884c7b6819d Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 3 Feb 2023 15:12:24 -0500
Subject: [PATCH 544/555] mner: one more leak

---
 test/matchedNodeElmReader.cc | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 148c463d7..bc5d63bf3 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -100,16 +100,21 @@ int findRegionTag2Face(gmi_model* model, int* cAll, int nverts) {
       gmi_ent* minFace = gmi_find(model,2,cOrdered[inc]-2000000);  
       gmi_set* RegionsAdjMax = gmi_adjacent(model,maxFace,3);
       gmi_set* RegionsAdjMin = gmi_adjacent(model,minFace,3);
+      bool found=false;
       for (int i = 0; i < RegionsAdjMax->n; i++) {
         for (int j = 0; j < RegionsAdjMin->n; j++) {
           if(RegionsAdjMax->e[i]==RegionsAdjMin->e[j]){
             rtag=gmi_tag(model,RegionsAdjMax->e[j]);
-            return rtag;
+            found=true;
+            break;
           }
         }
       }
       free(RegionsAdjMax);
       free(RegionsAdjMin);
+      if(found) {
+        return rtag;
+      }
     }
   }
   return -1;

From 283fce401c3fe98e4eb2cb85e7aee34cbe0a3d8e Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 3 Feb 2023 16:14:12 -0500
Subject: [PATCH 545/555] chef: avoid deprecated mallinfo functions

github ci builds were failing on this
---
 phasta/CMakeLists.txt |  1 +
 phasta/phCook.cc      | 28 ++++------------------------
 2 files changed, 5 insertions(+), 24 deletions(-)

diff --git a/phasta/CMakeLists.txt b/phasta/CMakeLists.txt
index 9ee43eb26..0a785e268 100644
--- a/phasta/CMakeLists.txt
+++ b/phasta/CMakeLists.txt
@@ -115,6 +115,7 @@ target_link_libraries(ph
       parma
       pcu
       apf_zoltan
+      pumi
     )
 if(ENABLE_SIMMETRIX)
   target_link_libraries(ph PUBLIC
diff --git a/phasta/phCook.cc b/phasta/phCook.cc
index f733ade58..50fe3ae01 100644
--- a/phasta/phCook.cc
+++ b/phasta/phCook.cc
@@ -30,7 +30,7 @@
 #include <stdlib.h>
 #include <cstring>
 #include <iostream>
-#include <malloc.h>
+#include <pumi.h>
 
 static void print_stats(const char* name, double value)
 {
@@ -44,26 +44,6 @@ static void print_stats(const char* name, double value)
     printf("%s: min %f max %f avg %f imb %f\n", name, min, max, avg, imb);
 }
 
-#if defined(__linux__)
-
-static double get_chunks()
-{
-  struct mallinfo m = mallinfo();
-  return m.uordblks + m.hblkhd;
-}
-
-#else
-static double get_chunks()
-{
-  cheese
-  if(!PCU_Comm_Self())
-    printf("%s:%d: OS Not supported\n", __FILE__, __LINE__);
-  return(-1.0);
-}
-#endif
-
-
-
 #define SIZET(a) static_cast<size_t>(a)
 
 namespace {
@@ -185,16 +165,16 @@ namespace ph {
     {
       apf::MeshTag* order = NULL;
 
-      print_stats("malloc used before Bfs", get_chunks());
+      print_stats("malloc used before Bfs", pumi_getMem());
 
       if (in.isReorder && PCU_Comm_Peers() > 1)
         order = Parma_BfsReorder(m);
 
-      print_stats("malloc used before reorder", get_chunks());
+      print_stats("malloc used before reorder", pumi_getMem());
 
       apf::reorderMdsMesh(m,order);
 
-      print_stats("malloc used after reorder", get_chunks());
+      print_stats("malloc used after reorder", pumi_getMem());
 
     }
   }

From 2b6239573402d9c9d31befe957cc0d412f1f23c4 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 3 Feb 2023 21:11:54 -0500
Subject: [PATCH 546/555] silence clang15 unused-but-set-variable errors

---
 crv/crvAdapt.cc                               | 2 --
 crv/crvShapeHandler.cc                        | 3 ---
 crv/crvVtk.cc                                 | 2 --
 ma/maShape.cc                                 | 2 --
 parma/diffMC/maximalIndependentSet/misLuby.cc | 2 --
 phasta/phInterfaceCutter.cc                   | 2 --
 test/H1Shapes.cc                              | 4 ----
 test/L2Shapes.cc                              | 6 ------
 test/icesheet.cc                              | 3 ---
 test/nedelecShapes.cc                         | 4 ----
 10 files changed, 30 deletions(-)

diff --git a/crv/crvAdapt.cc b/crv/crvAdapt.cc
index e13ce5193..12d58abb1 100644
--- a/crv/crvAdapt.cc
+++ b/crv/crvAdapt.cc
@@ -168,14 +168,12 @@ static int fixInvalidElements(crv::Adapt* a)
             + crv::fixInvalidEdges(a);
   int originalCount = count;
   int prev_count;
-  int i = 0;
   do {
     if ( ! count)
       break;
     prev_count = count;
     count = crv::fixLargeBoundaryAngles(a)
           + crv::fixInvalidEdges(a);
-    ++i;
   } while(count < prev_count);
 
   crv::fixLargeBoundaryAngles(a);
diff --git a/crv/crvShapeHandler.cc b/crv/crvShapeHandler.cc
index 9ec487ec0..8040d8760 100644
--- a/crv/crvShapeHandler.cc
+++ b/crv/crvShapeHandler.cc
@@ -500,7 +500,6 @@ class BezierHandler : public ma::ShapeHandler
       int P = fs->getOrder();
       int n = fs->getEntityShape(apf::Mesh::EDGE)->countNodes();
 
-      int numNewElements = 0;
       int numNewTriangles = 0;
       int numMiddleEdges = 0; // upper bound
 
@@ -531,8 +530,6 @@ class BezierHandler : public ma::ShapeHandler
 
         if (newType == apf::Mesh::TRIANGLE)
           numNewTriangles++;
-        if (newType == apf::Mesh::TET)
-          numNewElements++;
         // zero new entities
         bool snap = isBoundaryEntity(mesh,newEntities[i]) && shouldSnap;
         if (newType != apf::Mesh::EDGE)
diff --git a/crv/crvVtk.cc b/crv/crvVtk.cc
index 5cdd0f4d9..dd72bafac 100644
--- a/crv/crvVtk.cc
+++ b/crv/crvVtk.cc
@@ -954,7 +954,6 @@ void writeCurvedWireFrame(apf::Mesh* m, int n, const char* prefix)
   apf::MeshEntity* ent;
   apf::MeshIterator* it;
   it = m->begin(1);
-  int count = 0;
   while( (ent = m->iterate(it)) )
   {
     if(!m->isOwned(ent)) continue;
@@ -989,7 +988,6 @@ void writeCurvedWireFrame(apf::Mesh* m, int n, const char* prefix)
       v[1] = vs[i+1];
       apf::buildElement(wireMesh, 0, apf::Mesh::EDGE, v);
     }
-    count++;
   }
   m->end(it);
   apf::printStats(wireMesh);
diff --git a/ma/maShape.cc b/ma/maShape.cc
index 54b8936a8..c3479a07c 100644
--- a/ma/maShape.cc
+++ b/ma/maShape.cc
@@ -141,12 +141,10 @@ void unMarkBadQuality(Adapt* a)
   Mesh* m = a->mesh;
   Iterator* it;
   Entity* e;
-  int count = 0;
   it = m->begin(m->getDimension());
   while ((e = m->iterate(it))) {
     if (getFlag(a, e, ma::BAD_QUALITY))
       clearFlag(a, e, ma::BAD_QUALITY);
-    count++;
   }
   m->end(it);
 }
diff --git a/parma/diffMC/maximalIndependentSet/misLuby.cc b/parma/diffMC/maximalIndependentSet/misLuby.cc
index 91162d23a..b2b2fbf5e 100644
--- a/parma/diffMC/maximalIndependentSet/misLuby.cc
+++ b/parma/diffMC/maximalIndependentSet/misLuby.cc
@@ -402,7 +402,6 @@ int mis(partInfo& part, bool randNumsPredefined,bool isNeighbors) {
   vector<int> rmtNodesToRemove;
 
   int isInMis = 0;
-  int loopCount = 0;
   int tag = 0;
   int numNodesAdded;
   do {
@@ -461,7 +460,6 @@ int mis(partInfo& part, bool randNumsPredefined,bool isNeighbors) {
     rmtNodesToRemove.clear();
 
     numNodesAdded = PCU_Add_Int(numNodesAdded);
-    loopCount++;
   } while (numNodesAdded > 0);
 
   return isInMis;
diff --git a/phasta/phInterfaceCutter.cc b/phasta/phInterfaceCutter.cc
index 24a59bba9..90fe0d2dc 100644
--- a/phasta/phInterfaceCutter.cc
+++ b/phasta/phInterfaceCutter.cc
@@ -246,7 +246,6 @@ int migrateInterface(apf::Mesh2*& m, ph::BCs& bcs) {
   apf::Migration* plan = new apf::Migration(m);
   apf::Parts residence;
 
-  int nDG = 0;
   while ((f = m->iterate(it))) {
     apf::ModelEntity* me = m->toModel(f);
     if (m->getModelType(me) != faceDim)
@@ -256,7 +255,6 @@ int migrateInterface(apf::Mesh2*& m, ph::BCs& bcs) {
     if (!ph::isInterface(m->getModel(), gf, fbcs))
       continue;
 
-    ++nDG;
     apf::DgCopies dgCopies;
     m->getDgCopies(f, dgCopies);
 
diff --git a/test/H1Shapes.cc b/test/H1Shapes.cc
index 0ac0e8c9e..d69b978f1 100644
--- a/test/H1Shapes.cc
+++ b/test/H1Shapes.cc
@@ -148,7 +148,6 @@ void testH1(
       if(0==PCU_Comm_Self())
         lion_oprint(1, "computing dofs for dimension %d\n", d);
     it = m->begin(d);
-    int count = 0;
     while( (ent = m->iterate(it)) ) {
       int type = m->getType(ent);
       int non = h1Field->getShape()->countNodesOn(type);
@@ -170,7 +169,6 @@ void testH1(
         }
       }
       apf::destroyMeshElement(me);
-      count++;
     }
     m->end(it);
   }
@@ -181,7 +179,6 @@ void testH1(
 
     double L2Error = 0.;
     it = m->begin(d);
-    int count = 0;
     while( (ent = m->iterate(it)) ) {
       apf::MeshElement* me = apf::createMeshElement(m, ent);
       apf::Vector3 x;
@@ -209,7 +206,6 @@ void testH1(
       L2Error += err;
       apf::destroyElement(el);
       apf::destroyMeshElement(me);
-      count++;
     }
     m->end(it);
 
diff --git a/test/L2Shapes.cc b/test/L2Shapes.cc
index f8554b1d8..a2db1e071 100644
--- a/test/L2Shapes.cc
+++ b/test/L2Shapes.cc
@@ -104,7 +104,6 @@ void testL2(
       if(0==PCU_Comm_Self())
         lion_oprint(1, "computing dofs for dimension %d\n", d);
     it = m->begin(d);
-    int count = 0;
     while( (ent = m->iterate(it)) ) {
       int type = m->getType(ent);
       int non = l2Field->getShape()->countNodesOn(type);
@@ -119,7 +118,6 @@ void testL2(
       	apf::setVector(l2Field, ent, i, value);
       }
       apf::destroyMeshElement(me);
-      count++;
     }
     m->end(it);
   }
@@ -128,7 +126,6 @@ void testL2(
   // Verify that interpolated solution field agrees with exact field.
   double L2ErrorE = 0.;
   it = m->begin(dim);
-  int count = 0;
   while( (ent = m->iterate(it)) ) {
     apf::MeshElement* me = apf::createMeshElement(m, ent);
     apf::Vector3 x;
@@ -146,7 +143,6 @@ void testL2(
     L2ErrorE += err;
     apf::destroyMeshElement(me);
     apf::destroyElement(el);
-    count++;
   }
   m->end(it);
 
@@ -181,7 +177,6 @@ void testL2writeNative(
       if(0==PCU_Comm_Self())
         lion_oprint(1, "computing dofs for dimension %d\n", d);
     it = m->begin(d);
-    int count = 0;
     while( (ent = m->iterate(it)) ) {
       int type = m->getType(ent);
       int non = l2Field->getShape()->countNodesOn(type);
@@ -195,7 +190,6 @@ void testL2writeNative(
         apf::setVector(l2Field, ent, i, value);
       }
       apf::destroyMeshElement(me);
-      count++;
     }
     m->end(it);
   }
diff --git a/test/icesheet.cc b/test/icesheet.cc
index 4b7345c7f..ac2decfb7 100644
--- a/test/icesheet.cc
+++ b/test/icesheet.cc
@@ -180,7 +180,6 @@ int setModelClassification(gmi_model* model,
 
 void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtxType) {
   int numbdryfaces = 0;
-  int markedfaces = 0;
   int skippedfaces = 0;
   std::map<int,int> faceClass;
 
@@ -205,8 +204,6 @@ void setFaceClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtx
       counttaggedvtx += vtx_type_num[i];
     PCU_ALWAYS_ASSERT(counttaggedvtx==3);
     int isSet = setModelClassification(model, mesh, vtx_type_num, face, faceClass);
-    if( isSet == 1 )
-      markedfaces++;
     if(isSet == 0)
       skippedfaces++;
     if( isSet == 0 || isSet == 1 )
diff --git a/test/nedelecShapes.cc b/test/nedelecShapes.cc
index cb404f2c1..9575e948e 100644
--- a/test/nedelecShapes.cc
+++ b/test/nedelecShapes.cc
@@ -110,7 +110,6 @@ void testNedelec(
     else
       lion_oprint(1, "computing dofs for dimension %d\n", d);
     it = m->begin(d);
-    int count = 0;
     while( (ent = m->iterate(it)) ) {
       int type = m->getType(ent);
       int non = ndField->getShape()->countNodesOn(type);
@@ -137,7 +136,6 @@ void testNedelec(
       	apf::setScalar(ndField, ent, i, dof);
       }
       apf::destroyMeshElement(me);
-      count++;
     }
     m->end(it);
   }
@@ -147,7 +145,6 @@ void testNedelec(
   double L2ErrorE = 0.;
   double L2ErrorCurlE = 0.;
   it = m->begin(3);
-  int count = 0;
   while( (ent = m->iterate(it)) ) {
     apf::MeshElement* me = apf::createMeshElement(m, ent);
     apf::Vector3 x;
@@ -168,7 +165,6 @@ void testNedelec(
     L2ErrorCurlE += eCurlValue * eCurlValue;
     apf::destroyMeshElement(me);
     apf::destroyElement(el);
-    count++;
   }
   m->end(it);
 

From 88989b6bf5556879ca2b9074cda09cca1febd7c2 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 3 Feb 2023 21:25:50 -0500
Subject: [PATCH 547/555] clang: check for mallinfo2

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3bcb909a7..8472a9bc3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -152,7 +152,7 @@ target_compile_features(core INTERFACE cxx_std_11)
 scorec_export_library(core)
 
 #check for mallinfo2
-if(${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU")
+if(${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU" OR ${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang")
   include(CheckCXXSymbolExists)
   check_cxx_symbol_exists(mallinfo2 "malloc.h" PUMI_HAS_MALLINFO2)
   if(PUMI_HAS_MALLINFO2)

From a3ea41098d267550889233c0ec1c4edd03eff3f1 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Fri, 3 Feb 2023 21:39:07 -0500
Subject: [PATCH 548/555] mner: always assert

fixes ci release build error
---
 test/matchedNodeElmReader.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index bc5d63bf3..185e540fb 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -614,7 +614,7 @@ std::vector<BlockInfo> readHeader(std::ifstream& fh) {
   rewindStream(fh);
   const int self = PCU_Comm_Self();;
   bool ret = seekPart(fh, std::to_string(self));
-  assert(ret);
+  PCU_ALWAYS_ASSERT(ret);
   auto blockInfo = readTopoBlockInfo(fh);
   assert(blockInfo.size()>0);
   for(auto b : blockInfo) {

From d683f10b377076fc4fd3693049a9d09a9e424e33 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Sat, 4 Feb 2023 13:29:25 -0500
Subject: [PATCH 549/555] pumi: move mem usage to pcu

#382
---
 pcu/CMakeLists.txt |  1 +
 pcu/PCU.h          |  3 +++
 phasta/phCook.cc   |  7 +++----
 pumi/pumi_sys.cc   | 46 +---------------------------------------------
 4 files changed, 8 insertions(+), 49 deletions(-)

diff --git a/pcu/CMakeLists.txt b/pcu/CMakeLists.txt
index cb4431963..91649e43f 100644
--- a/pcu/CMakeLists.txt
+++ b/pcu/CMakeLists.txt
@@ -14,6 +14,7 @@ set(SOURCES
   pcu_coll.c
   pcu_io.c
   pcu_buffer.c
+  pcu_mem.c
   pcu_mpi.c
   pcu_msg.c
   pcu_order.c
diff --git a/pcu/PCU.h b/pcu/PCU.h
index daecc2634..f9a53362e 100644
--- a/pcu/PCU.h
+++ b/pcu/PCU.h
@@ -128,6 +128,9 @@ void PCU_Protect(void);
 /*MPI_Wtime() equivalent*/
 double PCU_Time(void);
 
+/*Memory usage*/
+double PCU_GetMem(void);
+
 #ifdef __cplusplus
 } /* extern "C" */
 #endif
diff --git a/phasta/phCook.cc b/phasta/phCook.cc
index 50fe3ae01..264fb8c5d 100644
--- a/phasta/phCook.cc
+++ b/phasta/phCook.cc
@@ -30,7 +30,6 @@
 #include <stdlib.h>
 #include <cstring>
 #include <iostream>
-#include <pumi.h>
 
 static void print_stats(const char* name, double value)
 {
@@ -165,16 +164,16 @@ namespace ph {
     {
       apf::MeshTag* order = NULL;
 
-      print_stats("malloc used before Bfs", pumi_getMem());
+      print_stats("malloc used before Bfs", PCU_GetMem());
 
       if (in.isReorder && PCU_Comm_Peers() > 1)
         order = Parma_BfsReorder(m);
 
-      print_stats("malloc used before reorder", pumi_getMem());
+      print_stats("malloc used before reorder", PCU_GetMem());
 
       apf::reorderMdsMesh(m,order);
 
-      print_stats("malloc used after reorder", pumi_getMem());
+      print_stats("malloc used after reorder", PCU_GetMem());
 
     }
   }
diff --git a/pumi/pumi_sys.cc b/pumi/pumi_sys.cc
index 897e8dcde..b41be4ff3 100644
--- a/pumi/pumi_sys.cc
+++ b/pumi/pumi_sys.cc
@@ -62,53 +62,9 @@ double pumi_getTime()
   return double(ruse_now.ru_utime.tv_sec) + double(ruse_now.ru_utime.tv_usec)/1000000.0;
 }
 
-#if defined(__APPLE__)
-
-#include <mach/task.h>
-#include <mach/mach_init.h>
-
-#elif defined(__bgq__)
-
-//the BG/Q headers have warning-triggering
-//code in them.
-#if defined(__GNUC__) && !defined(__clang__)
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wunused-parameter"
-#endif
-
-#include <spi/include/kernel/memory.h>
-
-#if defined(__GNUC__) && !defined(__clang__)
-#pragma GCC diagnostic pop
-#endif
-
-#else
-
-#include <malloc.h> //warning - this is GNU-specific
-
-#endif
-
 double pumi_getMem()
 {
-  const double M = 1024*1024;
-#if defined(__APPLE__)
-  bool resident = true;
-  struct task_basic_info t_info;
-  mach_msg_type_number_t t_info_count = TASK_BASIC_INFO_COUNT;
-  task_info(current_task(), TASK_BASIC_INFO, (task_info_t)&t_info, &t_info_count);
-  size_t size = (resident ? t_info.resident_size : t_info.virtual_size);
-  return (double)size/M;
-#elif defined(__bgq__)
-  size_t heap;
-  Kernel_GetMemorySize(KERNEL_MEMSIZE_HEAP, &heap);
-  return (double)heap/M;
-#elif defined(__GNUG__) && defined(PUMI_HAS_MALLINFO2)
-  struct mallinfo2 meminfo_now = mallinfo2();
-  return ((double)meminfo_now.arena)/M;
-#elif defined(__GNUG__)
-  struct mallinfo meminfo_now = mallinfo();
-  return ((double)meminfo_now.arena)/M;
-#endif
+  return PCU_GetMem();
 }
 
 void pumi_printTimeMem(const char* msg, double time, double memory)

From fc03e735cd5aa6922d86319815d871f2a866a72a Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Sat, 4 Feb 2023 13:37:06 -0500
Subject: [PATCH 550/555] add new source file

---
 pcu/pcu_mem.c | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)
 create mode 100644 pcu/pcu_mem.c

diff --git a/pcu/pcu_mem.c b/pcu/pcu_mem.c
new file mode 100644
index 000000000..8ac2345c1
--- /dev/null
+++ b/pcu/pcu_mem.c
@@ -0,0 +1,58 @@
+/****************************************************************************** 
+
+  (c) 2023 Scientific Computation Research Center, 
+      Rensselaer Polytechnic Institute. All rights reserved.
+  
+  This work is open source software, licensed under the terms of the
+  BSD license as described in the LICENSE file in the top-level directory.
+ 
+*******************************************************************************/
+#include <PCU.h>
+
+#if defined(__APPLE__)
+
+#include <mach/task.h>
+#include <mach/mach_init.h>
+
+#elif defined(__bgq__)
+
+//the BG/Q headers have warning-triggering
+//code in them.
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-parameter"
+#endif
+
+#include <spi/include/kernel/memory.h>
+
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic pop
+#endif
+
+#else
+
+#include <malloc.h> //warning - this is GNU-specific
+
+#endif
+
+double PCU_GetMem() {
+  const double M = 1024*1024;
+#if defined(__APPLE__)
+  bool resident = true;
+  struct task_basic_info t_info;
+  mach_msg_type_number_t t_info_count = TASK_BASIC_INFO_COUNT;
+  task_info(current_task(), TASK_BASIC_INFO, (task_info_t)&t_info, &t_info_count);
+  size_t size = (resident ? t_info.resident_size : t_info.virtual_size);
+  return (double)size/M;
+#elif defined(__bgq__)
+  size_t heap;
+  Kernel_GetMemorySize(KERNEL_MEMSIZE_HEAP, &heap);
+  return (double)heap/M;
+#elif defined(__GNUC__) && defined(PUMI_HAS_MALLINFO2)
+  struct mallinfo2 meminfo_now = mallinfo2();
+  return ((double)meminfo_now.arena)/M;
+#elif defined(__GNUC__)
+  struct mallinfo meminfo_now = mallinfo();
+  return ((double)meminfo_now.arena)/M;
+#endif
+}

From e9772ac2a3a27fe6f017ec019068f0913f4be6bd Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Sat, 4 Feb 2023 13:55:11 -0500
Subject: [PATCH 551/555] pcu needs HAS_MALLINFO2 flag

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8472a9bc3..686180c80 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -157,7 +157,7 @@ if(${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU" OR ${CMAKE_CXX_COMPILER_ID} STREQUAL
   check_cxx_symbol_exists(mallinfo2 "malloc.h" PUMI_HAS_MALLINFO2)
   if(PUMI_HAS_MALLINFO2)
     target_compile_definitions(core INTERFACE -DPUMI_HAS_MALLINFO2)
-    target_compile_definitions(pumi PRIVATE -DPUMI_HAS_MALLINFO2)
+    target_compile_definitions(pcu PRIVATE -DPUMI_HAS_MALLINFO2)
   endif()
 endif()
 

From 1af6e4288d872e0c3478b7b66c228b2a3c9ec54a Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Thu, 9 Feb 2023 23:18:56 -0500
Subject: [PATCH 552/555] don't set cxx11 if 14 is needed

---
 CMakeLists.txt | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index de0eae975..7648e54d3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -16,7 +16,9 @@ option(USE_XSDK_DEFAULTS "enable the XDSK v0.3.0 default configuration" NO)
 #requre c++11 without extensions
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSION OFF)
-set(CMAKE_CXX_STANDARD 11)
+if(NOT ENABLE_CGNS)
+  set(CMAKE_CXX_STANDARD 11)
+endif()
 
 xsdk_begin_package()
 bob_begin_package()
@@ -36,6 +38,7 @@ if(NOT USE_XSDK_DEFAULTS)
   bob_end_cxx_flags()
   set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS}")
   if(ENABLE_CGNS) #takes precedence over SCOREC_ENABLE_CXX11
+    message(STATUS "enabling cxx14")
     bob_cxx14_flags()
   elseif(SCOREC_ENABLE_CXX11)
     bob_cxx11_flags()

From 58c5035fb1880d320738fdcd075d610a80ce940b Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Thu, 9 Feb 2023 23:19:26 -0500
Subject: [PATCH 553/555] VLAs are not std c++

---
 mds/mdsGmsh.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/mds/mdsGmsh.cc b/mds/mdsGmsh.cc
index 7d4a26567..4d0ce7875 100644
--- a/mds/mdsGmsh.cc
+++ b/mds/mdsGmsh.cc
@@ -242,7 +242,7 @@ void readNodesV4(Reader* r)
   getLine(r); // because readNode gets the next line we need this outside  for Nodes_Block
   for (long i = 0; i < Num_EntityBlocks; ++i){
     sscanf(r->line, "%ld %ld %ld %ld", &edim, &etag, &junk3, &Nodes_Block);
-    long blockMap[Nodes_Block];
+    long* blockMap = new long[Nodes_Block];
     for (long j = 0; j < Nodes_Block; ++j){
       getLine(r);
       sscanf(r->line, "%ld", &blockMap[j]);
@@ -250,6 +250,7 @@ void readNodesV4(Reader* r)
     getLine(r);
     for (long j = 0; j < Nodes_Block; ++j)
       readNode(r,blockMap[j]);  // has a genLine at end
+    delete [] blockMap;
   }    
   checkMarker(r, "$EndNodes");
 }

From 9395df6ebb8eb4045b69eea0730cb73bd86c2243 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Thu, 9 Feb 2023 23:37:26 -0500
Subject: [PATCH 554/555] cgns: pass longs to assemble

---
 mds/mdsCGNS.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/mds/mdsCGNS.cc b/mds/mdsCGNS.cc
index 955e95031..acdef4aae 100644
--- a/mds/mdsCGNS.cc
+++ b/mds/mdsCGNS.cc
@@ -1161,7 +1161,8 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM
         if (std::get<1>(ret) > 0)
         {
           const std::vector<cgsize_t> vertexIDs = std::get<0>(ret);
-          localElements.emplace_back(apf::assemble(mesh, vertexIDs.data(), std::get<1>(ret), type, globalToVert)); // corresponding finalize below
+          std::vector<long> vertexIDs_l(vertexIDs.begin(), vertexIDs.end());
+          localElements.emplace_back(apf::assemble(mesh, vertexIDs_l.data(), std::get<1>(ret), type, globalToVert)); // corresponding finalize below
           const auto nverts = sizes[0];
           const auto ordinates = ReadCGNSCoords(cgid, base, zone, ncoords, nverts, vertexIDs, globalToVert);
 

From 0ad0a9d69f2f7ade391544f110285f00f8d35391 Mon Sep 17 00:00:00 2001
From: Cameron Smith <smithc11@rpi.edu>
Date: Thu, 9 Feb 2023 23:37:50 -0500
Subject: [PATCH 555/555] VLAs are not std c++

---
 ree/reeCorrectedFlux.cc      | 9 ++++++---
 ree/reeEstimateError.cc      | 3 ++-
 test/matchedNodeElmReader.cc | 8 ++++----
 3 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/ree/reeCorrectedFlux.cc b/ree/reeCorrectedFlux.cc
index 634ed5b79..c9abfe895 100644
--- a/ree/reeCorrectedFlux.cc
+++ b/ree/reeCorrectedFlux.cc
@@ -50,7 +50,7 @@ static void setupCorrectFlux(
 
   cf->correctedFlux = correctedFlux;
   int nc = apf::countComponents(cf->correctedFlux);
-  double zeros[nc];
+  double* zeros = new double[nc];
   for (int i = 0; i < nc; i++)
     zeros[i] = 0.;
   apf::MeshEntity* tet;
@@ -66,6 +66,7 @@ static void setupCorrectFlux(
     apf::destroyMeshElement(me);
   }
   cf->mesh->end(it);
+  delete [] zeros;
 }
 
 typedef std::vector<apf::MeshEntity*> EntityVector;
@@ -229,24 +230,26 @@ static void computeCorrectedFlux(FaceCavity* fc)
     apf::Vector3 theta_plus_tk1 = theta_vector1 + tk1;
 
     // get and set components in the auxiliary field
-    double comp1[nc];
+    double* comp1 = new double[nc];
     apf::getComponents(fc->correctflux->correctedFlux, firstTet, n, comp1);
     int id1 = tet1_pos * 3;
     comp1[id1] = theta_plus_tk1[0];
     comp1[id1+1] = theta_plus_tk1[1];
     comp1[id1+2] = theta_plus_tk1[2];
     apf::setComponents(fc->correctflux->correctedFlux, firstTet, n, comp1);
+    delete [] comp1;
 
     if (up.n == 2) {
       tk2 = tk * -1.;
       apf::Vector3 theta_plus_tk2 = theta_vector2 + tk2;
-      double comp2[nc];
+      double* comp2 = new double[nc];
       apf::getComponents(fc->correctflux->correctedFlux, secondTet, n, comp2);
       int id2 = tet2_pos * 3;
       comp2[id2] = theta_plus_tk2[0];
       comp2[id2+1] = theta_plus_tk2[1];
       comp2[id2+2] = theta_plus_tk2[2];
       apf::setComponents(fc->correctflux->correctedFlux, secondTet, n, comp2);
+      delete [] comp2;
     }
   }
   apf::destroyElement(fel);
diff --git a/ree/reeEstimateError.cc b/ree/reeEstimateError.cc
index 3c7ee200f..c3a4ab060 100644
--- a/ree/reeEstimateError.cc
+++ b/ree/reeEstimateError.cc
@@ -155,13 +155,14 @@ static void computeLambdaVector(
           fJ, apf::getDimension(mesh, face));
 
       // obtain corrected flux vector
-      double comp[nc];
+      double* comp = new double[nc];
       apf::getComponents(flux_field, e, n, comp);
       apf::Vector3 theta_plus_tk;
       int index = ii*3;
       theta_plus_tk[0] = comp[index];
       theta_plus_tk[1] = comp[index+1];
       theta_plus_tk[2] = comp[index+2];
+      delete [] comp;
 
       // compute p+1 order 3D vector shapes
       apf::NewArray<apf::Vector3> tetVectorShapes (nedofs);
diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc
index 185e540fb..4dfdfabd0 100644
--- a/test/matchedNodeElmReader.cc
+++ b/test/matchedNodeElmReader.cc
@@ -73,10 +73,10 @@ apf::ModelEntity* getMdlVtx(apf::Mesh2* mesh, int tag) {
 }
 
 int findRegionTag2Face(gmi_model* model, int* cAll, int nverts) {
-  int cOrdered[nverts];
+  std::vector<int> cOrdered(nverts);
   int jmax=0;
   int rtag, cmax;
-  int cLocal[nverts];
+  std::vector<int> cLocal(nverts);
   for(int i=0; i<nverts; i++) cLocal[i]=cAll[i];
   for(int i=0; i<nverts; i++) {
     cmax=-100;
@@ -398,7 +398,7 @@ void setRgnClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtxC
     mesh->getAdjacent(rgn, 0, verts);
     int nverts = verts.size();
     int cmax=-100;
-    int cAll[nverts];
+    std::vector<int> cAll(nverts);
     for(int i=0; i<nverts; i++) {
       mesh->getIntTag(verts[i],vtxClass,&c);
       cmax=std::max(cmax,c);
@@ -407,7 +407,7 @@ void setRgnClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtxC
     if(cmax >= 3000000) {
       mesh->setModelEntity(rgn,getMdlRegion(mesh,cmax-3000000));
     } else {
-      int rtag = findRegionTag2Face(model, cAll, nverts);
+      int rtag = findRegionTag2Face(model, cAll.data(), nverts);
       assert(rtag!=-1); // bad input list of ctri (e.g., not two distinct faces)  POSSIBLE yet unseen/unhandled case
       mesh->setModelEntity(rgn,getMdlRegion(mesh,rtag));
     }