Date: 2013-07-02T14:45:40+00:00

Actual source code: plex.c
petsc-3.4.2 2013-07-02

      1: #include <petsc-private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
      2: #include <../src/sys/utils/hash.h>
      3: #include <petsc-private/isimpl.h>
      4: #include <petscsf.h>

      6: /* Logging support */
      7: PetscLogEvent DMPLEX_Distribute, DMPLEX_Stratify;

      9: PETSC_EXTERN PetscErrorCode VecView_Seq(Vec, PetscViewer);
     10: PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);

     14: PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
     15: {
     16:   DM             dm;
     17:   PetscBool      isvtk;

     21:   VecGetDM(v, &dm);
     22:   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
     23:   PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK, &isvtk);
     24:   if (isvtk) {
     25:     PetscViewerVTKFieldType ft = PETSC_VTK_POINT_FIELD;
     26:     PetscSection            section;
     27:     PetscInt                dim, pStart, pEnd, cStart, fStart, vStart, cdof = 0, fdof = 0, vdof = 0;

     29:     DMPlexGetDimension(dm, &dim);
     30:     DMGetDefaultSection(dm, &section);
     31:     DMPlexGetHeightStratum(dm, 0, &cStart, NULL);
     32:     DMPlexGetHeightStratum(dm, 1, &fStart, NULL);
     33:     DMPlexGetDepthStratum(dm, 0, &vStart, NULL);
     34:     PetscSectionGetChart(section, &pStart, &pEnd);
     35:     /* Assumes that numer of dofs per point of each stratum is constant, natural for VTK */
     36:     if ((cStart >= pStart) && (cStart < pEnd)) {PetscSectionGetDof(section, cStart, &cdof);}
     37:     if ((fStart >= pStart) && (fStart < pEnd)) {PetscSectionGetDof(section, fStart, &fdof);}
     38:     if ((vStart >= pStart) && (vStart < pEnd)) {PetscSectionGetDof(section, vStart, &vdof);}
     39:     if (cdof && fdof && vdof) { /* Actually Q2 or some such, but visualize as Q1 */
     40:       ft = (cdof == dim) ? PETSC_VTK_POINT_VECTOR_FIELD : PETSC_VTK_POINT_FIELD;
     41:     } else if (cdof && vdof) {
     42:       SETERRQ(PetscObjectComm((PetscObject)viewer),PETSC_ERR_SUP,"No support for viewing mixed space with dofs at both vertices and cells");
     43:     } else if (cdof) {
     44:       /* TODO: This assumption should be removed when there is a way of identifying whether a space is conceptually a
     45:        * vector or just happens to have the same number of dofs as the dimension. */
     46:       if (cdof == dim) {
     47:         ft = PETSC_VTK_CELL_VECTOR_FIELD;
     48:       } else {
     49:         ft = PETSC_VTK_CELL_FIELD;
     50:       }
     51:     } else if (vdof) {
     52:       if (vdof == dim) {
     53:         ft = PETSC_VTK_POINT_VECTOR_FIELD;
     54:       } else {
     55:         ft = PETSC_VTK_POINT_FIELD;
     56:       }
     57:     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Could not classify input Vec for VTK");

     59:     PetscObjectReference((PetscObject) dm); /* viewer drops reference */
     60:     PetscObjectReference((PetscObject) v);  /* viewer drops reference */
     61:     PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, ft, (PetscObject) v);
     62:   } else {
     63:     PetscBool isseq;

     65:     PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);
     66:     if (isseq) {
     67:       VecView_Seq(v, viewer);
     68:     } else {
     69:       VecView_MPI(v, viewer);
     70:     }
     71:   }
     72:   return(0);
     73: }

     77: PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
     78: {
     79:   DM             dm;
     80:   PetscBool      isvtk;

     84:   VecGetDM(v, &dm);
     85:   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
     86:   PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK, &isvtk);
     87:   if (isvtk) {
     88:     Vec         locv;
     89:     const char *name;

     91:     DMGetLocalVector(dm, &locv);
     92:     PetscObjectGetName((PetscObject) v, &name);
     93:     PetscObjectSetName((PetscObject) locv, name);
     94:     DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);
     95:     DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);
     96:     VecView_Plex_Local(locv, viewer);
     97:     DMRestoreLocalVector(dm, &locv);
     98:   } else {
     99:     PetscBool isseq;

    101:     PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);
    102:     if (isseq) {
    103:       VecView_Seq(v, viewer);
    104:     } else {
    105:       VecView_MPI(v, viewer);
    106:     }
    107:   }
    108:   return(0);
    109: }

    113: PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
    114: {
    115:   DM_Plex          *mesh = (DM_Plex*) dm->data;
    116:   DM                cdm;
    117:   DMLabel           markers;
    118:   PetscSection      coordSection;
    119:   Vec               coordinates;
    120:   PetscViewerFormat format;
    121:   PetscErrorCode    ierr;

    124:   DMGetCoordinateDM(dm, &cdm);
    125:   DMGetDefaultSection(cdm, &coordSection);
    126:   DMGetCoordinatesLocal(dm, &coordinates);
    127:   PetscViewerGetFormat(viewer, &format);
    128:   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
    129:     const char *name;
    130:     PetscInt    maxConeSize, maxSupportSize;
    131:     PetscInt    pStart, pEnd, p;
    132:     PetscMPIInt rank, size;

    134:     MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);
    135:     MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);
    136:     PetscObjectGetName((PetscObject) dm, &name);
    137:     DMPlexGetChart(dm, &pStart, &pEnd);
    138:     DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);
    139:     PetscViewerASCIISynchronizedAllow(viewer, PETSC_TRUE);
    140:     PetscViewerASCIIPrintf(viewer, "Mesh '%s':\n", name);
    141:     PetscViewerASCIISynchronizedPrintf(viewer, "Max sizes cone: %D support: %D\n", maxConeSize, maxSupportSize);
    142:     PetscViewerASCIIPrintf(viewer, "orientation is missing\n", name);
    143:     PetscViewerASCIIPrintf(viewer, "cap --> base:\n", name);
    144:     for (p = pStart; p < pEnd; ++p) {
    145:       PetscInt dof, off, s;

    147:       PetscSectionGetDof(mesh->supportSection, p, &dof);
    148:       PetscSectionGetOffset(mesh->supportSection, p, &off);
    149:       for (s = off; s < off+dof; ++s) {
    150:         PetscViewerASCIISynchronizedPrintf(viewer, "[%D]: %D ----> %D\n", rank, p, mesh->supports[s]);
    151:       }
    152:     }
    153:     PetscViewerFlush(viewer);
    154:     PetscViewerASCIIPrintf(viewer, "base <-- cap:\n", name);
    155:     for (p = pStart; p < pEnd; ++p) {
    156:       PetscInt dof, off, c;

    158:       PetscSectionGetDof(mesh->coneSection, p, &dof);
    159:       PetscSectionGetOffset(mesh->coneSection, p, &off);
    160:       for (c = off; c < off+dof; ++c) {
    161:         PetscViewerASCIISynchronizedPrintf(viewer, "[%D]: %D <---- %D (%D)\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]);
    162:       }
    163:     }
    164:     PetscViewerFlush(viewer);
    165:     PetscSectionGetChart(coordSection, &pStart, NULL);
    166:     if (pStart >= 0) {PetscSectionVecView(coordSection, coordinates, viewer);}
    167:     DMPlexGetLabel(dm, "marker", &markers);
    168:     DMLabelView(markers,viewer);
    169:     if (size > 1) {
    170:       PetscSF sf;

    172:       DMGetPointSF(dm, &sf);
    173:       PetscSFView(sf, viewer);
    174:     }
    175:     PetscViewerFlush(viewer);
    176:   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
    177:     const char  *name;
    178:     const char  *colors[3] = {"red", "blue", "green"};
    179:     const int    numColors  = 3;
    180:     PetscReal    scale      = 2.0;
    181:     PetscScalar *coords;
    182:     PetscInt     depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, e, p;
    183:     PetscMPIInt  rank, size;

    185:     DMPlexGetDepth(dm, &depth);
    186:     MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);
    187:     MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);
    188:     PetscObjectGetName((PetscObject) dm, &name);
    189:     PetscViewerASCIISynchronizedAllow(viewer, PETSC_TRUE);
    190:     PetscViewerASCIIPrintf(viewer, "\
    191: \\documentclass[crop,multi=false]{standalone}\n\n\
    192: \\usepackage{tikz}\n\
    193: \\usepackage{pgflibraryshapes}\n\
    194: \\usetikzlibrary{backgrounds}\n\
    195: \\usetikzlibrary{arrows}\n\
    196: \\begin{document}\n\
    197: \\section{%s}\n\
    198: \\begin{center}\n", name, 8.0/scale);
    199:     PetscViewerASCIIPrintf(viewer, "Mesh for process ");
    200:     for (p = 0; p < size; ++p) {
    201:       if (p > 0 && p == size-1) {
    202:         PetscViewerASCIIPrintf(viewer, ", and ", colors[p%numColors], p);
    203:       } else if (p > 0) {
    204:         PetscViewerASCIIPrintf(viewer, ", ", colors[p%numColors], p);
    205:       }
    206:       PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%D}", colors[p%numColors], p);
    207:     }
    208:     PetscViewerASCIIPrintf(viewer, ".\n\n\n\
    209: \\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n");
    210:     /* Plot vertices */
    211:     DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);
    212:     VecGetArray(coordinates, &coords);
    213:     PetscViewerASCIIPrintf(viewer, "\\path\n");
    214:     for (v = vStart; v < vEnd; ++v) {
    215:       PetscInt off, dof, d;

    217:       PetscSectionGetDof(coordSection, v, &dof);
    218:       PetscSectionGetOffset(coordSection, v, &off);
    219:       PetscViewerASCIISynchronizedPrintf(viewer, "(");
    220:       for (d = 0; d < dof; ++d) {
    221:         if (d > 0) {PetscViewerASCIISynchronizedPrintf(viewer, ",");}
    222:         PetscViewerASCIISynchronizedPrintf(viewer, "%G", scale*PetscRealPart(coords[off+d]));
    223:       }
    224:       PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%D) [draw,shape=circle,color=%s] {%D} --\n", v, rank, colors[rank%numColors], v);
    225:     }
    226:     VecRestoreArray(coordinates, &coords);
    227:     PetscViewerFlush(viewer);
    228:     PetscViewerASCIIPrintf(viewer, "(0,0);\n");
    229:     /* Plot edges */
    230:     VecGetArray(coordinates, &coords);
    231:     PetscViewerASCIIPrintf(viewer, "\\path\n");
    232:     if (depth > 1) {DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);}
    233:     for (e = eStart; e < eEnd; ++e) {
    234:       const PetscInt *cone;
    235:       PetscInt        coneSize, offA, offB, dof, d;

    237:       DMPlexGetConeSize(dm, e, &coneSize);
    238:       if (coneSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %d cone should have two vertices, not %d", e, coneSize);
    239:       DMPlexGetCone(dm, e, &cone);
    240:       PetscSectionGetDof(coordSection, cone[0], &dof);
    241:       PetscSectionGetOffset(coordSection, cone[0], &offA);
    242:       PetscSectionGetOffset(coordSection, cone[1], &offB);
    243:       PetscViewerASCIISynchronizedPrintf(viewer, "(");
    244:       for (d = 0; d < dof; ++d) {
    245:         if (d > 0) {PetscViewerASCIISynchronizedPrintf(viewer, ",");}
    246:         PetscViewerASCIISynchronizedPrintf(viewer, "%G", scale*0.5*PetscRealPart(coords[offA+d]+coords[offB+d]));
    247:       }
    248:       PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%D) [draw,shape=circle,color=%s] {%D} --\n", e, rank, colors[rank%numColors], e);
    249:     }
    250:     VecRestoreArray(coordinates, &coords);
    251:     PetscViewerFlush(viewer);
    252:     PetscViewerASCIIPrintf(viewer, "(0,0);\n");
    253:     /* Plot cells */
    254:     DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);
    255:     for (c = cStart; c < cEnd; ++c) {
    256:       PetscInt *closure = NULL;
    257:       PetscInt  closureSize, firstPoint = -1;

    259:       DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);
    260:       PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank%numColors]);
    261:       for (p = 0; p < closureSize*2; p += 2) {
    262:         const PetscInt point = closure[p];

    264:         if ((point < vStart) || (point >= vEnd)) continue;
    265:         if (firstPoint >= 0) {PetscViewerASCIISynchronizedPrintf(viewer, " -- ");}
    266:         PetscViewerASCIISynchronizedPrintf(viewer, "(%D_%D)", point, rank);
    267:         if (firstPoint < 0) firstPoint = point;
    268:       }
    269:       /* Why doesn't this work? PetscViewerASCIISynchronizedPrintf(viewer, " -- cycle;\n"); */
    270:       PetscViewerASCIISynchronizedPrintf(viewer, " -- (%D_%D);\n", firstPoint, rank);
    271:       DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);
    272:     }
    273:     PetscViewerFlush(viewer);
    274:     PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n\\end{center}\n");
    275:     PetscViewerASCIIPrintf(viewer, "\\end{document}\n", name);
    276:   } else {
    277:     MPI_Comm    comm;
    278:     PetscInt   *sizes;
    279:     PetscInt    locDepth, depth, dim, d;
    280:     PetscInt    pStart, pEnd, p;
    281:     PetscInt    numLabels, l;
    282:     PetscMPIInt size;

    284:     PetscObjectGetComm((PetscObject)dm,&comm);
    285:     MPI_Comm_size(comm, &size);
    286:     DMPlexGetDimension(dm, &dim);
    287:     PetscViewerASCIIPrintf(viewer, "Mesh in %D dimensions:\n", dim);
    288:     DMPlexGetDepth(dm, &locDepth);
    289:     MPI_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm);
    290:     PetscMalloc(size * sizeof(PetscInt), &sizes);
    291:     if (depth == 1) {
    292:       DMPlexGetDepthStratum(dm, 0, &pStart, &pEnd);
    293:       pEnd = pEnd - pStart;
    294:       MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);
    295:       PetscViewerASCIIPrintf(viewer, "  %D-cells:", 0);
    296:       for (p = 0; p < size; ++p) {PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);}
    297:       PetscViewerASCIIPrintf(viewer, "\n");
    298:       DMPlexGetHeightStratum(dm, 0, &pStart, &pEnd);
    299:       pEnd = pEnd - pStart;
    300:       MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);
    301:       PetscViewerASCIIPrintf(viewer, "  %D-cells:", dim);
    302:       for (p = 0; p < size; ++p) {PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);}
    303:       PetscViewerASCIIPrintf(viewer, "\n");
    304:     } else {
    305:       for (d = 0; d <= dim; d++) {
    306:         DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);
    307:         pEnd = pEnd - pStart;
    308:         MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);
    309:         PetscViewerASCIIPrintf(viewer, "  %D-cells:", d);
    310:         for (p = 0; p < size; ++p) {PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);}
    311:         PetscViewerASCIIPrintf(viewer, "\n");
    312:       }
    313:     }
    314:     PetscFree(sizes);
    315:     DMPlexGetNumLabels(dm, &numLabels);
    316:     if (numLabels) {PetscViewerASCIIPrintf(viewer, "Labels:\n");}
    317:     for (l = 0; l < numLabels; ++l) {
    318:       DMLabel         label;
    319:       const char     *name;
    320:       IS              valueIS;
    321:       const PetscInt *values;
    322:       PetscInt        numValues, v;

    324:       DMPlexGetLabelName(dm, l, &name);
    325:       DMPlexGetLabel(dm, name, &label);
    326:       DMLabelGetNumValues(label, &numValues);
    327:       PetscViewerASCIIPrintf(viewer, "  %s: %d strata of sizes (", name, numValues);
    328:       DMLabelGetValueIS(label, &valueIS);
    329:       ISGetIndices(valueIS, &values);
    330:       for (v = 0; v < numValues; ++v) {
    331:         PetscInt size;

    333:         DMLabelGetStratumSize(label, values[v], &size);
    334:         if (v > 0) {PetscViewerASCIIPrintf(viewer, ", ");}
    335:         PetscViewerASCIIPrintf(viewer, "%d", size);
    336:       }
    337:       PetscViewerASCIIPrintf(viewer, ")\n");
    338:       ISRestoreIndices(valueIS, &values);
    339:       ISDestroy(&valueIS);
    340:     }
    341:   }
    342:   return(0);
    343: }

    347: PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
    348: {
    349:   PetscBool      iascii, isbinary;

    355:   PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII, &iascii);
    356:   PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERBINARY, &isbinary);
    357:   if (iascii) {
    358:     DMPlexView_Ascii(dm, viewer);
    359: #if 0
    360:   } else if (isbinary) {
    361:     DMPlexView_Binary(dm, viewer);
    362: #endif
    363:   }
    364:   return(0);
    365: }

    369: PetscErrorCode DMDestroy_Plex(DM dm)
    370: {
    371:   DM_Plex       *mesh = (DM_Plex*) dm->data;
    372:   DMLabel        next  = mesh->labels;

    376:   if (--mesh->refct > 0) return(0);
    377:   PetscSectionDestroy(&mesh->coneSection);
    378:   PetscFree(mesh->cones);
    379:   PetscFree(mesh->coneOrientations);
    380:   PetscSectionDestroy(&mesh->supportSection);
    381:   PetscFree(mesh->supports);
    382:   PetscFree(mesh->facesTmp);
    383:   while (next) {
    384:     DMLabel tmp = next->next;

    386:     DMLabelDestroy(&next);
    387:     next = tmp;
    388:   }
    389:   DMLabelDestroy(&mesh->subpointMap);
    390:   ISDestroy(&mesh->globalVertexNumbers);
    391:   ISDestroy(&mesh->globalCellNumbers);
    392:   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
    393:   PetscFree(mesh);
    394:   return(0);
    395: }

    399: PetscErrorCode DMCreateMatrix_Plex(DM dm, MatType mtype, Mat *J)
    400: {
    401:   PetscSection   section, sectionGlobal;
    402:   PetscInt       bs = -1;
    403:   PetscInt       localSize;
    404:   PetscBool      isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock, isSymmetric;

    408: #if !defined(PETSC_USE_DYNAMIC_LIBRARIES)
    409:   MatInitializePackage();
    410: #endif
    411:   if (!mtype) mtype = MATAIJ;
    412:   DMGetDefaultSection(dm, &section);
    413:   DMGetDefaultGlobalSection(dm, &sectionGlobal);
    414:   /* PetscSectionGetStorageSize(sectionGlobal, &localSize); */
    415:   PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);
    416:   MatCreate(PetscObjectComm((PetscObject)dm), J);
    417:   MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);
    418:   MatSetType(*J, mtype);
    419:   MatSetFromOptions(*J);
    420:   PetscStrcmp(mtype, MATSHELL, &isShell);
    421:   PetscStrcmp(mtype, MATBAIJ, &isBlock);
    422:   PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);
    423:   PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);
    424:   PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);
    425:   PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);
    426:   PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);
    427:   /* Check for symmetric storage */
    428:   isSymmetric = (PetscBool) (isSymBlock || isSymSeqBlock || isSymMPIBlock);
    429:   if (isSymmetric) {
    430:     MatSetOption(*J, MAT_IGNORE_LOWER_TRIANGULAR, PETSC_TRUE);
    431:   }
    432:   if (!isShell) {
    433:     PetscBool fillMatrix = (PetscBool) !dm->prealloc_only;
    434:     PetscInt *dnz, *onz, *dnzu, *onzu, bsLocal, bsMax, bsMin;

    436:     if (bs < 0) {
    437:       if (isBlock || isSeqBlock || isMPIBlock || isSymBlock || isSymSeqBlock || isSymMPIBlock) {
    438:         PetscInt pStart, pEnd, p, dof, cdof;

    440:         PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);
    441:         for (p = pStart; p < pEnd; ++p) {
    442:           PetscSectionGetDof(sectionGlobal, p, &dof);
    443:           PetscSectionGetConstraintDof(sectionGlobal, p, &cdof);
    444:           if (dof-cdof) {
    445:             if (bs < 0) {
    446:               bs = dof-cdof;
    447:             } else if (bs != dof-cdof) {
    448:               /* Layout does not admit a pointwise block size */
    449:               bs = 1;
    450:               break;
    451:             }
    452:           }
    453:         }
    454:         /* Must have same blocksize on all procs (some might have no points) */
    455:         bsLocal = bs;
    456:         MPI_Allreduce(&bsLocal, &bsMax, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm));
    457:         bsLocal = bs < 0 ? bsMax : bs;
    458:         MPI_Allreduce(&bsLocal, &bsMin, 1, MPIU_INT, MPI_MIN, PetscObjectComm((PetscObject)dm));
    459:         if (bsMin != bsMax) {
    460:           bs = 1;
    461:         } else {
    462:           bs = bsMax;
    463:         }
    464:       } else {
    465:         bs = 1;
    466:       }
    467:     }
    468:     PetscMalloc4(localSize/bs, PetscInt, &dnz, localSize/bs, PetscInt, &onz, localSize/bs, PetscInt, &dnzu, localSize/bs, PetscInt, &onzu);
    469:     PetscMemzero(dnz,  localSize/bs * sizeof(PetscInt));
    470:     PetscMemzero(onz,  localSize/bs * sizeof(PetscInt));
    471:     PetscMemzero(dnzu, localSize/bs * sizeof(PetscInt));
    472:     PetscMemzero(onzu, localSize/bs * sizeof(PetscInt));
    473:     DMPlexPreallocateOperator(dm, bs, section, sectionGlobal, dnz, onz, dnzu, onzu, *J, fillMatrix);
    474:     PetscFree4(dnz, onz, dnzu, onzu);
    475:   }
    476:   return(0);
    477: }

    481: /*@
    482:   DMPlexGetDimension - Return the topological mesh dimension

    484:   Not collective

    486:   Input Parameter:
    487: . mesh - The DMPlex

    489:   Output Parameter:
    490: . dim - The topological mesh dimension

    492:   Level: beginner

    494: .seealso: DMPlexCreate()
    495: @*/
    496: PetscErrorCode DMPlexGetDimension(DM dm, PetscInt *dim)
    497: {
    498:   DM_Plex *mesh = (DM_Plex*) dm->data;

    503:   *dim = mesh->dim;
    504:   return(0);
    505: }

    509: /*@
    510:   DMPlexSetDimension - Set the topological mesh dimension

    512:   Collective on mesh

    514:   Input Parameters:
    515: + mesh - The DMPlex
    516: - dim - The topological mesh dimension

    518:   Level: beginner

    520: .seealso: DMPlexCreate()
    521: @*/
    522: PetscErrorCode DMPlexSetDimension(DM dm, PetscInt dim)
    523: {
    524:   DM_Plex *mesh = (DM_Plex*) dm->data;

    529:   mesh->dim               = dim;
    530:   mesh->preallocCenterDim = dim;
    531:   return(0);
    532: }

    536: /*@
    537:   DMPlexGetChart - Return the interval for all mesh points [pStart, pEnd)

    539:   Not collective

    541:   Input Parameter:
    542: . mesh - The DMPlex

    544:   Output Parameters:
    545: + pStart - The first mesh point
    546: - pEnd   - The upper bound for mesh points

    548:   Level: beginner

    550: .seealso: DMPlexCreate(), DMPlexSetChart()
    551: @*/
    552: PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
    553: {
    554:   DM_Plex       *mesh = (DM_Plex*) dm->data;

    559:   PetscSectionGetChart(mesh->coneSection, pStart, pEnd);
    560:   return(0);
    561: }

    565: /*@
    566:   DMPlexSetChart - Set the interval for all mesh points [pStart, pEnd)

    568:   Not collective

    570:   Input Parameters:
    571: + mesh - The DMPlex
    572: . pStart - The first mesh point
    573: - pEnd   - The upper bound for mesh points

    575:   Output Parameters:

    577:   Level: beginner

    579: .seealso: DMPlexCreate(), DMPlexGetChart()
    580: @*/
    581: PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
    582: {
    583:   DM_Plex       *mesh = (DM_Plex*) dm->data;

    588:   PetscSectionSetChart(mesh->coneSection, pStart, pEnd);
    589:   PetscSectionSetChart(mesh->supportSection, pStart, pEnd);
    590:   return(0);
    591: }

    595: /*@
    596:   DMPlexGetConeSize - Return the number of in-edges for this point in the Sieve DAG

    598:   Not collective

    600:   Input Parameters:
    601: + mesh - The DMPlex
    602: - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()

    604:   Output Parameter:
    605: . size - The cone size for point p

    607:   Level: beginner

    609: .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
    610: @*/
    611: PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
    612: {
    613:   DM_Plex       *mesh = (DM_Plex*) dm->data;

    619:   PetscSectionGetDof(mesh->coneSection, p, size);
    620:   return(0);
    621: }

    625: /*@
    626:   DMPlexSetConeSize - Set the number of in-edges for this point in the Sieve DAG

    628:   Not collective

    630:   Input Parameters:
    631: + mesh - The DMPlex
    632: . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
    633: - size - The cone size for point p

    635:   Output Parameter:

    637:   Note:
    638:   This should be called after DMPlexSetChart().

    640:   Level: beginner

    642: .seealso: DMPlexCreate(), DMPlexGetConeSize(), DMPlexSetChart()
    643: @*/
    644: PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
    645: {
    646:   DM_Plex       *mesh = (DM_Plex*) dm->data;

    651:   PetscSectionSetDof(mesh->coneSection, p, size);

    653:   mesh->maxConeSize = PetscMax(mesh->maxConeSize, size);
    654:   return(0);
    655: }

    659: /*@C
    660:   DMPlexGetCone - Return the points on the in-edges for this point in the Sieve DAG

    662:   Not collective

    664:   Input Parameters:
    665: + mesh - The DMPlex
    666: - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()

    668:   Output Parameter:
    669: . cone - An array of points which are on the in-edges for point p

    671:   Level: beginner

    673:   Fortran Notes:
    674:   Since it returns an array, this routine is only available in Fortran 90, and you must
    675:   include petsc.h90 in your code.

    677:   You must also call DMPlexRestoreCone() after you finish using the returned array.

    679: .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart()
    680: @*/
    681: PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
    682: {
    683:   DM_Plex       *mesh = (DM_Plex*) dm->data;
    684:   PetscInt       off;

    690:   PetscSectionGetOffset(mesh->coneSection, p, &off);
    691:   *cone = &mesh->cones[off];
    692:   return(0);
    693: }

    697: /*@
    698:   DMPlexSetCone - Set the points on the in-edges for this point in the Sieve DAG

    700:   Not collective

    702:   Input Parameters:
    703: + mesh - The DMPlex
    704: . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
    705: - cone - An array of points which are on the in-edges for point p

    707:   Output Parameter:

    709:   Note:
    710:   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().

    712:   Level: beginner

    714: .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
    715: @*/
    716: PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
    717: {
    718:   DM_Plex       *mesh = (DM_Plex*) dm->data;
    719:   PetscInt       pStart, pEnd;
    720:   PetscInt       dof, off, c;

    725:   PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);
    726:   PetscSectionGetDof(mesh->coneSection, p, &dof);
    728:   PetscSectionGetOffset(mesh->coneSection, p, &off);
    729:   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
    730:   for (c = 0; c < dof; ++c) {
    731:     if ((cone[c] < pStart) || (cone[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", cone[c], pStart, pEnd);
    732:     mesh->cones[off+c] = cone[c];
    733:   }
    734:   return(0);
    735: }

    739: /*@C
    740:   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the Sieve DAG

    742:   Not collective

    744:   Input Parameters:
    745: + mesh - The DMPlex
    746: - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()

    748:   Output Parameter:
    749: . coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
    750:                     integer giving the prescription for cone traversal. If it is negative, the cone is
    751:                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
    752:                     the index of the cone point on which to start.

    754:   Level: beginner

    756:   Fortran Notes:
    757:   Since it returns an array, this routine is only available in Fortran 90, and you must
    758:   include petsc.h90 in your code.

    760:   You must also call DMPlexRestoreConeOrientation() after you finish using the returned array.

    762: .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetCone(), DMPlexSetChart()
    763: @*/
    764: PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
    765: {
    766:   DM_Plex       *mesh = (DM_Plex*) dm->data;
    767:   PetscInt       off;

    772: #if defined(PETSC_USE_DEBUG)
    773:   {
    774:     PetscInt dof;
    775:     PetscSectionGetDof(mesh->coneSection, p, &dof);
    777:   }
    778: #endif
    779:   PetscSectionGetOffset(mesh->coneSection, p, &off);

    781:   *coneOrientation = &mesh->coneOrientations[off];
    782:   return(0);
    783: }

    787: /*@
    788:   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the Sieve DAG

    790:   Not collective

    792:   Input Parameters:
    793: + mesh - The DMPlex
    794: . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
    795: - coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
    796:                     integer giving the prescription for cone traversal. If it is negative, the cone is
    797:                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
    798:                     the index of the cone point on which to start.

    800:   Output Parameter:

    802:   Note:
    803:   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().

    805:   Level: beginner

    807: .seealso: DMPlexCreate(), DMPlexGetConeOrientation(), DMPlexSetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
    808: @*/
    809: PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
    810: {
    811:   DM_Plex       *mesh = (DM_Plex*) dm->data;
    812:   PetscInt       pStart, pEnd;
    813:   PetscInt       dof, off, c;

    818:   PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);
    819:   PetscSectionGetDof(mesh->coneSection, p, &dof);
    821:   PetscSectionGetOffset(mesh->coneSection, p, &off);
    822:   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
    823:   for (c = 0; c < dof; ++c) {
    824:     PetscInt cdof, o = coneOrientation[c];

    826:     PetscSectionGetDof(mesh->coneSection, mesh->cones[off+c], &cdof);
    827:     if (o && ((o < -(cdof+1)) || (o >= cdof))) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %D is not in the valid range [%D. %D)", o, -(cdof+1), cdof);
    828:     mesh->coneOrientations[off+c] = o;
    829:   }
    830:   return(0);
    831: }

    835: PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
    836: {
    837:   DM_Plex       *mesh = (DM_Plex*) dm->data;
    838:   PetscInt       pStart, pEnd;
    839:   PetscInt       dof, off;

    844:   PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);
    845:   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
    846:   if ((conePoint < pStart) || (conePoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", conePoint, pStart, pEnd);
    847:   PetscSectionGetDof(mesh->coneSection, p, &dof);
    848:   PetscSectionGetOffset(mesh->coneSection, p, &off);
    849:   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
    850:   mesh->cones[off+conePos] = conePoint;
    851:   return(0);
    852: }

    856: PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
    857: {
    858:   DM_Plex       *mesh = (DM_Plex*) dm->data;
    859:   PetscInt       pStart, pEnd;
    860:   PetscInt       dof, off;

    865:   PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);
    866:   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
    867:   PetscSectionGetDof(mesh->coneSection, p, &dof);
    868:   PetscSectionGetOffset(mesh->coneSection, p, &off);
    869:   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
    870:   mesh->coneOrientations[off+conePos] = coneOrientation;
    871:   return(0);
    872: }

    876: /*@
    877:   DMPlexGetSupportSize - Return the number of out-edges for this point in the Sieve DAG

    879:   Not collective

    881:   Input Parameters:
    882: + mesh - The DMPlex
    883: - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()

    885:   Output Parameter:
    886: . size - The support size for point p

    888:   Level: beginner

    890: .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart(), DMPlexGetConeSize()
    891: @*/
    892: PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
    893: {
    894:   DM_Plex       *mesh = (DM_Plex*) dm->data;

    900:   PetscSectionGetDof(mesh->supportSection, p, size);
    901:   return(0);
    902: }

    906: /*@
    907:   DMPlexSetSupportSize - Set the number of out-edges for this point in the Sieve DAG

    909:   Not collective

    911:   Input Parameters:
    912: + mesh - The DMPlex
    913: . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
    914: - size - The support size for point p

    916:   Output Parameter:

    918:   Note:
    919:   This should be called after DMPlexSetChart().

    921:   Level: beginner

    923: .seealso: DMPlexCreate(), DMPlexGetSupportSize(), DMPlexSetChart()
    924: @*/
    925: PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
    926: {
    927:   DM_Plex       *mesh = (DM_Plex*) dm->data;

    932:   PetscSectionSetDof(mesh->supportSection, p, size);

    934:   mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, size);
    935:   return(0);
    936: }

    940: /*@C
    941:   DMPlexGetSupport - Return the points on the out-edges for this point in the Sieve DAG

    943:   Not collective

    945:   Input Parameters:
    946: + mesh - The DMPlex
    947: - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()

    949:   Output Parameter:
    950: . support - An array of points which are on the out-edges for point p

    952:   Level: beginner

    954:   Fortran Notes:
    955:   Since it returns an array, this routine is only available in Fortran 90, and you must
    956:   include petsc.h90 in your code.

    958:   You must also call DMPlexRestoreSupport() after you finish using the returned array.

    960: .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
    961: @*/
    962: PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
    963: {
    964:   DM_Plex       *mesh = (DM_Plex*) dm->data;
    965:   PetscInt       off;

    971:   PetscSectionGetOffset(mesh->supportSection, p, &off);
    972:   *support = &mesh->supports[off];
    973:   return(0);
    974: }

    978: /*@
    979:   DMPlexSetSupport - Set the points on the out-edges for this point in the Sieve DAG

    981:   Not collective

    983:   Input Parameters:
    984: + mesh - The DMPlex
    985: . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
    986: - support - An array of points which are on the in-edges for point p

    988:   Output Parameter:

    990:   Note:
    991:   This should be called after all calls to DMPlexSetSupportSize() and DMSetUp().

    993:   Level: beginner

    995: .seealso: DMPlexCreate(), DMPlexGetSupport(), DMPlexSetChart(), DMPlexSetSupportSize(), DMSetUp()
    996: @*/
    997: PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
    998: {
    999:   DM_Plex       *mesh = (DM_Plex*) dm->data;
    1000:   PetscInt       pStart, pEnd;
    1001:   PetscInt       dof, off, c;

    1006:   PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);
    1007:   PetscSectionGetDof(mesh->supportSection, p, &dof);
    1009:   PetscSectionGetOffset(mesh->supportSection, p, &off);
    1010:   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
    1011:   for (c = 0; c < dof; ++c) {
    1012:     if ((support[c] < pStart) || (support[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", support[c], pStart, pEnd);
    1013:     mesh->supports[off+c] = support[c];
    1014:   }
    1015:   return(0);
    1016: }

    1020: PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
    1021: {
    1022:   DM_Plex       *mesh = (DM_Plex*) dm->data;
    1023:   PetscInt       pStart, pEnd;
    1024:   PetscInt       dof, off;

    1029:   PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);
    1030:   PetscSectionGetDof(mesh->supportSection, p, &dof);
    1031:   PetscSectionGetOffset(mesh->supportSection, p, &off);
    1032:   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
    1033:   if ((supportPoint < pStart) || (supportPoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", supportPoint, pStart, pEnd);
    1034:   if (supportPos >= dof) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %D of point %D is not in the valid range [0, %D)", supportPos, p, dof);
    1035:   mesh->supports[off+supportPos] = supportPoint;
    1036:   return(0);
    1037: }

    1041: /*@C
    1042:   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the Sieve DAG

    1044:   Not collective

    1046:   Input Parameters:
    1047: + mesh - The DMPlex
    1048: . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
    1049: . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
    1050: - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used

    1052:   Output Parameters:
    1053: + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
    1054: - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]

    1056:   Note:
    1057:   If using internal storage (points is NULL on input), each call overwrites the last output.

    1059:   Fortran Notes:
    1060:   Since it returns an array, this routine is only available in Fortran 90, and you must
    1061:   include petsc.h90 in your code.

    1063:   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.

    1065:   Level: beginner

    1067: .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
    1068: @*/
    1069: PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
    1070: {
    1071:   DM_Plex        *mesh = (DM_Plex*) dm->data;
    1072:   PetscInt       *closure, *fifo;
    1073:   const PetscInt *tmp = NULL, *tmpO = NULL;
    1074:   PetscInt        tmpSize, t;
    1075:   PetscInt        depth       = 0, maxSize;
    1076:   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
    1077:   PetscErrorCode  ierr;

    1081:   DMPlexGetDepth(dm, &depth);
    1082:   maxSize = 2*PetscMax(PetscMax(PetscPowInt(mesh->maxConeSize,depth+1),PetscPowInt(mesh->maxSupportSize,depth+1)),depth+1);
    1083:   DMGetWorkArray(dm, maxSize, PETSC_INT, &fifo);
    1084:   if (*points) {
    1085:     closure = *points;
    1086:   } else {
    1087:     DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);
    1088:   }
    1089:   closure[0] = p; closure[1] = 0;
    1090:   /* This is only 1-level */
    1091:   if (useCone) {
    1092:     DMPlexGetConeSize(dm, p, &tmpSize);
    1093:     DMPlexGetCone(dm, p, &tmp);
    1094:     DMPlexGetConeOrientation(dm, p, &tmpO);
    1095:   } else {
    1096:     DMPlexGetSupportSize(dm, p, &tmpSize);
    1097:     DMPlexGetSupport(dm, p, &tmp);
    1098:   }
    1099:   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
    1100:     const PetscInt cp = tmp[t];
    1101:     const PetscInt co = tmpO ? tmpO[t] : 0;

    1103:     closure[closureSize]   = cp;
    1104:     closure[closureSize+1] = co;
    1105:     fifo[fifoSize]         = cp;
    1106:     fifo[fifoSize+1]       = co;
    1107:   }
    1108:   while (fifoSize - fifoStart) {
    1109:     const PetscInt q   = fifo[fifoStart];
    1110:     const PetscInt o   = fifo[fifoStart+1];
    1111:     const PetscInt rev = o >= 0 ? 0 : 1;
    1112:     const PetscInt off = rev ? -(o+1) : o;

    1114:     if (useCone) {
    1115:       DMPlexGetConeSize(dm, q, &tmpSize);
    1116:       DMPlexGetCone(dm, q, &tmp);
    1117:       DMPlexGetConeOrientation(dm, q, &tmpO);
    1118:     } else {
    1119:       DMPlexGetSupportSize(dm, q, &tmpSize);
    1120:       DMPlexGetSupport(dm, q, &tmp);
    1121:       tmpO = NULL;
    1122:     }
    1123:     for (t = 0; t < tmpSize; ++t) {
    1124:       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
    1125:       const PetscInt cp = tmp[i];
    1126:       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
    1127:       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
    1128:        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
    1129:       PetscInt       co = tmpO ? tmpO[i] : 0;
    1130:       PetscInt       c;

    1132:       if (rev) {
    1133:         PetscInt childSize, coff;
    1134:         DMPlexGetConeSize(dm, cp, &childSize);
    1135:         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
    1136:         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
    1137:       }
    1138:       /* Check for duplicate */
    1139:       for (c = 0; c < closureSize; c += 2) {
    1140:         if (closure[c] == cp) break;
    1141:       }
    1142:       if (c == closureSize) {
    1143:         closure[closureSize]   = cp;
    1144:         closure[closureSize+1] = co;
    1145:         fifo[fifoSize]         = cp;
    1146:         fifo[fifoSize+1]       = co;
    1147:         closureSize           += 2;
    1148:         fifoSize              += 2;
    1149:       }
    1150:     }
    1151:     fifoStart += 2;
    1152:   }
    1153:   if (numPoints) *numPoints = closureSize/2;
    1154:   if (points)    *points    = closure;
    1155:   DMRestoreWorkArray(dm, maxSize, PETSC_INT, &fifo);
    1156:   return(0);
    1157: }

    1161: /*@C
    1162:   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the Sieve DAG

    1164:   Not collective

    1166:   Input Parameters:
    1167: + mesh - The DMPlex
    1168: . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
    1169: . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
    1170: - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used

    1172:   Output Parameters:
    1173: + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
    1174: - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]

    1176:   Note:
    1177:   If not using internal storage (points is not NULL on input), this call is unnecessary

    1179:   Fortran Notes:
    1180:   Since it returns an array, this routine is only available in Fortran 90, and you must
    1181:   include petsc.h90 in your code.

    1183:   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.

    1185:   Level: beginner

    1187: .seealso: DMPlexGetTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
    1188: @*/
    1189: PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
    1190: {

    1195:   DMRestoreWorkArray(dm, 0, PETSC_INT, points);
    1196:   return(0);
    1197: }

    1201: /*@
    1202:   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the Sieve DAG

    1204:   Not collective

    1206:   Input Parameter:
    1207: . mesh - The DMPlex

    1209:   Output Parameters:
    1210: + maxConeSize - The maximum number of in-edges
    1211: - maxSupportSize - The maximum number of out-edges

    1213:   Level: beginner

    1215: .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
    1216: @*/
    1217: PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
    1218: {
    1219:   DM_Plex *mesh = (DM_Plex*) dm->data;

    1223:   if (maxConeSize)    *maxConeSize    = mesh->maxConeSize;
    1224:   if (maxSupportSize) *maxSupportSize = mesh->maxSupportSize;
    1225:   return(0);
    1226: }

    1230: PetscErrorCode DMSetUp_Plex(DM dm)
    1231: {
    1232:   DM_Plex       *mesh = (DM_Plex*) dm->data;
    1233:   PetscInt       size;

    1238:   PetscSectionSetUp(mesh->coneSection);
    1239:   PetscSectionGetStorageSize(mesh->coneSection, &size);
    1240:   PetscMalloc(size * sizeof(PetscInt), &mesh->cones);
    1241:   PetscMalloc(size * sizeof(PetscInt), &mesh->coneOrientations);
    1242:   PetscMemzero(mesh->coneOrientations, size * sizeof(PetscInt));
    1243:   if (mesh->maxSupportSize) {
    1244:     PetscSectionSetUp(mesh->supportSection);
    1245:     PetscSectionGetStorageSize(mesh->supportSection, &size);
    1246:     PetscMalloc(size * sizeof(PetscInt), &mesh->supports);
    1247:   }
    1248:   return(0);
    1249: }

    1253: PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, PetscInt fields[], IS *is, DM *subdm)
    1254: {
    1255:   PetscSection   section, sectionGlobal;
    1256:   PetscInt      *subIndices;
    1257:   PetscInt       subSize = 0, subOff = 0, nF, f, pStart, pEnd, p;

    1261:   if (!numFields) return(0);
    1262:   DMGetDefaultSection(dm, &section);
    1263:   DMGetDefaultGlobalSection(dm, &sectionGlobal);
    1264:   if (!section) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Must set default section for DMPlex before splitting fields");
    1265:   if (!sectionGlobal) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Must set default global section for DMPlex before splitting fields");
    1266:   PetscSectionGetNumFields(section, &nF);
    1267:   if (numFields > nF) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Number of requested fields %d greater than number of DM fields %d", numFields, nF);
    1268:   PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);
    1269:   for (p = pStart; p < pEnd; ++p) {
    1270:     PetscInt gdof;

    1272:     PetscSectionGetDof(sectionGlobal, p, &gdof);
    1273:     if (gdof > 0) {
    1274:       for (f = 0; f < numFields; ++f) {
    1275:         PetscInt fdof, fcdof;

    1277:         PetscSectionGetFieldDof(section, p, fields[f], &fdof);
    1278:         PetscSectionGetFieldConstraintDof(section, p, fields[f], &fcdof);
    1279:         subSize += fdof-fcdof;
    1280:       }
    1281:     }
    1282:   }
    1283:   PetscMalloc(subSize * sizeof(PetscInt), &subIndices);
    1284:   for (p = pStart; p < pEnd; ++p) {
    1285:     PetscInt gdof, goff;

    1287:     PetscSectionGetDof(sectionGlobal, p, &gdof);
    1288:     if (gdof > 0) {
    1289:       PetscSectionGetOffset(sectionGlobal, p, &goff);
    1290:       for (f = 0; f < numFields; ++f) {
    1291:         PetscInt fdof, fcdof, fc, f2, poff = 0;

    1293:         /* Can get rid of this loop by storing field information in the global section */
    1294:         for (f2 = 0; f2 < fields[f]; ++f2) {
    1295:           PetscSectionGetFieldDof(section, p, f2, &fdof);
    1296:           PetscSectionGetFieldConstraintDof(section, p, f2, &fcdof);
    1297:           poff += fdof-fcdof;
    1298:         }
    1299:         PetscSectionGetFieldDof(section, p, fields[f], &fdof);
    1300:         PetscSectionGetFieldConstraintDof(section, p, fields[f], &fcdof);
    1301:         for (fc = 0; fc < fdof-fcdof; ++fc, ++subOff) {
    1302:           subIndices[subOff] = goff+poff+fc;
    1303:         }
    1304:       }
    1305:     }
    1306:   }
    1307:   if (is) {ISCreateGeneral(PetscObjectComm((PetscObject)dm), subSize, subIndices, PETSC_OWN_POINTER, is);}
    1308:   if (subdm) {
    1309:     PetscSection subsection;
    1310:     PetscBool    haveNull = PETSC_FALSE;
    1311:     PetscInt     f, nf = 0;

    1313:     DMPlexClone(dm, subdm);
    1314:     PetscSectionCreateSubsection(section, numFields, fields, &subsection);
    1315:     DMSetDefaultSection(*subdm, subsection);
    1316:     PetscSectionDestroy(&subsection);
    1317:     for (f = 0; f < numFields; ++f) {
    1318:       (*subdm)->nullspaceConstructors[f] = dm->nullspaceConstructors[fields[f]];
    1319:       if ((*subdm)->nullspaceConstructors[f]) {
    1320:         haveNull = PETSC_TRUE;
    1321:         nf       = f;
    1322:       }
    1323:     }
    1324:     if (haveNull) {
    1325:       MatNullSpace nullSpace;

    1327:       (*(*subdm)->nullspaceConstructors[nf])(*subdm, nf, &nullSpace);
    1328:       PetscObjectCompose((PetscObject) *is, "nullspace", (PetscObject) nullSpace);
    1329:       MatNullSpaceDestroy(&nullSpace);
    1330:     }
    1331:     if (dm->fields) {
    1332:       if (nF != dm->numFields) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "The number of DM fields %d does not match the number of Section fields %d", dm->numFields, nF);
    1333:       DMSetNumFields(*subdm, numFields);
    1334:       for (f = 0; f < numFields; ++f) {
    1335:         PetscObjectListDuplicate(dm->fields[fields[f]]->olist, &(*subdm)->fields[f]->olist);
    1336:       }
    1337:       if (numFields == 1) {
    1338:         MatNullSpace space;
    1339:         Mat          pmat;

    1341:         PetscObjectQuery((*subdm)->fields[0], "nullspace", (PetscObject*) &space);
    1342:         if (space) {PetscObjectCompose((PetscObject) *is, "nullspace", (PetscObject) space);}
    1343:         PetscObjectQuery((*subdm)->fields[0], "nearnullspace", (PetscObject*) &space);
    1344:         if (space) {PetscObjectCompose((PetscObject) *is, "nearnullspace", (PetscObject) space);}
    1345:         PetscObjectQuery((*subdm)->fields[0], "pmat", (PetscObject*) &pmat);
    1346:         if (pmat) {PetscObjectCompose((PetscObject) *is, "pmat", (PetscObject) pmat);}
    1347:       }
    1348:     }
    1349:   }
    1350:   return(0);
    1351: }

    1355: /*@
    1356:   DMPlexSymmetrize - Creates support (out-edge) information from cone (in-edge) inoformation

    1358:   Not collective

    1360:   Input Parameter:
    1361: . mesh - The DMPlex

    1363:   Output Parameter:

    1365:   Note:
    1366:   This should be called after all calls to DMPlexSetCone()

    1368:   Level: beginner

    1370: .seealso: DMPlexCreate(), DMPlexSetChart(), DMPlexSetConeSize(), DMPlexSetCone()
    1371: @*/
    1372: PetscErrorCode DMPlexSymmetrize(DM dm)
    1373: {
    1374:   DM_Plex       *mesh = (DM_Plex*) dm->data;
    1375:   PetscInt      *offsets;
    1376:   PetscInt       supportSize;
    1377:   PetscInt       pStart, pEnd, p;

    1382:   if (mesh->supports) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
    1383:   /* Calculate support sizes */
    1384:   DMPlexGetChart(dm, &pStart, &pEnd);
    1385:   for (p = pStart; p < pEnd; ++p) {
    1386:     PetscInt dof, off, c;

    1388:     PetscSectionGetDof(mesh->coneSection, p, &dof);
    1389:     PetscSectionGetOffset(mesh->coneSection, p, &off);
    1390:     for (c = off; c < off+dof; ++c) {
    1391:       PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1);
    1392:     }
    1393:   }
    1394:   for (p = pStart; p < pEnd; ++p) {
    1395:     PetscInt dof;

    1397:     PetscSectionGetDof(mesh->supportSection, p, &dof);

    1399:     mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, dof);
    1400:   }
    1401:   PetscSectionSetUp(mesh->supportSection);
    1402:   /* Calculate supports */
    1403:   PetscSectionGetStorageSize(mesh->supportSection, &supportSize);
    1404:   PetscMalloc(supportSize * sizeof(PetscInt), &mesh->supports);
    1405:   PetscMalloc((pEnd - pStart) * sizeof(PetscInt), &offsets);
    1406:   PetscMemzero(offsets, (pEnd - pStart) * sizeof(PetscInt));
    1407:   for (p = pStart; p < pEnd; ++p) {
    1408:     PetscInt dof, off, c;

    1410:     PetscSectionGetDof(mesh->coneSection, p, &dof);
    1411:     PetscSectionGetOffset(mesh->coneSection, p, &off);
    1412:     for (c = off; c < off+dof; ++c) {
    1413:       const PetscInt q = mesh->cones[c];
    1414:       PetscInt       offS;

    1416:       PetscSectionGetOffset(mesh->supportSection, q, &offS);

    1418:       mesh->supports[offS+offsets[q]] = p;
    1419:       ++offsets[q];
    1420:     }
    1421:   }
    1422:   PetscFree(offsets);
    1423:   return(0);
    1424: }

    1428: PetscErrorCode DMPlexSetDepth_Private(DM dm, PetscInt p, PetscInt *depth)
    1429: {
    1430:   PetscInt       d;

    1434:   DMPlexGetLabelValue(dm, "depth", p, &d);
    1435:   if (d < 0) {
    1436:     /* We are guaranteed that the point has a cone since the depth was not yet set */
    1437:     const PetscInt *cone = NULL;
    1438:     PetscInt        dCone;

    1440:     DMPlexGetCone(dm, p, &cone);
    1441:     DMPlexSetDepth_Private(dm, cone[0], &dCone);
    1442:     d    = dCone+1;
    1443:     DMPlexSetLabelValue(dm, "depth", p, d);
    1444:   }
    1445:   *depth = d;
    1446:   return(0);
    1447: }

    1451: /*@
    1452:   DMPlexStratify - The Sieve DAG for most topologies is a graded poset (http://en.wikipedia.org/wiki/Graded_poset), and
    1453:   can be illustrated by Hasse Diagram (a http://en.wikipedia.org/wiki/Hasse_diagram). The strata group all points of the
    1454:   same grade, and this function calculates the strata. This grade can be seen as the height (or depth) of the point in
    1455:   the DAG.

    1457:   Not collective

    1459:   Input Parameter:
    1460: . mesh - The DMPlex

    1462:   Output Parameter:

    1464:   Notes:
    1465:   The normal association for the point grade is element dimension (or co-dimension). For instance, all vertices would
    1466:   have depth 0, and all edges depth 1. Likewise, all cells heights would have height 0, and all faces height 1.

    1468:   This should be called after all calls to DMPlexSymmetrize()

    1470:   Level: beginner

    1472: .seealso: DMPlexCreate(), DMPlexSymmetrize()
    1473: @*/
    1474: PetscErrorCode DMPlexStratify(DM dm)
    1475: {
    1476:   DM_Plex       *mesh = (DM_Plex*) dm->data;
    1477:   PetscInt       pStart, pEnd, p;
    1478:   PetscInt       numRoots = 0, numLeaves = 0;

    1483:   PetscLogEventBegin(DMPLEX_Stratify,dm,0,0,0);
    1484:   /* Calculate depth */
    1485:   PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);
    1486:   /* Initialize roots and count leaves */
    1487:   for (p = pStart; p < pEnd; ++p) {
    1488:     PetscInt coneSize, supportSize;

    1490:     DMPlexGetConeSize(dm, p, &coneSize);
    1491:     DMPlexGetSupportSize(dm, p, &supportSize);
    1492:     if (!coneSize && supportSize) {
    1493:       ++numRoots;
    1494:       DMPlexSetLabelValue(dm, "depth", p, 0);
    1495:     } else if (!supportSize && coneSize) {
    1496:       ++numLeaves;
    1497:     } else if (!supportSize && !coneSize) {
    1498:       /* Isolated points */
    1499:       DMPlexSetLabelValue(dm, "depth", p, 0);
    1500:     }
    1501:   }
    1502:   if (numRoots + numLeaves == (pEnd - pStart)) {
    1503:     for (p = pStart; p < pEnd; ++p) {
    1504:       PetscInt coneSize, supportSize;

    1506:       DMPlexGetConeSize(dm, p, &coneSize);
    1507:       DMPlexGetSupportSize(dm, p, &supportSize);
    1508:       if (!supportSize && coneSize) {
    1509:         DMPlexSetLabelValue(dm, "depth", p, 1);
    1510:       }
    1511:     }
    1512:   } else {
    1513:     /* This might be slow since lookup is not fast */
    1514:     for (p = pStart; p < pEnd; ++p) {
    1515:       PetscInt depth;

    1517:       DMPlexSetDepth_Private(dm, p, &depth);
    1518:     }
    1519:   }
    1520:   PetscLogEventEnd(DMPLEX_Stratify,dm,0,0,0);
    1521:   return(0);
    1522: }

    1526: /*@C
    1527:   DMPlexGetJoin - Get an array for the join of the set of points

    1529:   Not Collective

    1531:   Input Parameters:
    1532: + dm - The DMPlex object
    1533: . numPoints - The number of input points for the join
    1534: - points - The input points

    1536:   Output Parameters:
    1537: + numCoveredPoints - The number of points in the join
    1538: - coveredPoints - The points in the join

    1540:   Level: intermediate

    1542:   Note: Currently, this is restricted to a single level join

    1544:   Fortran Notes:
    1545:   Since it returns an array, this routine is only available in Fortran 90, and you must
    1546:   include petsc.h90 in your code.

    1548:   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.

    1550: .keywords: mesh
    1551: .seealso: DMPlexRestoreJoin(), DMPlexGetMeet()
    1552: @*/
    1553: PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
    1554: {
    1555:   DM_Plex       *mesh = (DM_Plex*) dm->data;
    1556:   PetscInt      *join[2];
    1557:   PetscInt       joinSize, i = 0;
    1558:   PetscInt       dof, off, p, c, m;

    1566:   DMGetWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[0]);
    1567:   DMGetWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1]);
    1568:   /* Copy in support of first point */
    1569:   PetscSectionGetDof(mesh->supportSection, points[0], &dof);
    1570:   PetscSectionGetOffset(mesh->supportSection, points[0], &off);
    1571:   for (joinSize = 0; joinSize < dof; ++joinSize) {
    1572:     join[i][joinSize] = mesh->supports[off+joinSize];
    1573:   }
    1574:   /* Check each successive support */
    1575:   for (p = 1; p < numPoints; ++p) {
    1576:     PetscInt newJoinSize = 0;

    1578:     PetscSectionGetDof(mesh->supportSection, points[p], &dof);
    1579:     PetscSectionGetOffset(mesh->supportSection, points[p], &off);
    1580:     for (c = 0; c < dof; ++c) {
    1581:       const PetscInt point = mesh->supports[off+c];

    1583:       for (m = 0; m < joinSize; ++m) {
    1584:         if (point == join[i][m]) {
    1585:           join[1-i][newJoinSize++] = point;
    1586:           break;
    1587:         }
    1588:       }
    1589:     }
    1590:     joinSize = newJoinSize;
    1591:     i        = 1-i;
    1592:   }
    1593:   *numCoveredPoints = joinSize;
    1594:   *coveredPoints    = join[i];
    1595:   DMRestoreWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1-i]);
    1596:   return(0);
    1597: }

    1601: /*@C
    1602:   DMPlexRestoreJoin - Restore an array for the join of the set of points

    1604:   Not Collective

    1606:   Input Parameters:
    1607: + dm - The DMPlex object
    1608: . numPoints - The number of input points for the join
    1609: - points - The input points

    1611:   Output Parameters:
    1612: + numCoveredPoints - The number of points in the join
    1613: - coveredPoints - The points in the join

    1615:   Fortran Notes:
    1616:   Since it returns an array, this routine is only available in Fortran 90, and you must
    1617:   include petsc.h90 in your code.

    1619:   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.

    1621:   Level: intermediate

    1623: .keywords: mesh
    1624: .seealso: DMPlexGetJoin(), DMPlexGetFullJoin(), DMPlexGetMeet()
    1625: @*/
    1626: PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
    1627: {

    1633:   DMRestoreWorkArray(dm, 0, PETSC_INT, (void*) coveredPoints);
    1634:   return(0);
    1635: }

    1639: /*@C
    1640:   DMPlexGetFullJoin - Get an array for the join of the set of points

    1642:   Not Collective

    1644:   Input Parameters:
    1645: + dm - The DMPlex object
    1646: . numPoints - The number of input points for the join
    1647: - points - The input points

    1649:   Output Parameters:
    1650: + numCoveredPoints - The number of points in the join
    1651: - coveredPoints - The points in the join

    1653:   Fortran Notes:
    1654:   Since it returns an array, this routine is only available in Fortran 90, and you must
    1655:   include petsc.h90 in your code.

    1657:   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.

    1659:   Level: intermediate

    1661: .keywords: mesh
    1662: .seealso: DMPlexGetJoin(), DMPlexRestoreJoin(), DMPlexGetMeet()
    1663: @*/
    1664: PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
    1665: {
    1666:   DM_Plex       *mesh = (DM_Plex*) dm->data;
    1667:   PetscInt      *offsets, **closures;
    1668:   PetscInt      *join[2];
    1669:   PetscInt       depth = 0, maxSize, joinSize = 0, i = 0;
    1670:   PetscInt       p, d, c, m;

    1679:   DMPlexGetDepth(dm, &depth);
    1680:   PetscMalloc(numPoints * sizeof(PetscInt*), &closures);
    1681:   PetscMemzero(closures,numPoints*sizeof(PetscInt*));
    1682:   DMGetWorkArray(dm, numPoints*(depth+2), PETSC_INT, &offsets);
    1683:   maxSize = PetscPowInt(mesh->maxSupportSize,depth+1);
    1684:   DMGetWorkArray(dm, maxSize, PETSC_INT, &join[0]);
    1685:   DMGetWorkArray(dm, maxSize, PETSC_INT, &join[1]);

    1687:   for (p = 0; p < numPoints; ++p) {
    1688:     PetscInt closureSize;

    1690:     DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]);

    1692:     offsets[p*(depth+2)+0] = 0;
    1693:     for (d = 0; d < depth+1; ++d) {
    1694:       PetscInt pStart, pEnd, i;

    1696:       DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);
    1697:       for (i = offsets[p*(depth+2)+d]; i < closureSize; ++i) {
    1698:         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
    1699:           offsets[p*(depth+2)+d+1] = i;
    1700:           break;
    1701:         }
    1702:       }
    1703:       if (i == closureSize) offsets[p*(depth+2)+d+1] = i;
    1704:     }
    1705:     if (offsets[p*(depth+2)+depth+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(depth+2)+depth+1], closureSize);
    1706:   }
    1707:   for (d = 0; d < depth+1; ++d) {
    1708:     PetscInt dof;

    1710:     /* Copy in support of first point */
    1711:     dof = offsets[d+1] - offsets[d];
    1712:     for (joinSize = 0; joinSize < dof; ++joinSize) {
    1713:       join[i][joinSize] = closures[0][(offsets[d]+joinSize)*2];
    1714:     }
    1715:     /* Check each successive cone */
    1716:     for (p = 1; p < numPoints && joinSize; ++p) {
    1717:       PetscInt newJoinSize = 0;

    1719:       dof = offsets[p*(depth+2)+d+1] - offsets[p*(depth+2)+d];
    1720:       for (c = 0; c < dof; ++c) {
    1721:         const PetscInt point = closures[p][(offsets[p*(depth+2)+d]+c)*2];

    1723:         for (m = 0; m < joinSize; ++m) {
    1724:           if (point == join[i][m]) {
    1725:             join[1-i][newJoinSize++] = point;
    1726:             break;
    1727:           }
    1728:         }
    1729:       }
    1730:       joinSize = newJoinSize;
    1731:       i        = 1-i;
    1732:     }
    1733:     if (joinSize) break;
    1734:   }
    1735:   *numCoveredPoints = joinSize;
    1736:   *coveredPoints    = join[i];
    1737:   for (p = 0; p < numPoints; ++p) {
    1738:     DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]);
    1739:   }
    1740:   PetscFree(closures);
    1741:   DMRestoreWorkArray(dm, numPoints*(depth+2), PETSC_INT, &offsets);
    1742:   DMRestoreWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1-i]);
    1743:   return(0);
    1744: }

    1748: /*@C
    1749:   DMPlexGetMeet - Get an array for the meet of the set of points

    1751:   Not Collective

    1753:   Input Parameters:
    1754: + dm - The DMPlex object
    1755: . numPoints - The number of input points for the meet
    1756: - points - The input points

    1758:   Output Parameters:
    1759: + numCoveredPoints - The number of points in the meet
    1760: - coveredPoints - The points in the meet

    1762:   Level: intermediate

    1764:   Note: Currently, this is restricted to a single level meet

    1766:   Fortran Notes:
    1767:   Since it returns an array, this routine is only available in Fortran 90, and you must
    1768:   include petsc.h90 in your code.

    1770:   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.

    1772: .keywords: mesh
    1773: .seealso: DMPlexRestoreMeet(), DMPlexGetJoin()
    1774: @*/
    1775: PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt **coveringPoints)
    1776: {
    1777:   DM_Plex       *mesh = (DM_Plex*) dm->data;
    1778:   PetscInt      *meet[2];
    1779:   PetscInt       meetSize, i = 0;
    1780:   PetscInt       dof, off, p, c, m;

    1788:   DMGetWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[0]);
    1789:   DMGetWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1]);
    1790:   /* Copy in cone of first point */
    1791:   PetscSectionGetDof(mesh->coneSection, points[0], &dof);
    1792:   PetscSectionGetOffset(mesh->coneSection, points[0], &off);
    1793:   for (meetSize = 0; meetSize < dof; ++meetSize) {
    1794:     meet[i][meetSize] = mesh->cones[off+meetSize];
    1795:   }
    1796:   /* Check each successive cone */
    1797:   for (p = 1; p < numPoints; ++p) {
    1798:     PetscInt newMeetSize = 0;

    1800:     PetscSectionGetDof(mesh->coneSection, points[p], &dof);
    1801:     PetscSectionGetOffset(mesh->coneSection, points[p], &off);
    1802:     for (c = 0; c < dof; ++c) {
    1803:       const PetscInt point = mesh->cones[off+c];

    1805:       for (m = 0; m < meetSize; ++m) {
    1806:         if (point == meet[i][m]) {
    1807:           meet[1-i][newMeetSize++] = point;
    1808:           break;
    1809:         }
    1810:       }
    1811:     }
    1812:     meetSize = newMeetSize;
    1813:     i        = 1-i;
    1814:   }
    1815:   *numCoveringPoints = meetSize;
    1816:   *coveringPoints    = meet[i];
    1817:   DMRestoreWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1-i]);
    1818:   return(0);
    1819: }

    1823: /*@C
    1824:   DMPlexRestoreMeet - Restore an array for the meet of the set of points

    1826:   Not Collective

    1828:   Input Parameters:
    1829: + dm - The DMPlex object
    1830: . numPoints - The number of input points for the meet
    1831: - points - The input points

    1833:   Output Parameters:
    1834: + numCoveredPoints - The number of points in the meet
    1835: - coveredPoints - The points in the meet

    1837:   Level: intermediate

    1839:   Fortran Notes:
    1840:   Since it returns an array, this routine is only available in Fortran 90, and you must
    1841:   include petsc.h90 in your code.

    1843:   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.

    1845: .keywords: mesh
    1846: .seealso: DMPlexGetMeet(), DMPlexGetFullMeet(), DMPlexGetJoin()
    1847: @*/
    1848: PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
    1849: {

    1855:   DMRestoreWorkArray(dm, 0, PETSC_INT, (void*) coveredPoints);
    1856:   return(0);
    1857: }

    1861: /*@C
    1862:   DMPlexGetFullMeet - Get an array for the meet of the set of points

    1864:   Not Collective

    1866:   Input Parameters:
    1867: + dm - The DMPlex object
    1868: . numPoints - The number of input points for the meet
    1869: - points - The input points

    1871:   Output Parameters:
    1872: + numCoveredPoints - The number of points in the meet
    1873: - coveredPoints - The points in the meet

    1875:   Level: intermediate

    1877:   Fortran Notes:
    1878:   Since it returns an array, this routine is only available in Fortran 90, and you must
    1879:   include petsc.h90 in your code.

    1881:   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.

    1883: .keywords: mesh
    1884: .seealso: DMPlexGetMeet(), DMPlexRestoreMeet(), DMPlexGetJoin()
    1885: @*/
    1886: PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
    1887: {
    1888:   DM_Plex       *mesh = (DM_Plex*) dm->data;
    1889:   PetscInt      *offsets, **closures;
    1890:   PetscInt      *meet[2];
    1891:   PetscInt       height = 0, maxSize, meetSize = 0, i = 0;
    1892:   PetscInt       p, h, c, m;

    1901:   DMPlexGetDepth(dm, &height);
    1902:   PetscMalloc(numPoints * sizeof(PetscInt*), &closures);
    1903:   DMGetWorkArray(dm, numPoints*(height+2), PETSC_INT, &offsets);
    1904:   maxSize = PetscPowInt(mesh->maxConeSize,height+1);
    1905:   DMGetWorkArray(dm, maxSize, PETSC_INT, &meet[0]);
    1906:   DMGetWorkArray(dm, maxSize, PETSC_INT, &meet[1]);

    1908:   for (p = 0; p < numPoints; ++p) {
    1909:     PetscInt closureSize;

    1911:     DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]);

    1913:     offsets[p*(height+2)+0] = 0;
    1914:     for (h = 0; h < height+1; ++h) {
    1915:       PetscInt pStart, pEnd, i;

    1917:       DMPlexGetHeightStratum(dm, h, &pStart, &pEnd);
    1918:       for (i = offsets[p*(height+2)+h]; i < closureSize; ++i) {
    1919:         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
    1920:           offsets[p*(height+2)+h+1] = i;
    1921:           break;
    1922:         }
    1923:       }
    1924:       if (i == closureSize) offsets[p*(height+2)+h+1] = i;
    1925:     }
    1926:     if (offsets[p*(height+2)+height+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(height+2)+height+1], closureSize);
    1927:   }
    1928:   for (h = 0; h < height+1; ++h) {
    1929:     PetscInt dof;

    1931:     /* Copy in cone of first point */
    1932:     dof = offsets[h+1] - offsets[h];
    1933:     for (meetSize = 0; meetSize < dof; ++meetSize) {
    1934:       meet[i][meetSize] = closures[0][(offsets[h]+meetSize)*2];
    1935:     }
    1936:     /* Check each successive cone */
    1937:     for (p = 1; p < numPoints && meetSize; ++p) {
    1938:       PetscInt newMeetSize = 0;

    1940:       dof = offsets[p*(height+2)+h+1] - offsets[p*(height+2)+h];
    1941:       for (c = 0; c < dof; ++c) {
    1942:         const PetscInt point = closures[p][(offsets[p*(height+2)+h]+c)*2];

    1944:         for (m = 0; m < meetSize; ++m) {
    1945:           if (point == meet[i][m]) {
    1946:             meet[1-i][newMeetSize++] = point;
    1947:             break;
    1948:           }
    1949:         }
    1950:       }
    1951:       meetSize = newMeetSize;
    1952:       i        = 1-i;
    1953:     }
    1954:     if (meetSize) break;
    1955:   }
    1956:   *numCoveredPoints = meetSize;
    1957:   *coveredPoints    = meet[i];
    1958:   for (p = 0; p < numPoints; ++p) {
    1959:     DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]);
    1960:   }
    1961:   PetscFree(closures);
    1962:   DMRestoreWorkArray(dm, numPoints*(height+2), PETSC_INT, &offsets);
    1963:   DMRestoreWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1-i]);
    1964:   return(0);
    1965: }

    1969: PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
    1970: {
    1971:   MPI_Comm       comm;

    1975:   PetscObjectGetComm((PetscObject)dm,&comm);
    1977:   switch (cellDim) {
    1978:   case 0:
    1979:     *numFaceVertices = 0;
    1980:     break;
    1981:   case 1:
    1982:     *numFaceVertices = 1;
    1983:     break;
    1984:   case 2:
    1985:     switch (numCorners) {
    1986:     case 3: /* triangle */
    1987:       *numFaceVertices = 2; /* Edge has 2 vertices */
    1988:       break;
    1989:     case 4: /* quadrilateral */
    1990:       *numFaceVertices = 2; /* Edge has 2 vertices */
    1991:       break;
    1992:     case 6: /* quadratic triangle, tri and quad cohesive Lagrange cells */
    1993:       *numFaceVertices = 3; /* Edge has 3 vertices */
    1994:       break;
    1995:     case 9: /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
    1996:       *numFaceVertices = 3; /* Edge has 3 vertices */
    1997:       break;
    1998:     default:
    1999:       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %d for dimension %d", numCorners, cellDim);
    2000:     }
    2001:     break;
    2002:   case 3:
    2003:     switch (numCorners) {
    2004:     case 4: /* tetradehdron */
    2005:       *numFaceVertices = 3; /* Face has 3 vertices */
    2006:       break;
    2007:     case 6: /* tet cohesive cells */
    2008:       *numFaceVertices = 4; /* Face has 4 vertices */
    2009:       break;
    2010:     case 8: /* hexahedron */
    2011:       *numFaceVertices = 4; /* Face has 4 vertices */
    2012:       break;
    2013:     case 9: /* tet cohesive Lagrange cells */
    2014:       *numFaceVertices = 6; /* Face has 6 vertices */
    2015:       break;
    2016:     case 10: /* quadratic tetrahedron */
    2017:       *numFaceVertices = 6; /* Face has 6 vertices */
    2018:       break;
    2019:     case 12: /* hex cohesive Lagrange cells */
    2020:       *numFaceVertices = 6; /* Face has 6 vertices */
    2021:       break;
    2022:     case 18: /* quadratic tet cohesive Lagrange cells */
    2023:       *numFaceVertices = 6; /* Face has 6 vertices */
    2024:       break;
    2025:     case 27: /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
    2026:       *numFaceVertices = 9; /* Face has 9 vertices */
    2027:       break;
    2028:     default:
    2029:       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %d for dimension %d", numCorners, cellDim);
    2030:     }
    2031:     break;
    2032:   default:
    2033:     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %d", cellDim);
    2034:   }
    2035:   return(0);
    2036: }

    2040: /* Trys to give the mesh a consistent orientation */
    2041: PetscErrorCode DMPlexOrient(DM dm)
    2042: {
    2043:   PetscBT        seenCells, flippedCells, seenFaces;
    2044:   PetscInt      *faceFIFO, fTop, fBottom;
    2045:   PetscInt       dim, h, cStart, cEnd, c, fStart, fEnd, face, maxConeSize, *revcone, *revconeO;

    2049:   /* Truth Table
    2050:      mismatch    flips   do action   mismatch   flipA ^ flipB   action
    2051:          F       0 flips     no         F             F           F
    2052:          F       1 flip      yes        F             T           T
    2053:          F       2 flips     no         T             F           T
    2054:          T       0 flips     yes        T             T           F
    2055:          T       1 flip      no
    2056:          T       2 flips     yes
    2057:   */
    2058:   DMPlexGetDimension(dm, &dim);
    2059:   DMPlexGetVTKCellHeight(dm, &h);
    2060:   DMPlexGetHeightStratum(dm, h,   &cStart, &cEnd);
    2061:   DMPlexGetHeightStratum(dm, h+1, &fStart, &fEnd);
    2062:   PetscBTCreate(cEnd - cStart, &seenCells);
    2063:   PetscBTMemzero(cEnd - cStart, seenCells);
    2064:   PetscBTCreate(cEnd - cStart, &flippedCells);
    2065:   PetscBTMemzero(cEnd - cStart, flippedCells);
    2066:   PetscBTCreate(fEnd - fStart, &seenFaces);
    2067:   PetscBTMemzero(fEnd - fStart, seenFaces);
    2068:   PetscMalloc((fEnd - fStart) * sizeof(PetscInt), &faceFIFO);
    2069:   fTop = fBottom = 0;
    2070:   /* Initialize FIFO with first cell */
    2071:   {
    2072:     const PetscInt *cone;
    2073:     PetscInt        coneSize;

    2075:     DMPlexGetConeSize(dm, cStart, &coneSize);
    2076:     DMPlexGetCone(dm, cStart, &cone);
    2077:     for (c = 0; c < coneSize; ++c) {
    2078:       faceFIFO[fBottom++] = cone[c];
    2079:       PetscBTSet(seenFaces, cone[c]-fStart);
    2080:     }
    2081:   }
    2082:   /* Consider each face in FIFO */
    2083:   while (fTop < fBottom) {
    2084:     const PetscInt *support, *coneA, *coneB, *coneOA, *coneOB;
    2085:     PetscInt        supportSize, coneSizeA, coneSizeB, posA = -1, posB = -1;
    2086:     PetscInt        seenA, flippedA, seenB, flippedB, mismatch;

    2088:     face = faceFIFO[fTop++];
    2089:     DMPlexGetSupportSize(dm, face, &supportSize);
    2090:     DMPlexGetSupport(dm, face, &support);
    2091:     if (supportSize < 2) continue;
    2092:     if (supportSize != 2) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Faces should separate only two cells, not %d", supportSize);
    2093:     seenA    = PetscBTLookup(seenCells,    support[0]-cStart);
    2094:     flippedA = PetscBTLookup(flippedCells, support[0]-cStart);
    2095:     seenB    = PetscBTLookup(seenCells,    support[1]-cStart);
    2096:     flippedB = PetscBTLookup(flippedCells, support[1]-cStart);

    2098:     DMPlexGetConeSize(dm, support[0], &coneSizeA);
    2099:     DMPlexGetConeSize(dm, support[1], &coneSizeB);
    2100:     DMPlexGetCone(dm, support[0], &coneA);
    2101:     DMPlexGetCone(dm, support[1], &coneB);
    2102:     DMPlexGetConeOrientation(dm, support[0], &coneOA);
    2103:     DMPlexGetConeOrientation(dm, support[1], &coneOB);
    2104:     for (c = 0; c < coneSizeA; ++c) {
    2105:       if (!PetscBTLookup(seenFaces, coneA[c]-fStart)) {
    2106:         faceFIFO[fBottom++] = coneA[c];
    2107:         PetscBTSet(seenFaces, coneA[c]-fStart);
    2108:       }
    2109:       if (coneA[c] == face) posA = c;
    2110:       if (fBottom > fEnd-fStart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Face %d was pushed exceeding capacity %d > %d", coneA[c], fBottom, fEnd-fStart);
    2111:     }
    2112:     if (posA < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d could not be located in cell %d", face, support[0]);
    2113:     for (c = 0; c < coneSizeB; ++c) {
    2114:       if (!PetscBTLookup(seenFaces, coneB[c]-fStart)) {
    2115:         faceFIFO[fBottom++] = coneB[c];
    2116:         PetscBTSet(seenFaces, coneB[c]-fStart);
    2117:       }
    2118:       if (coneB[c] == face) posB = c;
    2119:       if (fBottom > fEnd-fStart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Face %d was pushed exceeding capacity %d > %d", coneA[c], fBottom, fEnd-fStart);
    2120:     }
    2121:     if (posB < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d could not be located in cell %d", face, support[1]);

    2123:     if (dim == 1) {
    2124:       mismatch = posA == posB;
    2125:     } else {
    2126:       mismatch = coneOA[posA] == coneOB[posB];
    2127:     }

    2129:     if (mismatch ^ (flippedA ^ flippedB)) {
    2130:       if (seenA && seenB) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Previously seen cells %d and %d do not match: Fault mesh is non-orientable", support[0], support[1]);
    2131:       if (!seenA && !flippedA) {
    2132:         PetscBTSet(flippedCells, support[0]-cStart);
    2133:       } else if (!seenB && !flippedB) {
    2134:         PetscBTSet(flippedCells, support[1]-cStart);
    2135:       } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Inconsistent mesh orientation: Fault mesh is non-orientable");
    2136:     } else if (flippedA && flippedB) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Attempt to flip already flipped cell: Fault mesh is non-orientable");
    2137:     PetscBTSet(seenCells, support[0]-cStart);
    2138:     PetscBTSet(seenCells, support[1]-cStart);
    2139:   }

    2141:   DMPlexGetMaxSizes(dm, &maxConeSize, NULL);
    2142:   DMGetWorkArray(dm, maxConeSize, PETSC_INT, &revcone);
    2143:   DMGetWorkArray(dm, maxConeSize, PETSC_INT, &revconeO);
    2144:   for (c = cStart; c < cEnd; ++c) {
    2145:     const PetscInt *cone, *coneO;
    2146:     PetscInt        coneSize, faceSize, cp;

    2148:     if (!PetscBTLookup(flippedCells, c-cStart)) continue;
    2149:     DMPlexGetConeSize(dm, c, &coneSize);
    2150:     DMPlexGetCone(dm, c, &cone);
    2151:     DMPlexGetConeOrientation(dm, c, &coneO);
    2152:     for (cp = 0; cp < coneSize; ++cp) {
    2153:       const PetscInt rcp = coneSize-cp-1;

    2155:       DMPlexGetConeSize(dm, cone[rcp], &faceSize);
    2156:       revcone[cp]  = cone[rcp];
    2157:       revconeO[cp] = coneO[rcp] >= 0 ? -(faceSize-coneO[rcp]) : faceSize+coneO[rcp];
    2158:     }
    2159:     DMPlexSetCone(dm, c, revcone);
    2160:     DMPlexSetConeOrientation(dm, c, revconeO);
    2161:   }
    2162:   DMRestoreWorkArray(dm, maxConeSize, PETSC_INT, &revcone);
    2163:   DMRestoreWorkArray(dm, maxConeSize, PETSC_INT, &revconeO);
    2164:   PetscBTDestroy(&seenCells);
    2165:   PetscBTDestroy(&flippedCells);
    2166:   PetscBTDestroy(&seenFaces);
    2167:   PetscFree(faceFIFO);
    2168:   return(0);
    2169: }

    2173: static PetscErrorCode DMPlexGetAdjacencySingleLevel_Internal(DM dm, PetscInt p, PetscBool useClosure, const PetscInt *tmpClosure, PetscInt *adjSize, PetscInt adj[])
    2174: {
    2175:   const PetscInt *support = NULL;
    2176:   PetscInt        numAdj   = 0, maxAdjSize = *adjSize, supportSize, s;
    2177:   PetscErrorCode  ierr;

    2180:   if (useClosure) {
    2181:     DMPlexGetConeSize(dm, p, &supportSize);
    2182:     DMPlexGetCone(dm, p, &support);
    2183:     for (s = 0; s < supportSize; ++s) {
    2184:       const PetscInt *cone = NULL;
    2185:       PetscInt        coneSize, c, q;

    2187:       DMPlexGetSupportSize(dm, support[s], &coneSize);
    2188:       DMPlexGetSupport(dm, support[s], &cone);
    2189:       for (c = 0; c < coneSize; ++c) {
    2190:         for (q = 0; q < numAdj || (adj[numAdj++] = cone[c],0); ++q) {
    2191:           if (cone[c] == adj[q]) break;
    2192:         }
    2193:         if (numAdj > maxAdjSize) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid mesh exceeded adjacency allocation (%D)", maxAdjSize);
    2194:       }
    2195:     }
    2196:   } else {
    2197:     DMPlexGetSupportSize(dm, p, &supportSize);
    2198:     DMPlexGetSupport(dm, p, &support);
    2199:     for (s = 0; s < supportSize; ++s) {
    2200:       const PetscInt *cone = NULL;
    2201:       PetscInt        coneSize, c, q;

    2203:       DMPlexGetConeSize(dm, support[s], &coneSize);
    2204:       DMPlexGetCone(dm, support[s], &cone);
    2205:       for (c = 0; c < coneSize; ++c) {
    2206:         for (q = 0; q < numAdj || (adj[numAdj++] = cone[c],0); ++q) {
    2207:           if (cone[c] == adj[q]) break;
    2208:         }
    2209:         if (numAdj > maxAdjSize) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid mesh exceeded adjacency allocation (%D)", maxAdjSize);
    2210:       }
    2211:     }
    2212:   }
    2213:   *adjSize = numAdj;
    2214:   return(0);
    2215: }

    2219: PetscErrorCode DMPlexCreateNeighborCSR(DM dm, PetscInt cellHeight, PetscInt *numVertices, PetscInt **offsets, PetscInt **adjacency)
    2220: {
    2221:   const PetscInt maxFaceCases = 30;
    2222:   PetscInt       numFaceCases = 0;
    2223:   PetscInt       numFaceVertices[30]; /* maxFaceCases, C89 sucks sucks sucks */
    2224:   PetscInt      *off, *adj;
    2225:   PetscInt      *neighborCells, *tmpClosure;
    2226:   PetscInt       maxConeSize, maxSupportSize, maxClosure, maxNeighbors;
    2227:   PetscInt       dim, cellDim, depth = 0, faceDepth, cStart, cEnd, c, numCells, cell;

    2231:   /* For parallel partitioning, I think you have to communicate supports */
    2232:   DMPlexGetDimension(dm, &dim);
    2233:   cellDim = dim - cellHeight;
    2234:   DMPlexGetDepth(dm, &depth);
    2235:   DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);
    2236:   DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);
    2237:   if (cEnd - cStart == 0) {
    2238:     if (numVertices) *numVertices = 0;
    2239:     if (offsets)   *offsets   = NULL;
    2240:     if (adjacency) *adjacency = NULL;
    2241:     return(0);
    2242:   }
    2243:   numCells  = cEnd - cStart;
    2244:   faceDepth = depth - cellHeight;
    2245:   /* Setup face recognition */
    2246:   if (faceDepth == 1) {
    2247:     PetscInt cornersSeen[30] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; /* Could use PetscBT */

    2249:     for (c = cStart; c < cEnd; ++c) {
    2250:       PetscInt corners;

    2252:       DMPlexGetConeSize(dm, c, &corners);
    2253:       if (!cornersSeen[corners]) {
    2254:         PetscInt nFV;

    2256:         if (numFaceCases >= maxFaceCases) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Exceeded maximum number of face recognition cases");
    2257:         cornersSeen[corners] = 1;

    2259:         DMPlexGetNumFaceVertices(dm, cellDim, corners, &nFV);

    2261:         numFaceVertices[numFaceCases++] = nFV;
    2262:       }
    2263:     }
    2264:   }
    2265:   maxClosure   = 2*PetscMax(PetscPowInt(maxConeSize,depth+1),PetscPowInt(maxSupportSize,depth+1));
    2266:   maxNeighbors = PetscPowInt(maxConeSize,depth+1)*PetscPowInt(maxSupportSize,depth+1);
    2267:   PetscMalloc2(maxNeighbors,PetscInt,&neighborCells,maxClosure,PetscInt,&tmpClosure);
    2268:   PetscMalloc((numCells+1) * sizeof(PetscInt), &off);
    2269:   PetscMemzero(off, (numCells+1) * sizeof(PetscInt));
    2270:   /* Count neighboring cells */
    2271:   for (cell = cStart; cell < cEnd; ++cell) {
    2272:     PetscInt numNeighbors = maxNeighbors, n;

    2274:     DMPlexGetAdjacencySingleLevel_Internal(dm, cell, PETSC_TRUE, tmpClosure, &numNeighbors, neighborCells);
    2275:     /* Get meet with each cell, and check with recognizer (could optimize to check each pair only once) */
    2276:     for (n = 0; n < numNeighbors; ++n) {
    2277:       PetscInt        cellPair[2];
    2278:       PetscBool       found    = faceDepth > 1 ? PETSC_TRUE : PETSC_FALSE;
    2279:       PetscInt        meetSize = 0;
    2280:       const PetscInt *meet    = NULL;

    2282:       cellPair[0] = cell; cellPair[1] = neighborCells[n];
    2283:       if (cellPair[0] == cellPair[1]) continue;
    2284:       if (!found) {
    2285:         DMPlexGetMeet(dm, 2, cellPair, &meetSize, &meet);
    2286:         if (meetSize) {
    2287:           PetscInt f;

    2289:           for (f = 0; f < numFaceCases; ++f) {
    2290:             if (numFaceVertices[f] == meetSize) {
    2291:               found = PETSC_TRUE;
    2292:               break;
    2293:             }
    2294:           }
    2295:         }
    2296:         DMPlexRestoreMeet(dm, 2, cellPair, &meetSize, &meet);
    2297:       }
    2298:       if (found) ++off[cell-cStart+1];
    2299:     }
    2300:   }
    2301:   /* Prefix sum */
    2302:   for (cell = 1; cell <= numCells; ++cell) off[cell] += off[cell-1];

    2304:   if (adjacency) {
    2305:     PetscMalloc(off[numCells] * sizeof(PetscInt), &adj);
    2306:     /* Get neighboring cells */
    2307:     for (cell = cStart; cell < cEnd; ++cell) {
    2308:       PetscInt numNeighbors = maxNeighbors, n;
    2309:       PetscInt cellOffset   = 0;

    2311:       DMPlexGetAdjacencySingleLevel_Internal(dm, cell, PETSC_TRUE, tmpClosure, &numNeighbors, neighborCells);
    2312:       /* Get meet with each cell, and check with recognizer (could optimize to check each pair only once) */
    2313:       for (n = 0; n < numNeighbors; ++n) {
    2314:         PetscInt        cellPair[2];
    2315:         PetscBool       found    = faceDepth > 1 ? PETSC_TRUE : PETSC_FALSE;
    2316:         PetscInt        meetSize = 0;
    2317:         const PetscInt *meet    = NULL;

    2319:         cellPair[0] = cell; cellPair[1] = neighborCells[n];
    2320:         if (cellPair[0] == cellPair[1]) continue;
    2321:         if (!found) {
    2322:           DMPlexGetMeet(dm, 2, cellPair, &meetSize, &meet);
    2323:           if (meetSize) {
    2324:             PetscInt f;

    2326:             for (f = 0; f < numFaceCases; ++f) {
    2327:               if (numFaceVertices[f] == meetSize) {
    2328:                 found = PETSC_TRUE;
    2329:                 break;
    2330:               }
    2331:             }
    2332:           }
    2333:           DMPlexRestoreMeet(dm, 2, cellPair, &meetSize, &meet);
    2334:         }
    2335:         if (found) {
    2336:           adj[off[cell-cStart]+cellOffset] = neighborCells[n];
    2337:           ++cellOffset;
    2338:         }
    2339:       }
    2340:     }
    2341:   }
    2342:   PetscFree2(neighborCells,tmpClosure);
    2343:   if (numVertices) *numVertices = numCells;
    2344:   if (offsets)   *offsets   = off;
    2345:   if (adjacency) *adjacency = adj;
    2346:   return(0);
    2347: }

    2349: #if defined(PETSC_HAVE_CHACO)
    2350: #if defined(PETSC_HAVE_UNISTD_H)
    2351: #include <unistd.h>
    2352: #endif
    2353: /* Chaco does not have an include file */
    2354: PETSC_EXTERN int interface(int nvtxs, int *start, int *adjacency, int *vwgts,
    2355:                        float *ewgts, float *x, float *y, float *z, char *outassignname,
    2356:                        char *outfilename, short *assignment, int architecture, int ndims_tot,
    2357:                        int mesh_dims[3], double *goal, int global_method, int local_method,
    2358:                        int rqi_flag, int vmax, int ndims, double eigtol, long seed);

    2360: extern int FREE_GRAPH;

    2364: PetscErrorCode DMPlexPartition_Chaco(DM dm, PetscInt numVertices, PetscInt start[], PetscInt adjacency[], PetscSection *partSection, IS *partition)
    2365: {
    2366:   enum {DEFAULT_METHOD = 1, INERTIAL_METHOD = 3};
    2367:   MPI_Comm       comm;
    2368:   int            nvtxs          = numVertices; /* number of vertices in full graph */
    2369:   int           *vwgts          = NULL;   /* weights for all vertices */
    2370:   float         *ewgts          = NULL;   /* weights for all edges */
    2371:   float         *x              = NULL, *y = NULL, *z = NULL; /* coordinates for inertial method */
    2372:   char          *outassignname  = NULL;   /*  name of assignment output file */
    2373:   char          *outfilename    = NULL;   /* output file name */
    2374:   int            architecture   = 1;      /* 0 => hypercube, d => d-dimensional mesh */
    2375:   int            ndims_tot      = 0;      /* total number of cube dimensions to divide */
    2376:   int            mesh_dims[3];            /* dimensions of mesh of processors */
    2377:   double        *goal          = NULL;    /* desired set sizes for each set */
    2378:   int            global_method = 1;       /* global partitioning algorithm */
    2379:   int            local_method  = 1;       /* local partitioning algorithm */
    2380:   int            rqi_flag      = 0;       /* should I use RQI/Symmlq eigensolver? */
    2381:   int            vmax          = 200;     /* how many vertices to coarsen down to? */
    2382:   int            ndims         = 1;       /* number of eigenvectors (2^d sets) */
    2383:   double         eigtol        = 0.001;   /* tolerance on eigenvectors */
    2384:   long           seed          = 123636512; /* for random graph mutations */
    2385:   short int     *assignment;              /* Output partition */
    2386:   int            fd_stdout, fd_pipe[2];
    2387:   PetscInt      *points;
    2388:   PetscMPIInt    commSize;
    2389:   int            i, v, p;

    2393:   PetscObjectGetComm((PetscObject)dm,&comm);
    2394:   MPI_Comm_size(comm, &commSize);
    2395:   if (!numVertices) {
    2396:     PetscSectionCreate(comm, partSection);
    2397:     PetscSectionSetChart(*partSection, 0, commSize);
    2398:     PetscSectionSetUp(*partSection);
    2399:     ISCreateGeneral(comm, 0, NULL, PETSC_OWN_POINTER, partition);
    2400:     return(0);
    2401:   }
    2402:   FREE_GRAPH = 0;                         /* Do not let Chaco free my memory */
    2403:   for (i = 0; i < start[numVertices]; ++i) ++adjacency[i];

    2405:   if (global_method == INERTIAL_METHOD) {
    2406:     /* manager.createCellCoordinates(nvtxs, &x, &y, &z); */
    2407:     SETERRQ(comm, PETSC_ERR_SUP, "Inertial partitioning not yet supported");
    2408:   }
    2409:   mesh_dims[0] = commSize;
    2410:   mesh_dims[1] = 1;
    2411:   mesh_dims[2] = 1;
    2412:   PetscMalloc(nvtxs * sizeof(short int), &assignment);
    2413:   /* Chaco outputs to stdout. We redirect this to a buffer. */
    2414:   /* TODO: check error codes for UNIX calls */
    2415: #if defined(PETSC_HAVE_UNISTD_H)
    2416:   {
    2417:     int piperet;
    2418:     piperet = pipe(fd_pipe);
    2419:     if (piperet) SETERRQ(comm,PETSC_ERR_SYS,"Could not create pipe");
    2420:     fd_stdout = dup(1);
    2421:     close(1);
    2422:     dup2(fd_pipe[1], 1);
    2423:   }
    2424: #endif
    2425:   interface(nvtxs, (int*) start, (int*) adjacency, vwgts, ewgts, x, y, z, outassignname, outfilename,
    2426:                    assignment, architecture, ndims_tot, mesh_dims, goal, global_method, local_method, rqi_flag,
    2427:                    vmax, ndims, eigtol, seed);
    2428: #if defined(PETSC_HAVE_UNISTD_H)
    2429:   {
    2430:     char msgLog[10000];
    2431:     int  count;

    2433:     fflush(stdout);
    2434:     count = read(fd_pipe[0], msgLog, (10000-1)*sizeof(char));
    2435:     if (count < 0) count = 0;
    2436:     msgLog[count] = 0;
    2437:     close(1);
    2438:     dup2(fd_stdout, 1);
    2439:     close(fd_stdout);
    2440:     close(fd_pipe[0]);
    2441:     close(fd_pipe[1]);
    2442:     if (ierr) SETERRQ1(comm, PETSC_ERR_LIB, "Error in Chaco library: %s", msgLog);
    2443:   }
    2444: #endif
    2445:   /* Convert to PetscSection+IS */
    2446:   PetscSectionCreate(comm, partSection);
    2447:   PetscSectionSetChart(*partSection, 0, commSize);
    2448:   for (v = 0; v < nvtxs; ++v) {
    2449:     PetscSectionAddDof(*partSection, assignment[v], 1);
    2450:   }
    2451:   PetscSectionSetUp(*partSection);
    2452:   PetscMalloc(nvtxs * sizeof(PetscInt), &points);
    2453:   for (p = 0, i = 0; p < commSize; ++p) {
    2454:     for (v = 0; v < nvtxs; ++v) {
    2455:       if (assignment[v] == p) points[i++] = v;
    2456:     }
    2457:   }
    2458:   if (i != nvtxs) SETERRQ2(comm, PETSC_ERR_PLIB, "Number of points %D should be %D", i, nvtxs);
    2459:   ISCreateGeneral(comm, nvtxs, points, PETSC_OWN_POINTER, partition);
    2460:   if (global_method == INERTIAL_METHOD) {
    2461:     /* manager.destroyCellCoordinates(nvtxs, &x, &y, &z); */
    2462:   }
    2463:   PetscFree(assignment);
    2464:   for (i = 0; i < start[numVertices]; ++i) --adjacency[i];
    2465:   return(0);
    2466: }
    2467: #endif

    2469: #if defined(PETSC_HAVE_PARMETIS)
    2472: PetscErrorCode DMPlexPartition_ParMetis(DM dm, PetscInt numVertices, PetscInt start[], PetscInt adjacency[], PetscSection *partSection, IS *partition)
    2473: {
    2475:   SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "ParMetis not yet supported");
    2476:   return(0);
    2477: }
    2478: #endif

    2482: /* Expand the partition by BFS on the adjacency graph */
    2483: PetscErrorCode DMPlexEnlargePartition(DM dm, const PetscInt start[], const PetscInt adjacency[], PetscSection origPartSection, IS origPartition, PetscSection *partSection, IS *partition)
    2484: {
    2485:   PetscHashI      h;
    2486:   const PetscInt *points;
    2487:   PetscInt      **tmpPoints, *newPoints, totPoints = 0;
    2488:   PetscInt        pStart, pEnd, part, q;
    2489:   PetscErrorCode  ierr;

    2492:   PetscHashICreate(h);
    2493:   PetscSectionCreate(PetscObjectComm((PetscObject)dm), partSection);
    2494:   PetscSectionGetChart(origPartSection, &pStart, &pEnd);
    2495:   PetscSectionSetChart(*partSection, pStart, pEnd);
    2496:   ISGetIndices(origPartition, &points);
    2497:   PetscMalloc((pEnd - pStart) * sizeof(PetscInt*), &tmpPoints);
    2498:   for (part = pStart; part < pEnd; ++part) {
    2499:     PetscInt numPoints, nP, numNewPoints, off, p, n = 0;

    2501:     PetscHashIClear(h);
    2502:     PetscSectionGetDof(origPartSection, part, &numPoints);
    2503:     PetscSectionGetOffset(origPartSection, part, &off);
    2504:     /* Add all existing points to h */
    2505:     for (p = 0; p < numPoints; ++p) {
    2506:       const PetscInt point = points[off+p];
    2507:       PetscHashIAdd(h, point, 1);
    2508:     }
    2509:     PetscHashISize(h, nP);
    2510:     if (nP != numPoints) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Invalid partition has %d points, but only %d were unique", numPoints, nP);
    2511:     /* Add all points in next BFS level */
    2512:     /*   TODO We are brute forcing here, but could check the adjacency size to find the boundary */
    2513:     for (p = 0; p < numPoints; ++p) {
    2514:       const PetscInt point = points[off+p];
    2515:       PetscInt       s     = start[point], e = start[point+1], a;

    2517:       for (a = s; a < e; ++a) PetscHashIAdd(h, adjacency[a], 1);
    2518:     }
    2519:     PetscHashISize(h, numNewPoints);
    2520:     PetscSectionSetDof(*partSection, part, numNewPoints);
    2521:     PetscMalloc(numNewPoints * sizeof(PetscInt), &tmpPoints[part]);
    2522:     if (numNewPoints) PetscHashIGetKeys(h, n, tmpPoints[part]); /* Should not need this conditional */
    2523:     totPoints += numNewPoints;
    2524:   }
    2525:   ISRestoreIndices(origPartition, &points);
    2526:   PetscHashIDestroy(h);
    2527:   PetscSectionSetUp(*partSection);
    2528:   PetscMalloc(totPoints * sizeof(PetscInt), &newPoints);
    2529:   for (part = pStart, q = 0; part < pEnd; ++part) {
    2530:     PetscInt numPoints, p;

    2532:     PetscSectionGetDof(*partSection, part, &numPoints);
    2533:     for (p = 0; p < numPoints; ++p, ++q) newPoints[q] = tmpPoints[part][p];
    2534:     PetscFree(tmpPoints[part]);
    2535:   }
    2536:   PetscFree(tmpPoints);
    2537:   ISCreateGeneral(PetscObjectComm((PetscObject)dm), totPoints, newPoints, PETSC_OWN_POINTER, partition);
    2538:   return(0);
    2539: }

    2543: /*
    2544:   DMPlexCreatePartition - Create a non-overlapping partition of the points at the given height

    2546:   Collective on DM

    2548:   Input Parameters:
    2549:   + dm - The DM
    2550:   . height - The height for points in the partition
    2551:   - enlarge - Expand each partition with neighbors

    2553:   Output Parameters:
    2554:   + partSection - The PetscSection giving the division of points by partition
    2555:   . partition - The list of points by partition
    2556:   . origPartSection - If enlarge is true, the PetscSection giving the division of points before enlarging by partition, otherwise NULL
    2557:   - origPartition - If enlarge is true, the list of points before enlarging by partition, otherwise NULL

    2559:   Level: developer

    2561: .seealso DMPlexDistribute()
    2562: */
    2563: PetscErrorCode DMPlexCreatePartition(DM dm, PetscInt height, PetscBool enlarge, PetscSection *partSection, IS *partition, PetscSection *origPartSection, IS *origPartition)
    2564: {
    2565:   PetscMPIInt    size;

    2569:   MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);

    2571:   *origPartSection = NULL;
    2572:   *origPartition   = NULL;
    2573:   if (size == 1) {
    2574:     PetscInt *points;
    2575:     PetscInt  cStart, cEnd, c;

    2577:     DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);
    2578:     PetscSectionCreate(PetscObjectComm((PetscObject)dm), partSection);
    2579:     PetscSectionSetChart(*partSection, 0, size);
    2580:     PetscSectionSetDof(*partSection, 0, cEnd-cStart);
    2581:     PetscSectionSetUp(*partSection);
    2582:     PetscMalloc((cEnd - cStart) * sizeof(PetscInt), &points);
    2583:     for (c = cStart; c < cEnd; ++c) points[c] = c;
    2584:     ISCreateGeneral(PetscObjectComm((PetscObject)dm), cEnd-cStart, points, PETSC_OWN_POINTER, partition);
    2585:     return(0);
    2586:   }
    2587:   if (height == 0) {
    2588:     PetscInt  numVertices;
    2589:     PetscInt *start     = NULL;
    2590:     PetscInt *adjacency = NULL;

    2592:     DMPlexCreateNeighborCSR(dm, 0, &numVertices, &start, &adjacency);
    2593:     if (1) {
    2594: #if defined(PETSC_HAVE_CHACO)
    2595:       DMPlexPartition_Chaco(dm, numVertices, start, adjacency, partSection, partition);
    2596: #endif
    2597:     } else {
    2598: #if defined(PETSC_HAVE_PARMETIS)
    2599:       DMPlexPartition_ParMetis(dm, numVertices, start, adjacency, partSection, partition);
    2600: #endif
    2601:     }
    2602:     if (enlarge) {
    2603:       *origPartSection = *partSection;
    2604:       *origPartition   = *partition;

    2606:       DMPlexEnlargePartition(dm, start, adjacency, *origPartSection, *origPartition, partSection, partition);
    2607:     }
    2608:     PetscFree(start);
    2609:     PetscFree(adjacency);
    2610: # if 0
    2611:   } else if (height == 1) {
    2612:     /* Build the dual graph for faces and partition the hypergraph */
    2613:     PetscInt numEdges;

    2615:     buildFaceCSRV(mesh, mesh->getFactory()->getNumbering(mesh, mesh->depth()-1), &numEdges, &start, &adjacency, GraphPartitioner::zeroBase());
    2616:     GraphPartitioner().partition(numEdges, start, adjacency, partition, manager);
    2617:     destroyCSR(numEdges, start, adjacency);
    2618: #endif
    2619:   } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid partition height %D", height);
    2620:   return(0);
    2621: }

    2625: PetscErrorCode DMPlexCreatePartitionClosure(DM dm, PetscSection pointSection, IS pointPartition, PetscSection *section, IS *partition)
    2626: {
    2627:   /* const PetscInt  height = 0; */
    2628:   const PetscInt *partArray;
    2629:   PetscInt       *allPoints, *packPoints;
    2630:   PetscInt        rStart, rEnd, rank, pStart, pEnd, newSize;
    2631:   PetscErrorCode  ierr;
    2632:   PetscBT         bt;
    2633:   PetscSegBuffer  segpack,segpart;

    2636:   PetscSectionGetChart(pointSection, &rStart, &rEnd);
    2637:   ISGetIndices(pointPartition, &partArray);
    2638:   PetscSectionCreate(PetscObjectComm((PetscObject)dm), section);
    2639:   PetscSectionSetChart(*section, rStart, rEnd);
    2640:   DMPlexGetChart(dm,&pStart,&pEnd);
    2641:   PetscBTCreate(pEnd-pStart,&bt);
    2642:   PetscSegBufferCreate(sizeof(PetscInt),1000,&segpack);
    2643:   PetscSegBufferCreate(sizeof(PetscInt),1000,&segpart);
    2644:   for (rank = rStart; rank < rEnd; ++rank) {
    2645:     PetscInt partSize = 0, numPoints, offset, p, *PETSC_RESTRICT placePoints;

    2647:     PetscSectionGetDof(pointSection, rank, &numPoints);
    2648:     PetscSectionGetOffset(pointSection, rank, &offset);
    2649:     for (p = 0; p < numPoints; ++p) {
    2650:       PetscInt  point   = partArray[offset+p], closureSize, c;
    2651:       PetscInt *closure = NULL;

    2653:       /* TODO Include support for height > 0 case */
    2654:       DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);
    2655:       for (c=0; c<closureSize; c++) {
    2656:         PetscInt cpoint = closure[c*2];
    2657:         if (!PetscBTLookupSet(bt,cpoint-pStart)) {
    2658:           PetscInt *PETSC_RESTRICT pt;
    2659:           partSize++;
    2660:           PetscSegBufferGetInts(segpart,1,&pt);
    2661:           *pt = cpoint;
    2662:         }
    2663:       }
    2664:       DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);
    2665:     }
    2666:     PetscSectionSetDof(*section, rank, partSize);
    2667:     PetscSegBufferGetInts(segpack,partSize,&placePoints);
    2668:     PetscSegBufferExtractTo(segpart,placePoints);
    2669:     PetscSortInt(partSize,placePoints);
    2670:     for (p=0; p<partSize; p++) {PetscBTClear(bt,placePoints[p]-pStart);}
    2671:   }
    2672:   PetscBTDestroy(&bt);
    2673:   PetscSegBufferDestroy(&segpart);

    2675:   PetscSectionSetUp(*section);
    2676:   PetscSectionGetStorageSize(*section, &newSize);
    2677:   PetscMalloc(newSize * sizeof(PetscInt), &allPoints);

    2679:   PetscSegBufferExtractInPlace(segpack,&packPoints);
    2680:   for (rank = rStart; rank < rEnd; ++rank) {
    2681:     PetscInt numPoints, offset;

    2683:     PetscSectionGetDof(*section, rank, &numPoints);
    2684:     PetscSectionGetOffset(*section, rank, &offset);
    2685:     PetscMemcpy(&allPoints[offset], packPoints, numPoints * sizeof(PetscInt));
    2686:     packPoints += numPoints;
    2687:   }

    2689:   PetscSegBufferDestroy(&segpack);
    2690:   ISRestoreIndices(pointPartition, &partArray);
    2691:   ISCreateGeneral(PetscObjectComm((PetscObject)dm), newSize, allPoints, PETSC_OWN_POINTER, partition);
    2692:   return(0);
    2693: }

    2697: /*
    2698:   Input Parameters:
    2699: . originalSection
    2700: , originalVec

    2702:   Output Parameters:
    2703: . newSection
    2704: . newVec
    2705: */
    2706: PetscErrorCode DMPlexDistributeField(DM dm, PetscSF pointSF, PetscSection originalSection, Vec originalVec, PetscSection newSection, Vec newVec)
    2707: {
    2708:   PetscSF        fieldSF;
    2709:   PetscInt      *remoteOffsets, fieldSize;
    2710:   PetscScalar   *originalValues, *newValues;

    2714:   PetscSFDistributeSection(pointSF, originalSection, &remoteOffsets, newSection);

    2716:   PetscSectionGetStorageSize(newSection, &fieldSize);
    2717:   VecSetSizes(newVec, fieldSize, PETSC_DETERMINE);
    2718:   VecSetFromOptions(newVec);

    2720:   VecGetArray(originalVec, &originalValues);
    2721:   VecGetArray(newVec, &newValues);
    2722:   PetscSFCreateSectionSF(pointSF, originalSection, remoteOffsets, newSection, &fieldSF);
    2723:   PetscSFBcastBegin(fieldSF, MPIU_SCALAR, originalValues, newValues);
    2724:   PetscSFBcastEnd(fieldSF, MPIU_SCALAR, originalValues, newValues);
    2725:   PetscSFDestroy(&fieldSF);
    2726:   VecRestoreArray(newVec, &newValues);
    2727:   VecRestoreArray(originalVec, &originalValues);
    2728:   return(0);
    2729: }

    2733: /*@C
    2734:   DMPlexDistribute - Distributes the mesh and any associated sections.

    2736:   Not Collective

    2738:   Input Parameter:
    2739: + dm  - The original DMPlex object
    2740: . partitioner - The partitioning package, or NULL for the default
    2741: - overlap - The overlap of partitions, 0 is the default

    2743:   Output Parameter:
    2744: . parallelMesh - The distributed DMPlex object, or NULL

    2746:   Note: If the mesh was not distributed, the return value is NULL

    2748:   Level: intermediate

    2750: .keywords: mesh, elements
    2751: .seealso: DMPlexCreate(), DMPlexDistributeByFace()
    2752: @*/
    2753: PetscErrorCode DMPlexDistribute(DM dm, const char partitioner[], PetscInt overlap, DM *dmParallel)
    2754: {
    2755:   DM_Plex               *mesh   = (DM_Plex*) dm->data, *pmesh;
    2756:   MPI_Comm               comm;
    2757:   const PetscInt         height = 0;
    2758:   PetscInt               dim, numRemoteRanks;
    2759:   IS                     origCellPart,        cellPart,        part;
    2760:   PetscSection           origCellPartSection, cellPartSection, partSection;
    2761:   PetscSFNode           *remoteRanks;
    2762:   PetscSF                partSF, pointSF, coneSF;
    2763:   ISLocalToGlobalMapping renumbering;
    2764:   PetscSection           originalConeSection, newConeSection;
    2765:   PetscInt              *remoteOffsets;
    2766:   PetscInt              *cones, *newCones, newConesSize;
    2767:   PetscBool              flg;
    2768:   PetscMPIInt            rank, numProcs, p;
    2769:   PetscErrorCode         ierr;

    2775:   PetscLogEventBegin(DMPLEX_Distribute,dm,0,0,0);
    2776:   PetscObjectGetComm((PetscObject)dm,&comm);
    2777:   MPI_Comm_rank(comm, &rank);
    2778:   MPI_Comm_size(comm, &numProcs);

    2780:   *dmParallel = NULL;
    2781:   if (numProcs == 1) return(0);

    2783:   DMPlexGetDimension(dm, &dim);
    2784:   /* Create cell partition - We need to rewrite to use IS, use the MatPartition stuff */
    2785:   if (overlap > 1) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Overlap > 1 not yet implemented");
    2786:   DMPlexCreatePartition(dm, height, overlap > 0 ? PETSC_TRUE : PETSC_FALSE, &cellPartSection, &cellPart, &origCellPartSection, &origCellPart);
    2787:   /* Create SF assuming a serial partition for all processes: Could check for IS length here */
    2788:   if (!rank) numRemoteRanks = numProcs;
    2789:   else       numRemoteRanks = 0;
    2790:   PetscMalloc(numRemoteRanks * sizeof(PetscSFNode), &remoteRanks);
    2791:   for (p = 0; p < numRemoteRanks; ++p) {
    2792:     remoteRanks[p].rank  = p;
    2793:     remoteRanks[p].index = 0;
    2794:   }
    2795:   PetscSFCreate(comm, &partSF);
    2796:   PetscSFSetGraph(partSF, 1, numRemoteRanks, NULL, PETSC_OWN_POINTER, remoteRanks, PETSC_OWN_POINTER);
    2797:   PetscOptionsHasName(((PetscObject) dm)->prefix, "-partition_view", &flg);
    2798:   if (flg) {
    2799:     PetscPrintf(comm, "Cell Partition:\n");
    2800:     PetscSectionView(cellPartSection, PETSC_VIEWER_STDOUT_WORLD);
    2801:     ISView(cellPart, NULL);
    2802:     if (origCellPart) {
    2803:       PetscPrintf(comm, "Original Cell Partition:\n");
    2804:       PetscSectionView(origCellPartSection, PETSC_VIEWER_STDOUT_WORLD);
    2805:       ISView(origCellPart, NULL);
    2806:     }
    2807:     PetscSFView(partSF, NULL);
    2808:   }
    2809:   /* Close the partition over the mesh */
    2810:   DMPlexCreatePartitionClosure(dm, cellPartSection, cellPart, &partSection, &part);
    2811:   ISDestroy(&cellPart);
    2812:   PetscSectionDestroy(&cellPartSection);
    2813:   /* Create new mesh */
    2814:   DMPlexCreate(comm, dmParallel);
    2815:   DMPlexSetDimension(*dmParallel, dim);
    2816:   PetscObjectSetName((PetscObject) *dmParallel, "Parallel Mesh");
    2817:   pmesh = (DM_Plex*) (*dmParallel)->data;
    2818:   /* Distribute sieve points and the global point numbering (replaces creating remote bases) */
    2819:   PetscSFConvertPartition(partSF, partSection, part, &renumbering, &pointSF);
    2820:   if (flg) {
    2821:     PetscPrintf(comm, "Point Partition:\n");
    2822:     PetscSectionView(partSection, PETSC_VIEWER_STDOUT_WORLD);
    2823:     ISView(part, NULL);
    2824:     PetscSFView(pointSF, NULL);
    2825:     PetscPrintf(comm, "Point Renumbering after partition:\n");
    2826:     ISLocalToGlobalMappingView(renumbering, NULL);
    2827:   }
    2828:   /* Distribute cone section */
    2829:   DMPlexGetConeSection(dm, &originalConeSection);
    2830:   DMPlexGetConeSection(*dmParallel, &newConeSection);
    2831:   PetscSFDistributeSection(pointSF, originalConeSection, &remoteOffsets, newConeSection);
    2832:   DMSetUp(*dmParallel);
    2833:   {
    2834:     PetscInt pStart, pEnd, p;

    2836:     PetscSectionGetChart(newConeSection, &pStart, &pEnd);
    2837:     for (p = pStart; p < pEnd; ++p) {
    2838:       PetscInt coneSize;
    2839:       PetscSectionGetDof(newConeSection, p, &coneSize);
    2840:       pmesh->maxConeSize = PetscMax(pmesh->maxConeSize, coneSize);
    2841:     }
    2842:   }
    2843:   /* Communicate and renumber cones */
    2844:   PetscSFCreateSectionSF(pointSF, originalConeSection, remoteOffsets, newConeSection, &coneSF);
    2845:   DMPlexGetCones(dm, &cones);
    2846:   DMPlexGetCones(*dmParallel, &newCones);
    2847:   PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);
    2848:   PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);
    2849:   PetscSectionGetStorageSize(newConeSection, &newConesSize);
    2850:   ISGlobalToLocalMappingApply(renumbering, IS_GTOLM_MASK, newConesSize, newCones, NULL, newCones);
    2851:   PetscOptionsHasName(((PetscObject) dm)->prefix, "-cones_view", &flg);
    2852:   if (flg) {
    2853:     PetscPrintf(comm, "Serial Cone Section:\n");
    2854:     PetscSectionView(originalConeSection, PETSC_VIEWER_STDOUT_WORLD);
    2855:     PetscPrintf(comm, "Parallel Cone Section:\n");
    2856:     PetscSectionView(newConeSection, PETSC_VIEWER_STDOUT_WORLD);
    2857:     PetscSFView(coneSF, NULL);
    2858:   }
    2859:   DMPlexGetConeOrientations(dm, &cones);
    2860:   DMPlexGetConeOrientations(*dmParallel, &newCones);
    2861:   PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);
    2862:   PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);
    2863:   PetscSFDestroy(&coneSF);
    2864:   /* Create supports and stratify sieve */
    2865:   {
    2866:     PetscInt pStart, pEnd;

    2868:     PetscSectionGetChart(pmesh->coneSection, &pStart, &pEnd);
    2869:     PetscSectionSetChart(pmesh->supportSection, pStart, pEnd);
    2870:   }
    2871:   DMPlexSymmetrize(*dmParallel);
    2872:   DMPlexStratify(*dmParallel);
    2873:   /* Distribute Coordinates */
    2874:   {
    2875:     PetscSection originalCoordSection, newCoordSection;
    2876:     Vec          originalCoordinates, newCoordinates;
    2877:     const char  *name;

    2879:     DMPlexGetCoordinateSection(dm, &originalCoordSection);
    2880:     DMPlexGetCoordinateSection(*dmParallel, &newCoordSection);
    2881:     DMGetCoordinatesLocal(dm, &originalCoordinates);
    2882:     VecCreate(comm, &newCoordinates);
    2883:     PetscObjectGetName((PetscObject) originalCoordinates, &name);
    2884:     PetscObjectSetName((PetscObject) newCoordinates, name);

    2886:     DMPlexDistributeField(dm, pointSF, originalCoordSection, originalCoordinates, newCoordSection, newCoordinates);
    2887:     DMSetCoordinatesLocal(*dmParallel, newCoordinates);
    2888:     VecDestroy(&newCoordinates);
    2889:   }
    2890:   /* Distribute labels */
    2891:   {
    2892:     DMLabel  next      = mesh->labels, newNext = pmesh->labels;
    2893:     PetscInt numLabels = 0, l;

    2895:     /* Bcast number of labels */
    2896:     while (next) {
    2897:       ++numLabels; next = next->next;
    2898:     }
    2899:     MPI_Bcast(&numLabels, 1, MPIU_INT, 0, comm);
    2900:     next = mesh->labels;
    2901:     for (l = 0; l < numLabels; ++l) {
    2902:       DMLabel         newLabel;
    2903:       const PetscInt *partArray;
    2904:       char           *name;
    2905:       PetscInt       *stratumSizes = NULL, *points = NULL;
    2906:       PetscMPIInt    *sendcnts     = NULL, *offsets = NULL, *displs = NULL;
    2907:       PetscInt        nameSize, s, p;
    2908:       PetscBool       isdepth;
    2909:       size_t          len = 0;

    2911:       /* Bcast name (could filter for no points) */
    2912:       if (!rank) {PetscStrlen(next->name, &len);}
    2913:       nameSize = len;
    2914:       MPI_Bcast(&nameSize, 1, MPIU_INT, 0, comm);
    2915:       PetscMalloc(nameSize+1, &name);
    2916:       if (!rank) {PetscMemcpy(name, next->name, nameSize+1);}
    2917:       MPI_Bcast(name, nameSize+1, MPI_CHAR, 0, comm);
    2918:       PetscStrcmp(name, "depth", &isdepth);
    2919:       if (isdepth) {            /* skip because "depth" is not distributed */
    2920:         PetscFree(name);
    2921:         if (!rank) next = next->next;
    2922:         continue;
    2923:       }
    2924:       PetscNew(struct _n_DMLabel, &newLabel);
    2925:       newLabel->name = name;
    2926:       /* Bcast numStrata (could filter for no points in stratum) */
    2927:       if (!rank) newLabel->numStrata = next->numStrata;
    2928:       MPI_Bcast(&newLabel->numStrata, 1, MPIU_INT, 0, comm);
    2929:       PetscMalloc3(newLabel->numStrata,PetscInt,&newLabel->stratumValues,
    2930:                           newLabel->numStrata,PetscInt,&newLabel->stratumSizes,
    2931:                           newLabel->numStrata+1,PetscInt,&newLabel->stratumOffsets);
    2932:       /* Bcast stratumValues (could filter for no points in stratum) */
    2933:       if (!rank) {PetscMemcpy(newLabel->stratumValues, next->stratumValues, next->numStrata * sizeof(PetscInt));}
    2934:       MPI_Bcast(newLabel->stratumValues, newLabel->numStrata, MPIU_INT, 0, comm);
    2935:       /* Find size on each process and Scatter */
    2936:       if (!rank) {
    2937:         ISGetIndices(part, &partArray);
    2938:         PetscMalloc(numProcs*next->numStrata * sizeof(PetscInt), &stratumSizes);
    2939:         PetscMemzero(stratumSizes, numProcs*next->numStrata * sizeof(PetscInt));
    2940:         for (s = 0; s < next->numStrata; ++s) {
    2941:           for (p = next->stratumOffsets[s]; p < next->stratumOffsets[s]+next->stratumSizes[s]; ++p) {
    2942:             const PetscInt point = next->points[p];
    2943:             PetscInt       proc;

    2945:             for (proc = 0; proc < numProcs; ++proc) {
    2946:               PetscInt dof, off, pPart;

    2948:               PetscSectionGetDof(partSection, proc, &dof);
    2949:               PetscSectionGetOffset(partSection, proc, &off);
    2950:               for (pPart = off; pPart < off+dof; ++pPart) {
    2951:                 if (partArray[pPart] == point) {
    2952:                   ++stratumSizes[proc*next->numStrata+s];
    2953:                   break;
    2954:                 }
    2955:               }
    2956:             }
    2957:           }
    2958:         }
    2959:         ISRestoreIndices(part, &partArray);
    2960:       }
    2961:       MPI_Scatter(stratumSizes, newLabel->numStrata, MPIU_INT, newLabel->stratumSizes, newLabel->numStrata, MPIU_INT, 0, comm);
    2962:       /* Calculate stratumOffsets */
    2963:       newLabel->stratumOffsets[0] = 0;
    2964:       for (s = 0; s < newLabel->numStrata; ++s) {
    2965:         newLabel->stratumOffsets[s+1] = newLabel->stratumSizes[s] + newLabel->stratumOffsets[s];
    2966:       }
    2967:       /* Pack points and Scatter */
    2968:       if (!rank) {
    2969:         PetscMalloc3(numProcs,PetscMPIInt,&sendcnts,numProcs,PetscMPIInt,&offsets,numProcs+1,PetscMPIInt,&displs);
    2970:         displs[0] = 0;
    2971:         for (p = 0; p < numProcs; ++p) {
    2972:           sendcnts[p] = 0;
    2973:           for (s = 0; s < next->numStrata; ++s) {
    2974:             sendcnts[p] += stratumSizes[p*next->numStrata+s];
    2975:           }
    2976:           offsets[p]  = displs[p];
    2977:           displs[p+1] = displs[p] + sendcnts[p];
    2978:         }
    2979:         PetscMalloc(displs[numProcs] * sizeof(PetscInt), &points);
    2980:         for (s = 0; s < next->numStrata; ++s) {
    2981:           for (p = next->stratumOffsets[s]; p < next->stratumOffsets[s]+next->stratumSizes[s]; ++p) {
    2982:             const PetscInt point = next->points[p];
    2983:             PetscInt       proc;

    2985:             for (proc = 0; proc < numProcs; ++proc) {
    2986:               PetscInt dof, off, pPart;

    2988:               PetscSectionGetDof(partSection, proc, &dof);
    2989:               PetscSectionGetOffset(partSection, proc, &off);
    2990:               for (pPart = off; pPart < off+dof; ++pPart) {
    2991:                 if (partArray[pPart] == point) {
    2992:                   points[offsets[proc]++] = point;
    2993:                   break;
    2994:                 }
    2995:               }
    2996:             }
    2997:           }
    2998:         }
    2999:       }
    3000:       PetscMalloc(newLabel->stratumOffsets[newLabel->numStrata] * sizeof(PetscInt), &newLabel->points);
    3001:       MPI_Scatterv(points, sendcnts, displs, MPIU_INT, newLabel->points, newLabel->stratumOffsets[newLabel->numStrata], MPIU_INT, 0, comm);
    3002:       PetscFree(points);
    3003:       PetscFree3(sendcnts,offsets,displs);
    3004:       PetscFree(stratumSizes);
    3005:       /* Renumber points */
    3006:       ISGlobalToLocalMappingApply(renumbering, IS_GTOLM_MASK, newLabel->stratumOffsets[newLabel->numStrata], newLabel->points, NULL, newLabel->points);
    3007:       /* Sort points */
    3008:       for (s = 0; s < newLabel->numStrata; ++s) {
    3009:         PetscSortInt(newLabel->stratumSizes[s], &newLabel->points[newLabel->stratumOffsets[s]]);
    3010:       }
    3011:       /* Insert into list */
    3012:       if (newNext) newNext->next = newLabel;
    3013:       else pmesh->labels = newLabel;
    3014:       newNext = newLabel;
    3015:       if (!rank) next = next->next;
    3016:     }
    3017:   }
    3018:   /* Cleanup Partition */
    3019:   ISLocalToGlobalMappingDestroy(&renumbering);
    3020:   PetscSFDestroy(&partSF);
    3021:   PetscSectionDestroy(&partSection);
    3022:   ISDestroy(&part);
    3023:   /* Create point SF for parallel mesh */
    3024:   {
    3025:     const PetscInt *leaves;
    3026:     PetscSFNode    *remotePoints, *rowners, *lowners;
    3027:     PetscInt        numRoots, numLeaves, numGhostPoints = 0, p, gp, *ghostPoints;
    3028:     PetscInt        pStart, pEnd;

    3030:     DMPlexGetChart(*dmParallel, &pStart, &pEnd);
    3031:     PetscSFGetGraph(pointSF, &numRoots, &numLeaves, &leaves, NULL);
    3032:     PetscMalloc2(numRoots,PetscSFNode,&rowners,numLeaves,PetscSFNode,&lowners);
    3033:     for (p=0; p<numRoots; p++) {
    3034:       rowners[p].rank  = -1;
    3035:       rowners[p].index = -1;
    3036:     }
    3037:     if (origCellPart) {
    3038:       /* Make sure cells in the original partition are not assigned to other procs */
    3039:       const PetscInt *origCells;

    3041:       ISGetIndices(origCellPart, &origCells);
    3042:       for (p = 0; p < numProcs; ++p) {
    3043:         PetscInt dof, off, d;

    3045:         PetscSectionGetDof(origCellPartSection, p, &dof);
    3046:         PetscSectionGetOffset(origCellPartSection, p, &off);
    3047:         for (d = off; d < off+dof; ++d) {
    3048:           rowners[origCells[d]].rank = p;
    3049:         }
    3050:       }
    3051:       ISRestoreIndices(origCellPart, &origCells);
    3052:     }
    3053:     ISDestroy(&origCellPart);
    3054:     PetscSectionDestroy(&origCellPartSection);

    3056:     PetscSFBcastBegin(pointSF, MPIU_2INT, rowners, lowners);
    3057:     PetscSFBcastEnd(pointSF, MPIU_2INT, rowners, lowners);
    3058:     for (p = 0; p < numLeaves; ++p) {
    3059:       if (lowners[p].rank < 0 || lowners[p].rank == rank) { /* Either put in a bid or we know we own it */
    3060:         lowners[p].rank  = rank;
    3061:         lowners[p].index = leaves ? leaves[p] : p;
    3062:       } else if (lowners[p].rank >= 0) { /* Point already claimed so flag so that MAXLOC does not listen to us */
    3063:         lowners[p].rank  = -2;
    3064:         lowners[p].index = -2;
    3065:       }
    3066:     }
    3067:     for (p=0; p<numRoots; p++) { /* Root must not participate in the rediction, flag so that MAXLOC does not use */
    3068:       rowners[p].rank  = -3;
    3069:       rowners[p].index = -3;
    3070:     }
    3071:     PetscSFReduceBegin(pointSF, MPIU_2INT, lowners, rowners, MPI_MAXLOC);
    3072:     PetscSFReduceEnd(pointSF, MPIU_2INT, lowners, rowners, MPI_MAXLOC);
    3073:     PetscSFBcastBegin(pointSF, MPIU_2INT, rowners, lowners);
    3074:     PetscSFBcastEnd(pointSF, MPIU_2INT, rowners, lowners);
    3075:     for (p = 0; p < numLeaves; ++p) {
    3076:       if (lowners[p].rank < 0 || lowners[p].index < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Cell partition corrupt: point not claimed");
    3077:       if (lowners[p].rank != rank) ++numGhostPoints;
    3078:     }
    3079:     PetscMalloc(numGhostPoints * sizeof(PetscInt),    &ghostPoints);
    3080:     PetscMalloc(numGhostPoints * sizeof(PetscSFNode), &remotePoints);
    3081:     for (p = 0, gp = 0; p < numLeaves; ++p) {
    3082:       if (lowners[p].rank != rank) {
    3083:         ghostPoints[gp]        = leaves ? leaves[p] : p;
    3084:         remotePoints[gp].rank  = lowners[p].rank;
    3085:         remotePoints[gp].index = lowners[p].index;
    3086:         ++gp;
    3087:       }
    3088:     }
    3089:     PetscFree2(rowners,lowners);
    3090:     PetscSFSetGraph((*dmParallel)->sf, pEnd - pStart, numGhostPoints, ghostPoints, PETSC_OWN_POINTER, remotePoints, PETSC_OWN_POINTER);
    3091:     PetscSFSetFromOptions((*dmParallel)->sf);
    3092:   }
    3093:   /* Cleanup */
    3094:   PetscSFDestroy(&pointSF);
    3095:   DMSetFromOptions(*dmParallel);
    3096:   PetscLogEventEnd(DMPLEX_Distribute,dm,0,0,0);
    3097:   return(0);
    3098: }

    3102: /* This is to fix the tetrahedron orientation from TetGen */
    3103: PETSC_UNUSED static PetscErrorCode DMPlexInvertCells_Internal(PetscInt numCells, PetscInt numCorners, int cells[])
    3104: {
    3105:   PetscInt c;

    3108:   if (numCorners != 4) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Cannot invert cells with %d corners", numCorners);
    3109:   for (c = 0; c < numCells; ++c) {
    3110:     int *cone = &cells[c*4], tmpc;

    3112:     tmpc    = cone[0];
    3113:     cone[0] = cone[1];
    3114:     cone[1] = tmpc;
    3115:   }
    3116:   return(0);
    3117: }

    3119: #if defined(PETSC_HAVE_TRIANGLE)
    3120: #include <triangle.h>

    3124: PetscErrorCode InitInput_Triangle(struct triangulateio *inputCtx)
    3125: {
    3127:   inputCtx->numberofpoints             = 0;
    3128:   inputCtx->numberofpointattributes    = 0;
    3129:   inputCtx->pointlist                  = NULL;
    3130:   inputCtx->pointattributelist         = NULL;
    3131:   inputCtx->pointmarkerlist            = NULL;
    3132:   inputCtx->numberofsegments           = 0;
    3133:   inputCtx->segmentlist                = NULL;
    3134:   inputCtx->segmentmarkerlist          = NULL;
    3135:   inputCtx->numberoftriangleattributes = 0;
    3136:   inputCtx->trianglelist               = NULL;
    3137:   inputCtx->numberofholes              = 0;
    3138:   inputCtx->holelist                   = NULL;
    3139:   inputCtx->numberofregions            = 0;
    3140:   inputCtx->regionlist                 = NULL;
    3141:   return(0);
    3142: }

    3146: PetscErrorCode InitOutput_Triangle(struct triangulateio *outputCtx)
    3147: {
    3149:   outputCtx->numberofpoints        = 0;
    3150:   outputCtx->pointlist             = NULL;
    3151:   outputCtx->pointattributelist    = NULL;
    3152:   outputCtx->pointmarkerlist       = NULL;
    3153:   outputCtx->numberoftriangles     = 0;
    3154:   outputCtx->trianglelist          = NULL;
    3155:   outputCtx->triangleattributelist = NULL;
    3156:   outputCtx->neighborlist          = NULL;
    3157:   outputCtx->segmentlist           = NULL;
    3158:   outputCtx->segmentmarkerlist     = NULL;
    3159:   outputCtx->numberofedges         = 0;
    3160:   outputCtx->edgelist              = NULL;
    3161:   outputCtx->edgemarkerlist        = NULL;
    3162:   return(0);
    3163: }

    3167: PetscErrorCode FiniOutput_Triangle(struct triangulateio *outputCtx)
    3168: {
    3170:   free(outputCtx->pointmarkerlist);
    3171:   free(outputCtx->edgelist);
    3172:   free(outputCtx->edgemarkerlist);
    3173:   free(outputCtx->trianglelist);
    3174:   free(outputCtx->neighborlist);
    3175:   return(0);
    3176: }

    3180: PetscErrorCode DMPlexGenerate_Triangle(DM boundary, PetscBool interpolate, DM *dm)
    3181: {
    3182:   MPI_Comm             comm;
    3183:   PetscInt             dim              = 2;
    3184:   const PetscBool      createConvexHull = PETSC_FALSE;
    3185:   const PetscBool      constrained      = PETSC_FALSE;
    3186:   struct triangulateio in;
    3187:   struct triangulateio out;
    3188:   PetscInt             vStart, vEnd, v, eStart, eEnd, e;
    3189:   PetscMPIInt          rank;
    3190:   PetscErrorCode       ierr;

    3193:   PetscObjectGetComm((PetscObject)boundary,&comm);
    3194:   MPI_Comm_rank(comm, &rank);
    3195:   InitInput_Triangle(&in);
    3196:   InitOutput_Triangle(&out);
    3197:   DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);

    3199:   in.numberofpoints = vEnd - vStart;
    3200:   if (in.numberofpoints > 0) {
    3201:     PetscSection coordSection;
    3202:     Vec          coordinates;
    3203:     PetscScalar *array;

    3205:     PetscMalloc(in.numberofpoints*dim * sizeof(double), &in.pointlist);
    3206:     PetscMalloc(in.numberofpoints * sizeof(int), &in.pointmarkerlist);
    3207:     DMGetCoordinatesLocal(boundary, &coordinates);
    3208:     DMPlexGetCoordinateSection(boundary, &coordSection);
    3209:     VecGetArray(coordinates, &array);
    3210:     for (v = vStart; v < vEnd; ++v) {
    3211:       const PetscInt idx = v - vStart;
    3212:       PetscInt       off, d;

    3214:       PetscSectionGetOffset(coordSection, v, &off);
    3215:       for (d = 0; d < dim; ++d) {
    3216:         in.pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
    3217:       }
    3218:       DMPlexGetLabelValue(boundary, "marker", v, &in.pointmarkerlist[idx]);
    3219:     }
    3220:     VecRestoreArray(coordinates, &array);
    3221:   }
    3222:   DMPlexGetHeightStratum(boundary, 0, &eStart, &eEnd);
    3223:   in.numberofsegments = eEnd - eStart;
    3224:   if (in.numberofsegments > 0) {
    3225:     PetscMalloc(in.numberofsegments*2 * sizeof(int), &in.segmentlist);
    3226:     PetscMalloc(in.numberofsegments   * sizeof(int), &in.segmentmarkerlist);
    3227:     for (e = eStart; e < eEnd; ++e) {
    3228:       const PetscInt  idx = e - eStart;
    3229:       const PetscInt *cone;

    3231:       DMPlexGetCone(boundary, e, &cone);

    3233:       in.segmentlist[idx*2+0] = cone[0] - vStart;
    3234:       in.segmentlist[idx*2+1] = cone[1] - vStart;

    3236:       DMPlexGetLabelValue(boundary, "marker", e, &in.segmentmarkerlist[idx]);
    3237:     }
    3238:   }
    3239: #if 0 /* Do not currently support holes */
    3240:   PetscReal *holeCoords;
    3241:   PetscInt   h, d;

    3243:   DMPlexGetHoles(boundary, &in.numberofholes, &holeCords);
    3244:   if (in.numberofholes > 0) {
    3245:     PetscMalloc(in.numberofholes*dim * sizeof(double), &in.holelist);
    3246:     for (h = 0; h < in.numberofholes; ++h) {
    3247:       for (d = 0; d < dim; ++d) {
    3248:         in.holelist[h*dim+d] = holeCoords[h*dim+d];
    3249:       }
    3250:     }
    3251:   }
    3252: #endif
    3253:   if (!rank) {
    3254:     char args[32];

    3256:     /* Take away 'Q' for verbose output */
    3257:     PetscStrcpy(args, "pqezQ");
    3258:     if (createConvexHull) {
    3259:       PetscStrcat(args, "c");
    3260:     }
    3261:     if (constrained) {
    3262:       PetscStrcpy(args, "zepDQ");
    3263:     }
    3264:     triangulate(args, &in, &out, NULL);
    3265:   }
    3266:   PetscFree(in.pointlist);
    3267:   PetscFree(in.pointmarkerlist);
    3268:   PetscFree(in.segmentlist);
    3269:   PetscFree(in.segmentmarkerlist);
    3270:   PetscFree(in.holelist);

    3272:   {
    3273:     const PetscInt numCorners  = 3;
    3274:     const PetscInt numCells    = out.numberoftriangles;
    3275:     const PetscInt numVertices = out.numberofpoints;
    3276:     const int     *cells      = out.trianglelist;
    3277:     const double  *meshCoords = out.pointlist;

    3279:     DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);
    3280:     /* Set labels */
    3281:     for (v = 0; v < numVertices; ++v) {
    3282:       if (out.pointmarkerlist[v]) {
    3283:         DMPlexSetLabelValue(*dm, "marker", v+numCells, out.pointmarkerlist[v]);
    3284:       }
    3285:     }
    3286:     if (interpolate) {
    3287:       for (e = 0; e < out.numberofedges; e++) {
    3288:         if (out.edgemarkerlist[e]) {
    3289:           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
    3290:           const PetscInt *edges;
    3291:           PetscInt        numEdges;

    3293:           DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);
    3294:           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
    3295:           DMPlexSetLabelValue(*dm, "marker", edges[0], out.edgemarkerlist[e]);
    3296:           DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);
    3297:         }
    3298:       }
    3299:     }
    3300:     DMPlexSetRefinementUniform(*dm, PETSC_FALSE);
    3301:   }
    3302: #if 0 /* Do not currently support holes */
    3303:   DMPlexCopyHoles(*dm, boundary);
    3304: #endif
    3305:   FiniOutput_Triangle(&out);
    3306:   return(0);
    3307: }

    3311: PetscErrorCode DMPlexRefine_Triangle(DM dm, double *maxVolumes, DM *dmRefined)
    3312: {
    3313:   MPI_Comm             comm;
    3314:   PetscInt             dim  = 2;
    3315:   struct triangulateio in;
    3316:   struct triangulateio out;
    3317:   PetscInt             vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
    3318:   PetscMPIInt          rank;
    3319:   PetscErrorCode       ierr;

    3322:   PetscObjectGetComm((PetscObject)dm,&comm);
    3323:   MPI_Comm_rank(comm, &rank);
    3324:   InitInput_Triangle(&in);
    3325:   InitOutput_Triangle(&out);
    3326:   DMPlexGetDepth(dm, &depth);
    3327:   MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);
    3328:   DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);

    3330:   in.numberofpoints = vEnd - vStart;
    3331:   if (in.numberofpoints > 0) {
    3332:     PetscSection coordSection;
    3333:     Vec          coordinates;
    3334:     PetscScalar *array;

    3336:     PetscMalloc(in.numberofpoints*dim * sizeof(double), &in.pointlist);
    3337:     PetscMalloc(in.numberofpoints * sizeof(int), &in.pointmarkerlist);
    3338:     DMGetCoordinatesLocal(dm, &coordinates);
    3339:     DMPlexGetCoordinateSection(dm, &coordSection);
    3340:     VecGetArray(coordinates, &array);
    3341:     for (v = vStart; v < vEnd; ++v) {
    3342:       const PetscInt idx = v - vStart;
    3343:       PetscInt       off, d;

    3345:       PetscSectionGetOffset(coordSection, v, &off);
    3346:       for (d = 0; d < dim; ++d) {
    3347:         in.pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
    3348:       }
    3349:       DMPlexGetLabelValue(dm, "marker", v, &in.pointmarkerlist[idx]);
    3350:     }
    3351:     VecRestoreArray(coordinates, &array);
    3352:   }
    3353:   DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);

    3355:   in.numberofcorners   = 3;
    3356:   in.numberoftriangles = cEnd - cStart;

    3358:   in.trianglearealist  = (double*) maxVolumes;
    3359:   if (in.numberoftriangles > 0) {
    3360:     PetscMalloc(in.numberoftriangles*in.numberofcorners * sizeof(int), &in.trianglelist);
    3361:     for (c = cStart; c < cEnd; ++c) {
    3362:       const PetscInt idx      = c - cStart;
    3363:       PetscInt      *closure = NULL;
    3364:       PetscInt       closureSize;

    3366:       DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);
    3367:       if ((closureSize != 4) && (closureSize != 7)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a triangle, %D vertices in closure", closureSize);
    3368:       for (v = 0; v < 3; ++v) {
    3369:         in.trianglelist[idx*in.numberofcorners + v] = closure[(v+closureSize-3)*2] - vStart;
    3370:       }
    3371:       DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);
    3372:     }
    3373:   }
    3374:   /* TODO: Segment markers are missing on input */
    3375: #if 0 /* Do not currently support holes */
    3376:   PetscReal *holeCoords;
    3377:   PetscInt   h, d;

    3379:   DMPlexGetHoles(boundary, &in.numberofholes, &holeCords);
    3380:   if (in.numberofholes > 0) {
    3381:     PetscMalloc(in.numberofholes*dim * sizeof(double), &in.holelist);
    3382:     for (h = 0; h < in.numberofholes; ++h) {
    3383:       for (d = 0; d < dim; ++d) {
    3384:         in.holelist[h*dim+d] = holeCoords[h*dim+d];
    3385:       }
    3386:     }
    3387:   }
    3388: #endif
    3389:   if (!rank) {
    3390:     char args[32];

    3392:     /* Take away 'Q' for verbose output */
    3393:     PetscStrcpy(args, "pqezQra");
    3394:     triangulate(args, &in, &out, NULL);
    3395:   }
    3396:   PetscFree(in.pointlist);
    3397:   PetscFree(in.pointmarkerlist);
    3398:   PetscFree(in.segmentlist);
    3399:   PetscFree(in.segmentmarkerlist);
    3400:   PetscFree(in.trianglelist);

    3402:   {
    3403:     const PetscInt numCorners  = 3;
    3404:     const PetscInt numCells    = out.numberoftriangles;
    3405:     const PetscInt numVertices = out.numberofpoints;
    3406:     const int     *cells      = out.trianglelist;
    3407:     const double  *meshCoords = out.pointlist;
    3408:     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;

    3410:     DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);
    3411:     /* Set labels */
    3412:     for (v = 0; v < numVertices; ++v) {
    3413:       if (out.pointmarkerlist[v]) {
    3414:         DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out.pointmarkerlist[v]);
    3415:       }
    3416:     }
    3417:     if (interpolate) {
    3418:       PetscInt e;

    3420:       for (e = 0; e < out.numberofedges; e++) {
    3421:         if (out.edgemarkerlist[e]) {
    3422:           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
    3423:           const PetscInt *edges;
    3424:           PetscInt        numEdges;

    3426:           DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);
    3427:           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
    3428:           DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out.edgemarkerlist[e]);
    3429:           DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);
    3430:         }
    3431:       }
    3432:     }
    3433:     DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);
    3434:   }
    3435: #if 0 /* Do not currently support holes */
    3436:   DMPlexCopyHoles(*dm, boundary);
    3437: #endif
    3438:   FiniOutput_Triangle(&out);
    3439:   return(0);
    3440: }
    3441: #endif

    3443: #if defined(PETSC_HAVE_TETGEN)
    3444: #include <tetgen.h>
    3447: PetscErrorCode DMPlexGenerate_Tetgen(DM boundary, PetscBool interpolate, DM *dm)
    3448: {
    3449:   MPI_Comm       comm;
    3450:   const PetscInt dim  = 3;
    3451:   ::tetgenio     in;
    3452:   ::tetgenio     out;
    3453:   PetscInt       vStart, vEnd, v, fStart, fEnd, f;
    3454:   PetscMPIInt    rank;

    3458:   PetscObjectGetComm((PetscObject)boundary,&comm);
    3459:   MPI_Comm_rank(comm, &rank);
    3460:   DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);
    3461:   in.numberofpoints = vEnd - vStart;
    3462:   if (in.numberofpoints > 0) {
    3463:     PetscSection coordSection;
    3464:     Vec          coordinates;
    3465:     PetscScalar *array;

    3467:     in.pointlist       = new double[in.numberofpoints*dim];
    3468:     in.pointmarkerlist = new int[in.numberofpoints];

    3470:     DMGetCoordinatesLocal(boundary, &coordinates);
    3471:     DMPlexGetCoordinateSection(boundary, &coordSection);
    3472:     VecGetArray(coordinates, &array);
    3473:     for (v = vStart; v < vEnd; ++v) {
    3474:       const PetscInt idx = v - vStart;
    3475:       PetscInt       off, d;

    3477:       PetscSectionGetOffset(coordSection, v, &off);
    3478:       for (d = 0; d < dim; ++d) in.pointlist[idx*dim + d] = array[off+d];
    3479:       DMPlexGetLabelValue(boundary, "marker", v, &in.pointmarkerlist[idx]);
    3480:     }
    3481:     VecRestoreArray(coordinates, &array);
    3482:   }
    3483:   DMPlexGetHeightStratum(boundary, 0, &fStart, &fEnd);

    3485:   in.numberoffacets = fEnd - fStart;
    3486:   if (in.numberoffacets > 0) {
    3487:     in.facetlist       = new tetgenio::facet[in.numberoffacets];
    3488:     in.facetmarkerlist = new int[in.numberoffacets];
    3489:     for (f = fStart; f < fEnd; ++f) {
    3490:       const PetscInt idx     = f - fStart;
    3491:       PetscInt      *points = NULL, numPoints, p, numVertices = 0, v;

    3493:       in.facetlist[idx].numberofpolygons = 1;
    3494:       in.facetlist[idx].polygonlist      = new tetgenio::polygon[in.facetlist[idx].numberofpolygons];
    3495:       in.facetlist[idx].numberofholes    = 0;
    3496:       in.facetlist[idx].holelist         = NULL;

    3498:       DMPlexGetTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);
    3499:       for (p = 0; p < numPoints*2; p += 2) {
    3500:         const PetscInt point = points[p];
    3501:         if ((point >= vStart) && (point < vEnd)) points[numVertices++] = point;
    3502:       }

    3504:       tetgenio::polygon *poly = in.facetlist[idx].polygonlist;
    3505:       poly->numberofvertices = numVertices;
    3506:       poly->vertexlist       = new int[poly->numberofvertices];
    3507:       for (v = 0; v < numVertices; ++v) {
    3508:         const PetscInt vIdx = points[v] - vStart;
    3509:         poly->vertexlist[v] = vIdx;
    3510:       }
    3511:       DMPlexGetLabelValue(boundary, "marker", f, &in.facetmarkerlist[idx]);
    3512:       DMPlexRestoreTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);
    3513:     }
    3514:   }
    3515:   if (!rank) {
    3516:     char args[32];

    3518:     /* Take away 'Q' for verbose output */
    3519:     PetscStrcpy(args, "pqezQ");
    3520:     ::tetrahedralize(args, &in, &out);
    3521:   }
    3522:   {
    3523:     const PetscInt numCorners  = 4;
    3524:     const PetscInt numCells    = out.numberoftetrahedra;
    3525:     const PetscInt numVertices = out.numberofpoints;
    3526:     const double   *meshCoords = out.pointlist;
    3527:     int            *cells      = out.tetrahedronlist;

    3529:     DMPlexInvertCells_Internal(numCells, numCorners, cells);
    3530:     DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);
    3531:     /* Set labels */
    3532:     for (v = 0; v < numVertices; ++v) {
    3533:       if (out.pointmarkerlist[v]) {
    3534:         DMPlexSetLabelValue(*dm, "marker", v+numCells, out.pointmarkerlist[v]);
    3535:       }
    3536:     }
    3537:     if (interpolate) {
    3538:       PetscInt e;

    3540:       for (e = 0; e < out.numberofedges; e++) {
    3541:         if (out.edgemarkerlist[e]) {
    3542:           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
    3543:           const PetscInt *edges;
    3544:           PetscInt        numEdges;

    3546:           DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);
    3547:           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
    3548:           DMPlexSetLabelValue(*dm, "marker", edges[0], out.edgemarkerlist[e]);
    3549:           DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);
    3550:         }
    3551:       }
    3552:       for (f = 0; f < out.numberoftrifaces; f++) {
    3553:         if (out.trifacemarkerlist[f]) {
    3554:           const PetscInt  vertices[3] = {out.trifacelist[f*3+0]+numCells, out.trifacelist[f*3+1]+numCells, out.trifacelist[f*3+2]+numCells};
    3555:           const PetscInt *faces;
    3556:           PetscInt        numFaces;

    3558:           DMPlexGetJoin(*dm, 3, vertices, &numFaces, &faces);
    3559:           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
    3560:           DMPlexSetLabelValue(*dm, "marker", faces[0], out.trifacemarkerlist[f]);
    3561:           DMPlexRestoreJoin(*dm, 3, vertices, &numFaces, &faces);
    3562:         }
    3563:       }
    3564:     }
    3565:     DMPlexSetRefinementUniform(*dm, PETSC_FALSE);
    3566:   }
    3567:   return(0);
    3568: }

    3572: PetscErrorCode DMPlexRefine_Tetgen(DM dm, double *maxVolumes, DM *dmRefined)
    3573: {
    3574:   MPI_Comm       comm;
    3575:   const PetscInt dim  = 3;
    3576:   ::tetgenio     in;
    3577:   ::tetgenio     out;
    3578:   PetscInt       vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
    3579:   PetscMPIInt    rank;

    3583:   PetscObjectGetComm((PetscObject)dm,&comm);
    3584:   MPI_Comm_rank(comm, &rank);
    3585:   DMPlexGetDepth(dm, &depth);
    3586:   MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);
    3587:   DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);

    3589:   in.numberofpoints = vEnd - vStart;
    3590:   if (in.numberofpoints > 0) {
    3591:     PetscSection coordSection;
    3592:     Vec          coordinates;
    3593:     PetscScalar *array;

    3595:     in.pointlist       = new double[in.numberofpoints*dim];
    3596:     in.pointmarkerlist = new int[in.numberofpoints];

    3598:     DMGetCoordinatesLocal(dm, &coordinates);
    3599:     DMPlexGetCoordinateSection(dm, &coordSection);
    3600:     VecGetArray(coordinates, &array);
    3601:     for (v = vStart; v < vEnd; ++v) {
    3602:       const PetscInt idx = v - vStart;
    3603:       PetscInt       off, d;

    3605:       PetscSectionGetOffset(coordSection, v, &off);
    3606:       for (d = 0; d < dim; ++d) in.pointlist[idx*dim + d] = array[off+d];
    3607:       DMPlexGetLabelValue(dm, "marker", v, &in.pointmarkerlist[idx]);
    3608:     }
    3609:     VecRestoreArray(coordinates, &array);
    3610:   }
    3611:   DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);

    3613:   in.numberofcorners       = 4;
    3614:   in.numberoftetrahedra    = cEnd - cStart;
    3615:   in.tetrahedronvolumelist = (double*) maxVolumes;
    3616:   if (in.numberoftetrahedra > 0) {
    3617:     in.tetrahedronlist = new int[in.numberoftetrahedra*in.numberofcorners];
    3618:     for (c = cStart; c < cEnd; ++c) {
    3619:       const PetscInt idx      = c - cStart;
    3620:       PetscInt      *closure = NULL;
    3621:       PetscInt       closureSize;

    3623:       DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);
    3624:       if ((closureSize != 5) && (closureSize != 15)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a tetrahedron, %D vertices in closure", closureSize);
    3625:       for (v = 0; v < 4; ++v) {
    3626:         in.tetrahedronlist[idx*in.numberofcorners + v] = closure[(v+closureSize-4)*2] - vStart;
    3627:       }
    3628:       DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);
    3629:     }
    3630:   }
    3631:   /* TODO: Put in boundary faces with markers */
    3632:   if (!rank) {
    3633:     char args[32];

    3635:     /* Take away 'Q' for verbose output */
    3636:     /*PetscStrcpy(args, "qezQra"); */
    3637:     PetscStrcpy(args, "qezraVVVV");
    3638:     ::tetrahedralize(args, &in, &out);
    3639:   }
    3640:   in.tetrahedronvolumelist = NULL;

    3642:   {
    3643:     const PetscInt numCorners  = 4;
    3644:     const PetscInt numCells    = out.numberoftetrahedra;
    3645:     const PetscInt numVertices = out.numberofpoints;
    3646:     const double   *meshCoords = out.pointlist;
    3647:     int            *cells      = out.tetrahedronlist;

    3649:     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;

    3651:     DMPlexInvertCells_Internal(numCells, numCorners, cells);
    3652:     DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);
    3653:     /* Set labels */
    3654:     for (v = 0; v < numVertices; ++v) {
    3655:       if (out.pointmarkerlist[v]) {
    3656:         DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out.pointmarkerlist[v]);
    3657:       }
    3658:     }
    3659:     if (interpolate) {
    3660:       PetscInt e, f;

    3662:       for (e = 0; e < out.numberofedges; e++) {
    3663:         if (out.edgemarkerlist[e]) {
    3664:           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
    3665:           const PetscInt *edges;
    3666:           PetscInt        numEdges;

    3668:           DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);
    3669:           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
    3670:           DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out.edgemarkerlist[e]);
    3671:           DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);
    3672:         }
    3673:       }
    3674:       for (f = 0; f < out.numberoftrifaces; f++) {
    3675:         if (out.trifacemarkerlist[f]) {
    3676:           const PetscInt  vertices[3] = {out.trifacelist[f*3+0]+numCells, out.trifacelist[f*3+1]+numCells, out.trifacelist[f*3+2]+numCells};
    3677:           const PetscInt *faces;
    3678:           PetscInt        numFaces;

    3680:           DMPlexGetJoin(*dmRefined, 3, vertices, &numFaces, &faces);
    3681:           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
    3682:           DMPlexSetLabelValue(*dmRefined, "marker", faces[0], out.trifacemarkerlist[f]);
    3683:           DMPlexRestoreJoin(*dmRefined, 3, vertices, &numFaces, &faces);
    3684:         }
    3685:       }
    3686:     }
    3687:     DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);
    3688:   }
    3689:   return(0);
    3690: }
    3691: #endif

    3693: #if defined(PETSC_HAVE_CTETGEN)
    3694: #include "ctetgen.h"

    3698: PetscErrorCode DMPlexGenerate_CTetgen(DM boundary, PetscBool interpolate, DM *dm)
    3699: {
    3700:   MPI_Comm       comm;
    3701:   const PetscInt dim  = 3;
    3702:   PLC           *in, *out;
    3703:   PetscInt       verbose = 0, vStart, vEnd, v, fStart, fEnd, f;
    3704:   PetscMPIInt    rank;

    3708:   PetscObjectGetComm((PetscObject)boundary,&comm);
    3709:   PetscOptionsGetInt(((PetscObject) boundary)->prefix, "-ctetgen_verbose", &verbose, NULL);
    3710:   MPI_Comm_rank(comm, &rank);
    3711:   DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);
    3712:   PLCCreate(&in);
    3713:   PLCCreate(&out);

    3715:   in->numberofpoints = vEnd - vStart;
    3716:   if (in->numberofpoints > 0) {
    3717:     PetscSection coordSection;
    3718:     Vec          coordinates;
    3719:     PetscScalar *array;

    3721:     PetscMalloc(in->numberofpoints*dim * sizeof(PetscReal), &in->pointlist);
    3722:     PetscMalloc(in->numberofpoints     * sizeof(int),       &in->pointmarkerlist);
    3723:     DMGetCoordinatesLocal(boundary, &coordinates);
    3724:     DMPlexGetCoordinateSection(boundary, &coordSection);
    3725:     VecGetArray(coordinates, &array);
    3726:     for (v = vStart; v < vEnd; ++v) {
    3727:       const PetscInt idx = v - vStart;
    3728:       PetscInt       off, d, m;

    3730:       PetscSectionGetOffset(coordSection, v, &off);
    3731:       for (d = 0; d < dim; ++d) {
    3732:         in->pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
    3733:       }
    3734:       DMPlexGetLabelValue(boundary, "marker", v, &m);

    3736:       in->pointmarkerlist[idx] = (int) m;
    3737:     }
    3738:     VecRestoreArray(coordinates, &array);
    3739:   }
    3740:   DMPlexGetHeightStratum(boundary, 0, &fStart, &fEnd);

    3742:   in->numberoffacets = fEnd - fStart;
    3743:   if (in->numberoffacets > 0) {
    3744:     PetscMalloc(in->numberoffacets * sizeof(facet), &in->facetlist);
    3745:     PetscMalloc(in->numberoffacets * sizeof(int),   &in->facetmarkerlist);
    3746:     for (f = fStart; f < fEnd; ++f) {
    3747:       const PetscInt idx     = f - fStart;
    3748:       PetscInt      *points = NULL, numPoints, p, numVertices = 0, v, m;
    3749:       polygon       *poly;

    3751:       in->facetlist[idx].numberofpolygons = 1;

    3753:       PetscMalloc(in->facetlist[idx].numberofpolygons * sizeof(polygon), &in->facetlist[idx].polygonlist);

    3755:       in->facetlist[idx].numberofholes    = 0;
    3756:       in->facetlist[idx].holelist         = NULL;

    3758:       DMPlexGetTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);
    3759:       for (p = 0; p < numPoints*2; p += 2) {
    3760:         const PetscInt point = points[p];
    3761:         if ((point >= vStart) && (point < vEnd)) points[numVertices++] = point;
    3762:       }

    3764:       poly                   = in->facetlist[idx].polygonlist;
    3765:       poly->numberofvertices = numVertices;
    3766:       PetscMalloc(poly->numberofvertices * sizeof(int), &poly->vertexlist);
    3767:       for (v = 0; v < numVertices; ++v) {
    3768:         const PetscInt vIdx = points[v] - vStart;
    3769:         poly->vertexlist[v] = vIdx;
    3770:       }
    3771:       DMPlexGetLabelValue(boundary, "marker", f, &m);
    3772:       in->facetmarkerlist[idx] = (int) m;
    3773:       DMPlexRestoreTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);
    3774:     }
    3775:   }
    3776:   if (!rank) {
    3777:     TetGenOpts t;

    3779:     TetGenOptsInitialize(&t);
    3780:     t.in        = boundary; /* Should go away */
    3781:     t.plc       = 1;
    3782:     t.quality   = 1;
    3783:     t.edgesout  = 1;
    3784:     t.zeroindex = 1;
    3785:     t.quiet     = 1;
    3786:     t.verbose   = verbose;
    3787:     TetGenCheckOpts(&t);
    3788:     TetGenTetrahedralize(&t, in, out);
    3789:   }
    3790:   {
    3791:     const PetscInt numCorners  = 4;
    3792:     const PetscInt numCells    = out->numberoftetrahedra;
    3793:     const PetscInt numVertices = out->numberofpoints;
    3794:     const double   *meshCoords = out->pointlist;
    3795:     int            *cells      = out->tetrahedronlist;

    3797:     DMPlexInvertCells_Internal(numCells, numCorners, cells);
    3798:     DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);
    3799:     /* Set labels */
    3800:     for (v = 0; v < numVertices; ++v) {
    3801:       if (out->pointmarkerlist[v]) {
    3802:         DMPlexSetLabelValue(*dm, "marker", v+numCells, out->pointmarkerlist[v]);
    3803:       }
    3804:     }
    3805:     if (interpolate) {
    3806:       PetscInt e;

    3808:       for (e = 0; e < out->numberofedges; e++) {
    3809:         if (out->edgemarkerlist[e]) {
    3810:           const PetscInt  vertices[2] = {out->edgelist[e*2+0]+numCells, out->edgelist[e*2+1]+numCells};
    3811:           const PetscInt *edges;
    3812:           PetscInt        numEdges;

    3814:           DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);
    3815:           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
    3816:           DMPlexSetLabelValue(*dm, "marker", edges[0], out->edgemarkerlist[e]);
    3817:           DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);
    3818:         }
    3819:       }
    3820:       for (f = 0; f < out->numberoftrifaces; f++) {
    3821:         if (out->trifacemarkerlist[f]) {
    3822:           const PetscInt  vertices[3] = {out->trifacelist[f*3+0]+numCells, out->trifacelist[f*3+1]+numCells, out->trifacelist[f*3+2]+numCells};
    3823:           const PetscInt *faces;
    3824:           PetscInt        numFaces;

    3826:           DMPlexGetFullJoin(*dm, 3, vertices, &numFaces, &faces);
    3827:           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
    3828:           DMPlexSetLabelValue(*dm, "marker", faces[0], out->trifacemarkerlist[f]);
    3829:           DMPlexRestoreJoin(*dm, 3, vertices, &numFaces, &faces);
    3830:         }
    3831:       }
    3832:     }
    3833:     DMPlexSetRefinementUniform(*dm, PETSC_FALSE);
    3834:   }

    3836:   PLCDestroy(&in);
    3837:   PLCDestroy(&out);
    3838:   return(0);
    3839: }

    3843: PetscErrorCode DMPlexRefine_CTetgen(DM dm, PetscReal *maxVolumes, DM *dmRefined)
    3844: {
    3845:   MPI_Comm       comm;
    3846:   const PetscInt dim  = 3;
    3847:   PLC           *in, *out;
    3848:   PetscInt       verbose = 0, vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
    3849:   PetscMPIInt    rank;

    3853:   PetscObjectGetComm((PetscObject)dm,&comm);
    3854:   PetscOptionsGetInt(((PetscObject) dm)->prefix, "-ctetgen_verbose", &verbose, NULL);
    3855:   MPI_Comm_rank(comm, &rank);
    3856:   DMPlexGetDepth(dm, &depth);
    3857:   MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);
    3858:   DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);
    3859:   PLCCreate(&in);
    3860:   PLCCreate(&out);

    3862:   in->numberofpoints = vEnd - vStart;
    3863:   if (in->numberofpoints > 0) {
    3864:     PetscSection coordSection;
    3865:     Vec          coordinates;
    3866:     PetscScalar *array;

    3868:     PetscMalloc(in->numberofpoints*dim * sizeof(PetscReal), &in->pointlist);
    3869:     PetscMalloc(in->numberofpoints     * sizeof(int),       &in->pointmarkerlist);
    3870:     DMGetCoordinatesLocal(dm, &coordinates);
    3871:     DMPlexGetCoordinateSection(dm, &coordSection);
    3872:     VecGetArray(coordinates, &array);
    3873:     for (v = vStart; v < vEnd; ++v) {
    3874:       const PetscInt idx = v - vStart;
    3875:       PetscInt       off, d, m;

    3877:       PetscSectionGetOffset(coordSection, v, &off);
    3878:       for (d = 0; d < dim; ++d) {
    3879:         in->pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
    3880:       }
    3881:       DMPlexGetLabelValue(dm, "marker", v, &m);

    3883:       in->pointmarkerlist[idx] = (int) m;
    3884:     }
    3885:     VecRestoreArray(coordinates, &array);
    3886:   }
    3887:   DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);

    3889:   in->numberofcorners       = 4;
    3890:   in->numberoftetrahedra    = cEnd - cStart;
    3891:   in->tetrahedronvolumelist = maxVolumes;
    3892:   if (in->numberoftetrahedra > 0) {
    3893:     PetscMalloc(in->numberoftetrahedra*in->numberofcorners * sizeof(int), &in->tetrahedronlist);
    3894:     for (c = cStart; c < cEnd; ++c) {
    3895:       const PetscInt idx      = c - cStart;
    3896:       PetscInt      *closure = NULL;
    3897:       PetscInt       closureSize;

    3899:       DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);
    3900:       if ((closureSize != 5) && (closureSize != 15)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a tetrahedron, %D vertices in closure", closureSize);
    3901:       for (v = 0; v < 4; ++v) {
    3902:         in->tetrahedronlist[idx*in->numberofcorners + v] = closure[(v+closureSize-4)*2] - vStart;
    3903:       }
    3904:       DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);
    3905:     }
    3906:   }
    3907:   if (!rank) {
    3908:     TetGenOpts t;

    3910:     TetGenOptsInitialize(&t);

    3912:     t.in        = dm; /* Should go away */
    3913:     t.refine    = 1;
    3914:     t.varvolume = 1;
    3915:     t.quality   = 1;
    3916:     t.edgesout  = 1;
    3917:     t.zeroindex = 1;
    3918:     t.quiet     = 1;
    3919:     t.verbose   = verbose; /* Change this */

    3921:     TetGenCheckOpts(&t);
    3922:     TetGenTetrahedralize(&t, in, out);
    3923:   }
    3924:   {
    3925:     const PetscInt numCorners  = 4;
    3926:     const PetscInt numCells    = out->numberoftetrahedra;
    3927:     const PetscInt numVertices = out->numberofpoints;
    3928:     const double   *meshCoords = out->pointlist;
    3929:     int            *cells      = out->tetrahedronlist;
    3930:     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;

    3932:     DMPlexInvertCells_Internal(numCells, numCorners, cells);
    3933:     DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);
    3934:     /* Set labels */
    3935:     for (v = 0; v < numVertices; ++v) {
    3936:       if (out->pointmarkerlist[v]) {
    3937:         DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out->pointmarkerlist[v]);
    3938:       }
    3939:     }
    3940:     if (interpolate) {
    3941:       PetscInt e, f;

    3943:       for (e = 0; e < out->numberofedges; e++) {
    3944:         if (out->edgemarkerlist[e]) {
    3945:           const PetscInt  vertices[2] = {out->edgelist[e*2+0]+numCells, out->edgelist[e*2+1]+numCells};
    3946:           const PetscInt *edges;
    3947:           PetscInt        numEdges;

    3949:           DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);
    3950:           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
    3951:           DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out->edgemarkerlist[e]);
    3952:           DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);
    3953:         }
    3954:       }
    3955:       for (f = 0; f < out->numberoftrifaces; f++) {
    3956:         if (out->trifacemarkerlist[f]) {
    3957:           const PetscInt  vertices[3] = {out->trifacelist[f*3+0]+numCells, out->trifacelist[f*3+1]+numCells, out->trifacelist[f*3+2]+numCells};
    3958:           const PetscInt *faces;
    3959:           PetscInt        numFaces;

    3961:           DMPlexGetFullJoin(*dmRefined, 3, vertices, &numFaces, &faces);
    3962:           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
    3963:           DMPlexSetLabelValue(*dmRefined, "marker", faces[0], out->trifacemarkerlist[f]);
    3964:           DMPlexRestoreJoin(*dmRefined, 3, vertices, &numFaces, &faces);
    3965:         }
    3966:       }
    3967:     }
    3968:     DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);
    3969:   }
    3970:   PLCDestroy(&in);
    3971:   PLCDestroy(&out);
    3972:   return(0);
    3973: }
    3974: #endif

    3978: /*@C
    3979:   DMPlexGenerate - Generates a mesh.

    3981:   Not Collective

    3983:   Input Parameters:
    3984: + boundary - The DMPlex boundary object
    3985: . name - The mesh generation package name
    3986: - interpolate - Flag to create intermediate mesh elements

    3988:   Output Parameter:
    3989: . mesh - The DMPlex object

    3991:   Level: intermediate

    3993: .keywords: mesh, elements
    3994: .seealso: DMPlexCreate(), DMRefine()
    3995: @*/
    3996: PetscErrorCode DMPlexGenerate(DM boundary, const char name[], PetscBool interpolate, DM *mesh)
    3997: {
    3998:   PetscInt       dim;
    3999:   char           genname[1024];
    4000:   PetscBool      isTriangle = PETSC_FALSE, isTetgen = PETSC_FALSE, isCTetgen = PETSC_FALSE, flg;

    4006:   DMPlexGetDimension(boundary, &dim);
    4007:   PetscOptionsGetString(((PetscObject) boundary)->prefix, "-dm_plex_generator", genname, 1024, &flg);
    4008:   if (flg) name = genname;
    4009:   if (name) {
    4010:     PetscStrcmp(name, "triangle", &isTriangle);
    4011:     PetscStrcmp(name, "tetgen",   &isTetgen);
    4012:     PetscStrcmp(name, "ctetgen",  &isCTetgen);
    4013:   }
    4014:   switch (dim) {
    4015:   case 1:
    4016:     if (!name || isTriangle) {
    4017: #if defined(PETSC_HAVE_TRIANGLE)
    4018:       DMPlexGenerate_Triangle(boundary, interpolate, mesh);
    4019: #else
    4020:       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Mesh generation needs external package support.\nPlease reconfigure with --download-triangle.");
    4021: #endif
    4022:     } else SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Unknown 2D mesh generation package %s", name);
    4023:     break;
    4024:   case 2:
    4025:     if (!name || isCTetgen) {
    4026: #if defined(PETSC_HAVE_CTETGEN)
    4027:       DMPlexGenerate_CTetgen(boundary, interpolate, mesh);
    4028: #else
    4029:       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "CTetgen needs external package support.\nPlease reconfigure with --download-ctetgen.");
    4030: #endif
    4031:     } else if (isTetgen) {
    4032: #if defined(PETSC_HAVE_TETGEN)
    4033:       DMPlexGenerate_Tetgen(boundary, interpolate, mesh);
    4034: #else
    4035:       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Tetgen needs external package support.\nPlease reconfigure with --with-c-language=cxx --download-tetgen.");
    4036: #endif
    4037:     } else SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Unknown 3D mesh generation package %s", name);
    4038:     break;
    4039:   default:
    4040:     SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Mesh generation for a dimension %d boundary is not supported.", dim);
    4041:   }
    4042:   return(0);
    4043: }

    4045: typedef PetscInt CellRefiner;

    4049: PETSC_STATIC_INLINE PetscErrorCode GetDepthStart_Private(PetscInt depth, PetscInt depthSize[], PetscInt *cStart, PetscInt *fStart, PetscInt *eStart, PetscInt *vStart)
    4050: {
    4052:   if (cStart) *cStart = 0;
    4053:   if (vStart) *vStart = depthSize[depth];
    4054:   if (fStart) *fStart = depthSize[depth] + depthSize[0];
    4055:   if (eStart) *eStart = depthSize[depth] + depthSize[0] + depthSize[depth-1];
    4056:   return(0);
    4057: }

    4061: PETSC_STATIC_INLINE PetscErrorCode GetDepthEnd_Private(PetscInt depth, PetscInt depthSize[], PetscInt *cEnd, PetscInt *fEnd, PetscInt *eEnd, PetscInt *vEnd)
    4062: {
    4064:   if (cEnd) *cEnd = depthSize[depth];
    4065:   if (vEnd) *vEnd = depthSize[depth] + depthSize[0];
    4066:   if (fEnd) *fEnd = depthSize[depth] + depthSize[0] + depthSize[depth-1];
    4067:   if (eEnd) *eEnd = depthSize[depth] + depthSize[0] + depthSize[depth-1] + depthSize[1];
    4068:   return(0);
    4069: }

    4073: PetscErrorCode CellRefinerGetSizes(CellRefiner refiner, DM dm, PetscInt depthSize[])
    4074: {
    4075:   PetscInt       cStart, cEnd, cMax, vStart, vEnd, vMax, fStart, fEnd, fMax, eStart, eEnd, eMax;

    4079:   DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);
    4080:   DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);
    4081:   DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);
    4082:   DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);
    4083:   DMPlexGetHybridBounds(dm, &cMax, &fMax, &eMax, &vMax);
    4084:   switch (refiner) {
    4085:   case 1:
    4086:     /* Simplicial 2D */
    4087:     depthSize[0] = vEnd - vStart + fEnd - fStart;         /* Add a vertex on every face */
    4088:     depthSize[1] = 2*(fEnd - fStart) + 3*(cEnd - cStart); /* Every face is split into 2 faces and 3 faces are added for each cell */
    4089:     depthSize[2] = 4*(cEnd - cStart);                     /* Every cell split into 4 cells */
    4090:     break;
    4091:   case 3:
    4092:     /* Hybrid 2D */
    4093:     if (cMax < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No cell maximum specified in hybrid mesh");
    4094:     cMax = PetscMin(cEnd, cMax);
    4095:     if (fMax < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No face maximum specified in hybrid mesh");
    4096:     fMax         = PetscMin(fEnd, fMax);
    4097:     depthSize[0] = vEnd - vStart + fMax - fStart;                                         /* Add a vertex on every face, but not hybrid faces */
    4098:     depthSize[1] = 2*(fMax - fStart) + 3*(cMax - cStart) + (fEnd - fMax) + (cEnd - cMax); /* Every interior face is split into 2 faces, 3 faces are added for each interior cell, and one in each hybrid cell */
    4099:     depthSize[2] = 4*(cMax - cStart) + 2*(cEnd - cMax);                                   /* Interior cells split into 4 cells, Hybrid cells split into 2 cells */
    4100:     break;
    4101:   case 2:
    4102:     /* Hex 2D */
    4103:     depthSize[0] = vEnd - vStart + cEnd - cStart + fEnd - fStart; /* Add a vertex on every face and cell */
    4104:     depthSize[1] = 2*(fEnd - fStart) + 4*(cEnd - cStart);         /* Every face is split into 2 faces and 4 faces are added for each cell */
    4105:     depthSize[2] = 4*(cEnd - cStart);                             /* Every cell split into 4 cells */
    4106:     break;
    4107:   default:
    4108:     SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown cell refiner %d", refiner);
    4109:   }
    4110:   return(0);
    4111: }

    4115: PetscErrorCode CellRefinerSetConeSizes(CellRefiner refiner, DM dm, PetscInt depthSize[], DM rdm)
    4116: {
    4117:   PetscInt       depth, cStart, cStartNew, cEnd, cMax, c, vStart, vStartNew, vEnd, vMax, v, fStart, fStartNew, fEnd, fMax, f, eStart, eStartNew, eEnd, eMax, r;

    4121:   DMPlexGetDepth(dm, &depth);
    4122:   DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);
    4123:   DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);
    4124:   DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);
    4125:   DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);
    4126:   DMPlexGetHybridBounds(dm, &cMax, &fMax, &eMax, &vMax);
    4127:   GetDepthStart_Private(depth, depthSize, &cStartNew, &fStartNew, &eStartNew, &vStartNew);
    4128:   switch (refiner) {
    4129:   case 1:
    4130:     /* Simplicial 2D */
    4131:     /* All cells have 3 faces */
    4132:     for (c = cStart; c < cEnd; ++c) {
    4133:       for (r = 0; r < 4; ++r) {
    4134:         const PetscInt newp = (c - cStart)*4 + r;

    4136:         DMPlexSetConeSize(rdm, newp, 3);
    4137:       }
    4138:     }
    4139:     /* Split faces have 2 vertices and the same cells as the parent */
    4140:     for (f = fStart; f < fEnd; ++f) {
    4141:       for (r = 0; r < 2; ++r) {
    4142:         const PetscInt newp = fStartNew + (f - fStart)*2 + r;
    4143:         PetscInt       size;

    4145:         DMPlexSetConeSize(rdm, newp, 2);
    4146:         DMPlexGetSupportSize(dm, f, &size);
    4147:         DMPlexSetSupportSize(rdm, newp, size);
    4148:       }
    4149:     }
    4150:     /* Interior faces have 2 vertices and 2 cells */
    4151:     for (c = cStart; c < cEnd; ++c) {
    4152:       for (r = 0; r < 3; ++r) {
    4153:         const PetscInt newp = fStartNew + (fEnd - fStart)*2 + (c - cStart)*3 + r;

    4155:         DMPlexSetConeSize(rdm, newp, 2);
    4156:         DMPlexSetSupportSize(rdm, newp, 2);
    4157:       }
    4158:     }
    4159:     /* Old vertices have identical supports */
    4160:     for (v = vStart; v < vEnd; ++v) {
    4161:       const PetscInt newp = vStartNew + (v - vStart);
    4162:       PetscInt       size;

    4164:       DMPlexGetSupportSize(dm, v, &size);
    4165:       DMPlexSetSupportSize(rdm, newp, size);
    4166:     }
    4167:     /* Face vertices have 2 + cells*2 supports */
    4168:     for (f = fStart; f < fEnd; ++f) {
    4169:       const PetscInt newp = vStartNew + (vEnd - vStart) + (f - fStart);
    4170:       PetscInt       size;

    4172:       DMPlexGetSupportSize(dm, f, &size);
    4173:       DMPlexSetSupportSize(rdm, newp, 2 + size*2);
    4174:     }
    4175:     break;
    4176:   case 2:
    4177:     /* Hex 2D */
    4178:     /* All cells have 4 faces */
    4179:     for (c = cStart; c < cEnd; ++c) {
    4180:       for (r = 0; r < 4; ++r) {
    4181:         const PetscInt newp = (c - cStart)*4 + r;

    4183:         DMPlexSetConeSize(rdm, newp, 4);
    4184:       }
    4185:     }
    4186:     /* Split faces have 2 vertices and the same cells as the parent */
    4187:     for (f = fStart; f < fEnd; ++f) {
    4188:       for (r = 0; r < 2; ++r) {
    4189:         const PetscInt newp = fStartNew + (f - fStart)*2 + r;
    4190:         PetscInt       size;

    4192:         DMPlexSetConeSize(rdm, newp, 2);
    4193:         DMPlexGetSupportSize(dm, f, &size);
    4194:         DMPlexSetSupportSize(rdm, newp, size);
    4195:       }
    4196:     }
    4197:     /* Interior faces have 2 vertices and 2 cells */
    4198:     for (c = cStart; c < cEnd; ++c) {
    4199:       for (r = 0; r < 4; ++r) {
    4200:         const PetscInt newp = fStartNew + (fEnd - fStart)*2 + (c - cStart)*4 + r;

    4202:         DMPlexSetConeSize(rdm, newp, 2);
    4203:         DMPlexSetSupportSize(rdm, newp, 2);
    4204:       }
    4205:     }
    4206:     /* Old vertices have identical supports */
    4207:     for (v = vStart; v < vEnd; ++v) {
    4208:       const PetscInt newp = vStartNew + (v - vStart);
    4209:       PetscInt       size;

    4211:       DMPlexGetSupportSize(dm, v, &size);
    4212:       DMPlexSetSupportSize(rdm, newp, size);
    4213:     }
    4214:     /* Face vertices have 2 + cells supports */
    4215:     for (f = fStart; f < fEnd; ++f) {
    4216:       const PetscInt newp = vStartNew + (vEnd - vStart) + (f - fStart);
    4217:       PetscInt       size;

    4219:       DMPlexGetSupportSize(dm, f, &size);
    4220:       DMPlexSetSupportSize(rdm, newp, 2 + size);
    4221:     }
    4222:     /* Cell vertices have 4 supports */
    4223:     for (c = cStart; c < cEnd; ++c) {
    4224:       const PetscInt newp = vStartNew + (vEnd - vStart) + (fEnd - fStart) + (c - cStart);

    4226:       DMPlexSetSupportSize(rdm, newp, 4);
    4227:     }
    4228:     break;
    4229:   case 3:
    4230:     /* Hybrid 2D */
    4231:     if (cMax < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No cell maximum specified in hybrid mesh");
    4232:     cMax = PetscMin(cEnd, cMax);
    4233:     if (fMax < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No face maximum specified in hybrid mesh");
    4234:     fMax = PetscMin(fEnd, fMax);
    4235:     DMPlexSetHybridBounds(rdm, cStartNew + (cMax - cStart)*4, fStartNew + (fMax - fStart)*2 + (cMax - cStart)*3, PETSC_DETERMINE, PETSC_DETERMINE);
    4236:     /* Interior cells have 3 faces */
    4237:     for (c = cStart; c < cMax; ++c) {
    4238:       for (r = 0; r < 4; ++r) {
    4239:         const PetscInt newp = cStartNew + (c - cStart)*4 + r;

    4241:         DMPlexSetConeSize(rdm, newp, 3);
    4242:       }
    4243:     }
    4244:     /* Hybrid cells have 4 faces */
    4245:     for (c = cMax; c < cEnd; ++c) {
    4246:       for (r = 0; r < 2; ++r) {
    4247:         const PetscInt newp = cStartNew + (cMax - cStart)*4 + (c - cMax)*2 + r;

    4249:         DMPlexSetConeSize(rdm, newp, 4);
    4250:       }
    4251:     }
    4252:     /* Interior split faces have 2 vertices and the same cells as the parent */
    4253:     for (f = fStart; f < fMax; ++f) {
    4254:       for (r = 0; r < 2; ++r) {
    4255:         const PetscInt newp = fStartNew + (f - fStart)*2 + r;
    4256:         PetscInt       size;

    4258:         DMPlexSetConeSize(rdm, newp, 2);
    4259:         DMPlexGetSupportSize(dm, f, &size);
    4260:         DMPlexSetSupportSize(rdm, newp, size);
    4261:       }
    4262:     }
    4263:     /* Interior cell faces have 2 vertices and 2 cells */
    4264:     for (c = cStart; c < cMax; ++c) {
    4265:       for (r = 0; r < 3; ++r) {
    4266:         const PetscInt newp = fStartNew + (fMax - fStart)*2 + (c - cStart)*3 + r;

    4268:         DMPlexSetConeSize(rdm, newp, 2);
    4269:         DMPlexSetSupportSize(rdm, newp, 2);
    4270:       }
    4271:     }
    4272:     /* Hybrid faces have 2 vertices and the same cells */
    4273:     for (f = fMax; f < fEnd; ++f) {
    4274:       const PetscInt newp = fStartNew + (fMax - fStart)*2 + (cMax - cStart)*3 + (f - fMax);
    4275:       PetscInt       size;

    4277:       DMPlexSetConeSize(rdm, newp, 2);
    4278:       DMPlexGetSupportSize(dm, f, &size);
    4279:       DMPlexSetSupportSize(rdm, newp, size);
    4280:     }
    4281:     /* Hybrid cell faces have 2 vertices and 2 cells */
    4282:     for (c = cMax; c < cEnd; ++c) {
    4283:       const PetscInt newp = fStartNew + (fMax - fStart)*2 + (cMax - cStart)*3 + (fEnd - fMax) + (c - cMax);

    4285:       DMPlexSetConeSize(rdm, newp, 2);
    4286:       DMPlexSetSupportSize(rdm, newp, 2);
    4287:     }
    4288:     /* Old vertices have identical supports */
    4289:     for (v = vStart; v < vEnd; ++v) {
    4290:       const PetscInt newp = vStartNew + (v - vStart);
    4291:       PetscInt       size;

    4293:       DMPlexGetSupportSize(dm, v, &size);
    4294:       DMPlexSetSupportSize(rdm, newp, size);
    4295:     }
    4296:     /* Face vertices have 2 + (2 interior, 1 hybrid) supports */
    4297:     for (f = fStart; f < fMax; ++f) {
    4298:       const PetscInt newp = vStartNew + (vEnd - vStart) + (f - fStart);
    4299:       const PetscInt *support;
    4300:       PetscInt       size, newSize = 2, s;

    4302:       DMPlexGetSupportSize(dm, f, &size);
    4303:       DMPlexGetSupport(dm, f, &support);
    4304:       for (s = 0; s < size; ++s) {
    4305:         if (support[s] >= cMax) newSize += 1;
    4306:         else newSize += 2;
    4307:       }
    4308:       DMPlexSetSupportSize(rdm, newp, newSize);
    4309:     }
    4310:     break;
    4311:   default:
    4312:     SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown cell refiner %d", refiner);
    4313:   }
    4314:   return(0);
    4315: }

    4319: PetscErrorCode CellRefinerSetCones(CellRefiner refiner, DM dm, PetscInt depthSize[], DM rdm)
    4320: {
    4321:   PetscInt       depth, cStart, cEnd, cMax, cStartNew, cEndNew, c, vStart, vEnd, vMax, vStartNew, vEndNew, v, fStart, fEnd, fMax, fStartNew, fEndNew, f, eStart, eEnd, eMax, eStartNew, eEndNew, r, p;
    4322:   PetscInt       maxSupportSize, *supportRef;

    4326:   DMPlexGetDepth(dm, &depth);
    4327:   DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);
    4328:   DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);
    4329:   DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);
    4330:   DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);
    4331:   DMPlexGetHybridBounds(dm, &cMax, &fMax, &eMax, &vMax);
    4332:   GetDepthStart_Private(depth, depthSize, &cStartNew, &fStartNew, &eStartNew, &vStartNew);
    4333:   GetDepthEnd_Private(depth, depthSize, &cEndNew, &fEndNew, &eEndNew, &vEndNew);
    4334:   switch (refiner) {
    4335:   case 1:
    4336:     /* Simplicial 2D */
    4337:     /*
    4338:      2
    4339:      |\
    4340:      | \
    4341:      |  \
    4342:      |   \
    4343:      | C  \
    4344:      |     \
    4345:      |      \
    4346:      2---1---1
    4347:      |\  D  / \
    4348:      | 2   0   \
    4349:      |A \ /  B  \
    4350:      0---0-------1
    4351:      */
    4352:     /* All cells have 3 faces */
    4353:     for (c = cStart; c < cEnd; ++c) {
    4354:       const PetscInt  newp = cStartNew + (c - cStart)*4;
    4355:       const PetscInt *cone, *ornt;
    4356:       PetscInt        coneNew[3], orntNew[3];

    4358:       DMPlexGetCone(dm, c, &cone);
    4359:       DMPlexGetConeOrientation(dm, c, &ornt);
    4360:       /* A triangle */
    4361:       coneNew[0] = fStartNew + (cone[0] - fStart)*2 + (ornt[0] < 0 ? 1 : 0);
    4362:       orntNew[0] = ornt[0];
    4363:       coneNew[1] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*3 + 2;
    4364:       orntNew[1] = -2;
    4365:       coneNew[2] = fStartNew + (cone[2] - fStart)*2 + (ornt[2] < 0 ? 0 : 1);
    4366:       orntNew[2] = ornt[2];
    4367:       DMPlexSetCone(rdm, newp+0, coneNew);
    4368:       DMPlexSetConeOrientation(rdm, newp+0, orntNew);
    4369: #if 1
    4370:       if ((newp+0 < cStartNew) || (newp+0 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+0, cStartNew, cEndNew);
    4371:       for (p = 0; p < 3; ++p) {
    4372:         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
    4373:       }
    4374: #endif
    4375:       /* B triangle */
    4376:       coneNew[0] = fStartNew + (cone[0] - fStart)*2 + (ornt[0] < 0 ? 0 : 1);
    4377:       orntNew[0] = ornt[0];
    4378:       coneNew[1] = fStartNew + (cone[1] - fStart)*2 + (ornt[1] < 0 ? 1 : 0);
    4379:       orntNew[1] = ornt[1];
    4380:       coneNew[2] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*3 + 0;
    4381:       orntNew[2] = -2;
    4382:       DMPlexSetCone(rdm, newp+1, coneNew);
    4383:       DMPlexSetConeOrientation(rdm, newp+1, orntNew);
    4384: #if 1
    4385:       if ((newp+1 < cStartNew) || (newp+1 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+1, cStartNew, cEndNew);
    4386:       for (p = 0; p < 3; ++p) {
    4387:         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
    4388:       }
    4389: #endif
    4390:       /* C triangle */
    4391:       coneNew[0] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*3 + 1;
    4392:       orntNew[0] = -2;
    4393:       coneNew[1] = fStartNew + (cone[1] - fStart)*2 + (ornt[1] < 0 ? 0 : 1);
    4394:       orntNew[1] = ornt[1];
    4395:       coneNew[2] = fStartNew + (cone[2] - fStart)*2 + (ornt[2] < 0 ? 1 : 0);
    4396:       orntNew[2] = ornt[2];
    4397:       DMPlexSetCone(rdm, newp+2, coneNew);
    4398:       DMPlexSetConeOrientation(rdm, newp+2, orntNew);
    4399: #if 1
    4400:       if ((newp+2 < cStartNew) || (newp+2 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+2, cStartNew, cEndNew);
    4401:       for (p = 0; p < 3; ++p) {
    4402:         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
    4403:       }
    4404: #endif
    4405:       /* D triangle */
    4406:       coneNew[0] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*3 + 0;
    4407:       orntNew[0] = 0;
    4408:       coneNew[1] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*3 + 1;
    4409:       orntNew[1] = 0;
    4410:       coneNew[2] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*3 + 2;
    4411:       orntNew[2] = 0;
    4412:       DMPlexSetCone(rdm, newp+3, coneNew);
    4413:       DMPlexSetConeOrientation(rdm, newp+3, orntNew);
    4414: #if 1
    4415:       if ((newp+3 < cStartNew) || (newp+3 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+3, cStartNew, cEndNew);
    4416:       for (p = 0; p < 3; ++p) {
    4417:         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
    4418:       }
    4419: #endif
    4420:     }
    4421:     /* Split faces have 2 vertices and the same cells as the parent */
    4422:     DMPlexGetMaxSizes(dm, NULL, &maxSupportSize);
    4423:     PetscMalloc((2 + maxSupportSize*2) * sizeof(PetscInt), &supportRef);
    4424:     for (f = fStart; f < fEnd; ++f) {
    4425:       const PetscInt newv = vStartNew + (vEnd - vStart) + (f - fStart);

    4427:       for (r = 0; r < 2; ++r) {
    4428:         const PetscInt  newp = fStartNew + (f - fStart)*2 + r;
    4429:         const PetscInt *cone, *support;
    4430:         PetscInt        coneNew[2], coneSize, c, supportSize, s;

    4432:         DMPlexGetCone(dm, f, &cone);
    4433:         coneNew[0]       = vStartNew + (cone[0] - vStart);
    4434:         coneNew[1]       = vStartNew + (cone[1] - vStart);
    4435:         coneNew[(r+1)%2] = newv;
    4436:         DMPlexSetCone(rdm, newp, coneNew);
    4437: #if 1
    4438:         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
    4439:         for (p = 0; p < 2; ++p) {
    4440:           if ((coneNew[p] < vStartNew) || (coneNew[p] >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", coneNew[p], vStartNew, vEndNew);
    4441:         }
    4442: #endif
    4443:         DMPlexGetSupportSize(dm, f, &supportSize);
    4444:         DMPlexGetSupport(dm, f, &support);
    4445:         for (s = 0; s < supportSize; ++s) {
    4446:           DMPlexGetConeSize(dm, support[s], &coneSize);
    4447:           DMPlexGetCone(dm, support[s], &cone);
    4448:           for (c = 0; c < coneSize; ++c) {
    4449:             if (cone[c] == f) break;
    4450:           }
    4451:           supportRef[s] = cStartNew + (support[s] - cStart)*4 + (c+r)%3;
    4452:         }
    4453:         DMPlexSetSupport(rdm, newp, supportRef);
    4454: #if 1
    4455:         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
    4456:         for (p = 0; p < supportSize; ++p) {
    4457:           if ((supportRef[p] < cStartNew) || (supportRef[p] >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", supportRef[p], cStartNew, cEndNew);
    4458:         }
    4459: #endif
    4460:       }
    4461:     }
    4462:     /* Interior faces have 2 vertices and 2 cells */
    4463:     for (c = cStart; c < cEnd; ++c) {
    4464:       const PetscInt *cone;

    4466:       DMPlexGetCone(dm, c, &cone);
    4467:       for (r = 0; r < 3; ++r) {
    4468:         const PetscInt newp = fStartNew + (fEnd - fStart)*2 + (c - cStart)*3 + r;
    4469:         PetscInt       coneNew[2];
    4470:         PetscInt       supportNew[2];

    4472:         coneNew[0] = vStartNew + (vEnd - vStart) + (cone[r]       - fStart);
    4473:         coneNew[1] = vStartNew + (vEnd - vStart) + (cone[(r+1)%3] - fStart);
    4474:         DMPlexSetCone(rdm, newp, coneNew);
    4475: #if 1
    4476:         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
    4477:         for (p = 0; p < 2; ++p) {
    4478:           if ((coneNew[p] < vStartNew) || (coneNew[p] >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", coneNew[p], vStartNew, vEndNew);
    4479:         }
    4480: #endif
    4481:         supportNew[0] = (c - cStart)*4 + (r+1)%3;
    4482:         supportNew[1] = (c - cStart)*4 + 3;
    4483:         DMPlexSetSupport(rdm, newp, supportNew);
    4484: #if 1
    4485:         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
    4486:         for (p = 0; p < 2; ++p) {
    4487:           if ((supportNew[p] < cStartNew) || (supportNew[p] >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", supportNew[p], cStartNew, cEndNew);
    4488:         }
    4489: #endif
    4490:       }
    4491:     }
    4492:     /* Old vertices have identical supports */
    4493:     for (v = vStart; v < vEnd; ++v) {
    4494:       const PetscInt  newp = vStartNew + (v - vStart);
    4495:       const PetscInt *support, *cone;
    4496:       PetscInt        size, s;

    4498:       DMPlexGetSupportSize(dm, v, &size);
    4499:       DMPlexGetSupport(dm, v, &support);
    4500:       for (s = 0; s < size; ++s) {
    4501:         PetscInt r = 0;

    4503:         DMPlexGetCone(dm, support[s], &cone);
    4504:         if (cone[1] == v) r = 1;
    4505:         supportRef[s] = fStartNew + (support[s] - fStart)*2 + r;
    4506:       }
    4507:       DMPlexSetSupport(rdm, newp, supportRef);
    4508: #if 1
    4509:       if ((newp < vStartNew) || (newp >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", newp, vStartNew, vEndNew);
    4510:       for (p = 0; p < size; ++p) {
    4511:         if ((supportRef[p] < fStartNew) || (supportRef[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", supportRef[p], fStartNew, fEndNew);
    4512:       }
    4513: #endif
    4514:     }
    4515:     /* Face vertices have 2 + cells*2 supports */
    4516:     for (f = fStart; f < fEnd; ++f) {
    4517:       const PetscInt  newp = vStartNew + (vEnd - vStart) + (f - fStart);
    4518:       const PetscInt *cone, *support;
    4519:       PetscInt        size, s;

    4521:       DMPlexGetSupportSize(dm, f, &size);
    4522:       DMPlexGetSupport(dm, f, &support);
    4523:       supportRef[0] = fStartNew + (f - fStart)*2 + 0;
    4524:       supportRef[1] = fStartNew + (f - fStart)*2 + 1;
    4525:       for (s = 0; s < size; ++s) {
    4526:         PetscInt r = 0;

    4528:         DMPlexGetCone(dm, support[s], &cone);
    4529:         if      (cone[1] == f) r = 1;
    4530:         else if (cone[2] == f) r = 2;
    4531:         supportRef[2+s*2+0] = fStartNew + (fEnd - fStart)*2 + (support[s] - cStart)*3 + (r+2)%3;
    4532:         supportRef[2+s*2+1] = fStartNew + (fEnd - fStart)*2 + (support[s] - cStart)*3 + r;
    4533:       }
    4534:       DMPlexSetSupport(rdm, newp, supportRef);
    4535: #if 1
    4536:       if ((newp < vStartNew) || (newp >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", newp, vStartNew, vEndNew);
    4537:       for (p = 0; p < 2+size*2; ++p) {
    4538:         if ((supportRef[p] < fStartNew) || (supportRef[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", supportRef[p], fStartNew, fEndNew);
    4539:       }
    4540: #endif
    4541:     }
    4542:     PetscFree(supportRef);
    4543:     break;
    4544:   case 2:
    4545:     /* Hex 2D */
    4546:     /*
    4547:      3---------2---------2
    4548:      |         |         |
    4549:      |    D    2    C    |
    4550:      |         |         |
    4551:      3----3----0----1----1
    4552:      |         |         |
    4553:      |    A    0    B    |
    4554:      |         |         |
    4555:      0---------0---------1
    4556:      */
    4557:     /* All cells have 4 faces */
    4558:     for (c = cStart; c < cEnd; ++c) {
    4559:       const PetscInt  newp = (c - cStart)*4;
    4560:       const PetscInt *cone, *ornt;
    4561:       PetscInt        coneNew[4], orntNew[4];

    4563:       DMPlexGetCone(dm, c, &cone);
    4564:       DMPlexGetConeOrientation(dm, c, &ornt);
    4565:       /* A quad */
    4566:       coneNew[0] = fStartNew + (cone[0] - fStart)*2 + (ornt[0] < 0 ? 1 : 0);
    4567:       orntNew[0] = ornt[0];
    4568:       coneNew[1] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*4 + 0;
    4569:       orntNew[1] = 0;
    4570:       coneNew[2] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*4 + 3;
    4571:       orntNew[2] = -2;
    4572:       coneNew[3] = fStartNew + (cone[3] - fStart)*2 + (ornt[3] < 0 ? 0 : 1);
    4573:       orntNew[3] = ornt[3];
    4574:       DMPlexSetCone(rdm, newp+0, coneNew);
    4575:       DMPlexSetConeOrientation(rdm, newp+0, orntNew);
    4576: #if 1
    4577:       if ((newp+0 < cStartNew) || (newp+0 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+0, cStartNew, cEndNew);
    4578:       for (p = 0; p < 4; ++p) {
    4579:         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
    4580:       }
    4581: #endif
    4582:       /* B quad */
    4583:       coneNew[0] = fStartNew + (cone[0] - fStart)*2 + (ornt[0] < 0 ? 0 : 1);
    4584:       orntNew[0] = ornt[0];
    4585:       coneNew[1] = fStartNew + (cone[1] - fStart)*2 + (ornt[1] < 0 ? 1 : 0);
    4586:       orntNew[1] = ornt[1];
    4587:       coneNew[2] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*4 + 1;
    4588:       orntNew[2] = 0;
    4589:       coneNew[3] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*4 + 0;
    4590:       orntNew[3] = -2;
    4591:       DMPlexSetCone(rdm, newp+1, coneNew);
    4592:       DMPlexSetConeOrientation(rdm, newp+1, orntNew);
    4593: #if 1
    4594:       if ((newp+1 < cStartNew) || (newp+1 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+1, cStartNew, cEndNew);
    4595:       for (p = 0; p < 4; ++p) {
    4596:         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
    4597:       }
    4598: #endif
    4599:       /* C quad */
    4600:       coneNew[0] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*4 + 1;
    4601:       orntNew[0] = -2;
    4602:       coneNew[1] = fStartNew + (cone[1] - fStart)*2 + (ornt[1] < 0 ? 0 : 1);
    4603:       orntNew[1] = ornt[1];
    4604:       coneNew[2] = fStartNew + (cone[2] - fStart)*2 + (ornt[2] < 0 ? 1 : 0);
    4605:       orntNew[2] = ornt[2];
    4606:       coneNew[3] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*4 + 2;
    4607:       orntNew[3] = 0;
    4608:       DMPlexSetCone(rdm, newp+2, coneNew);
    4609:       DMPlexSetConeOrientation(rdm, newp+2, orntNew);
    4610: #if 1
    4611:       if ((newp+2 < cStartNew) || (newp+2 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+2, cStartNew, cEndNew);
    4612:       for (p = 0; p < 4; ++p) {
    4613:         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
    4614:       }
    4615: #endif
    4616:       /* D quad */
    4617:       coneNew[0] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*4 + 3;
    4618:       orntNew[0] = 0;
    4619:       coneNew[1] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*4 + 2;
    4620:       orntNew[1] = -2;
    4621:       coneNew[2] = fStartNew + (cone[2] - fStart)*2 + (ornt[2] < 0 ? 0 : 1);
    4622:       orntNew[2] = ornt[2];
    4623:       coneNew[3] = fStartNew + (cone[3] - fStart)*2 + (ornt[3] < 0 ? 1 : 0);
    4624:       orntNew[3] = ornt[3];
    4625:       DMPlexSetCone(rdm, newp+3, coneNew);
    4626:       DMPlexSetConeOrientation(rdm, newp+3, orntNew);
    4627: #if 1
    4628:       if ((newp+3 < cStartNew) || (newp+3 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+3, cStartNew, cEndNew);
    4629:       for (p = 0; p < 4; ++p) {
    4630:         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
    4631:       }
    4632: #endif
    4633:     }
    4634:     /* Split faces have 2 vertices and the same cells as the parent */
    4635:     DMPlexGetMaxSizes(dm, NULL, &maxSupportSize);
    4636:     PetscMalloc((2 + maxSupportSize*2) * sizeof(PetscInt), &supportRef);
    4637:     for (f = fStart; f < fEnd; ++f) {
    4638:       const PetscInt newv = vStartNew + (vEnd - vStart) + (f - fStart);

    4640:       for (r = 0; r < 2; ++r) {
    4641:         const PetscInt  newp = fStartNew + (f - fStart)*2 + r;
    4642:         const PetscInt *cone, *support;
    4643:         PetscInt        coneNew[2], coneSize, c, supportSize, s;

    4645:         DMPlexGetCone(dm, f, &cone);
    4646:         coneNew[0]       = vStartNew + (cone[0] - vStart);
    4647:         coneNew[1]       = vStartNew + (cone[1] - vStart);
    4648:         coneNew[(r+1)%2] = newv;
    4649:         DMPlexSetCone(rdm, newp, coneNew);
    4650: #if 1
    4651:         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
    4652:         for (p = 0; p < 2; ++p) {
    4653:           if ((coneNew[p] < vStartNew) || (coneNew[p] >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", coneNew[p], vStartNew, vEndNew);
    4654:         }
    4655: #endif
    4656:         DMPlexGetSupportSize(dm, f, &supportSize);
    4657:         DMPlexGetSupport(dm, f, &support);
    4658:         for (s = 0; s < supportSize; ++s) {
    4659:           DMPlexGetConeSize(dm, support[s], &coneSize);
    4660:           DMPlexGetCone(dm, support[s], &cone);
    4661:           for (c = 0; c < coneSize; ++c) {
    4662:             if (cone[c] == f) break;
    4663:           }
    4664:           supportRef[s] = cStartNew + (support[s] - cStart)*4 + (c+r)%4;
    4665:         }
    4666:         DMPlexSetSupport(rdm, newp, supportRef);
    4667: #if 1
    4668:         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
    4669:         for (p = 0; p < supportSize; ++p) {
    4670:           if ((supportRef[p] < cStartNew) || (supportRef[p] >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", supportRef[p], cStartNew, cEndNew);
    4671:         }
    4672: #endif
    4673:       }
    4674:     }
    4675:     /* Interior faces have 2 vertices and 2 cells */
    4676:     for (c = cStart; c < cEnd; ++c) {
    4677:       const PetscInt *cone;
    4678:       PetscInt        coneNew[2], supportNew[2];

    4680:       DMPlexGetCone(dm, c, &cone);
    4681:       for (r = 0; r < 4; ++r) {
    4682:         const PetscInt newp = fStartNew + (fEnd - fStart)*2 + (c - cStart)*4 + r;

    4684:         coneNew[0] = vStartNew + (vEnd - vStart) + (cone[r] - fStart);
    4685:         coneNew[1] = vStartNew + (vEnd - vStart) + (fEnd    - fStart) + (c - cStart);
    4686:         DMPlexSetCone(rdm, newp, coneNew);
    4687: #if 1
    4688:         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
    4689:         for (p = 0; p < 2; ++p) {
    4690:           if ((coneNew[p] < vStartNew) || (coneNew[p] >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", coneNew[p], vStartNew, vEndNew);
    4691:         }
    4692: #endif
    4693:         supportNew[0] = (c - cStart)*4 + r;
    4694:         supportNew[1] = (c - cStart)*4 + (r+1)%4;
    4695:         DMPlexSetSupport(rdm, newp, supportNew);
    4696: #if 1
    4697:         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
    4698:         for (p = 0; p < 2; ++p) {
    4699:           if ((supportNew[p] < cStartNew) || (supportNew[p] >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", supportNew[p], cStartNew, cEndNew);
    4700:         }
    4701: #endif
    4702:       }
    4703:     }
    4704:     /* Old vertices have identical supports */
    4705:     for (v = vStart; v < vEnd; ++v) {
    4706:       const PetscInt  newp = vStartNew + (v - vStart);
    4707:       const PetscInt *support, *cone;
    4708:       PetscInt        size, s;

    4710:       DMPlexGetSupportSize(dm, v, &size);
    4711:       DMPlexGetSupport(dm, v, &support);
    4712:       for (s = 0; s < size; ++s) {
    4713:         PetscInt r = 0;

    4715:         DMPlexGetCone(dm, support[s], &cone);
    4716:         if (cone[1] == v) r = 1;
    4717:         supportRef[s] = fStartNew + (support[s] - fStart)*2 + r;
    4718:       }
    4719:       DMPlexSetSupport(rdm, newp, supportRef);
    4720: #if 1
    4721:       if ((newp < vStartNew) || (newp >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", newp, vStartNew, vEndNew);
    4722:       for (p = 0; p < size; ++p) {
    4723:         if ((supportRef[p] < fStartNew) || (supportRef[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", supportRef[p], fStartNew, fEndNew);
    4724:       }
    4725: #endif
    4726:     }
    4727:     /* Face vertices have 2 + cells supports */
    4728:     for (f = fStart; f < fEnd; ++f) {
    4729:       const PetscInt  newp = vStartNew + (vEnd - vStart) + (f - fStart);
    4730:       const PetscInt *cone, *support;
    4731:       PetscInt        size, s;

    4733:       DMPlexGetSupportSize(dm, f, &size);
    4734:       DMPlexGetSupport(dm, f, &support);
    4735:       supportRef[0] = fStartNew + (f - fStart)*2 + 0;
    4736:       supportRef[1] = fStartNew + (f - fStart)*2 + 1;
    4737:       for (s = 0; s < size; ++s) {
    4738:         PetscInt r = 0;

    4740:         DMPlexGetCone(dm, support[s], &cone);
    4741:         if      (cone[1] == f) r = 1;
    4742:         else if (cone[2] == f) r = 2;
    4743:         else if (cone[3] == f) r = 3;
    4744:         supportRef[2+s] = fStartNew + (fEnd - fStart)*2 + (support[s] - cStart)*4 + r;
    4745:       }
    4746:       DMPlexSetSupport(rdm, newp, supportRef);
    4747: #if 1
    4748:       if ((newp < vStartNew) || (newp >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", newp, vStartNew, vEndNew);
    4749:       for (p = 0; p < 2+size; ++p) {
    4750:         if ((supportRef[p] < fStartNew) || (supportRef[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", supportRef[p], fStartNew, fEndNew);
    4751:       }
    4752: #endif
    4753:     }
    4754:     /* Cell vertices have 4 supports */
    4755:     for (c = cStart; c < cEnd; ++c) {
    4756:       const PetscInt newp = vStartNew + (vEnd - vStart) + (fEnd - fStart) + (c - cStart);
    4757:       PetscInt       supportNew[4];

    4759:       for (r = 0; r < 4; ++r) {
    4760:         supportNew[r] = fStartNew + (fEnd - fStart)*2 + (c - cStart)*4 + r;
    4761:       }
    4762:       DMPlexSetSupport(rdm, newp, supportNew);
    4763:     }
    4764:     break;
    4765:   case 3:
    4766:     if (cMax < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No cell maximum specified in hybrid mesh");
    4767:     cMax = PetscMin(cEnd, cMax);
    4768:     if (fMax < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No face maximum specified in hybrid mesh");
    4769:     fMax = PetscMin(fEnd, fMax);
    4770:     /* Interior cells have 3 faces */
    4771:     for (c = cStart; c < cMax; ++c) {
    4772:       const PetscInt  newp = cStartNew + (c - cStart)*4;
    4773:       const PetscInt *cone, *ornt;
    4774:       PetscInt        coneNew[3], orntNew[3];

    4776:       DMPlexGetCone(dm, c, &cone);
    4777:       DMPlexGetConeOrientation(dm, c, &ornt);
    4778:       /* A triangle */
    4779:       coneNew[0] = fStartNew + (cone[0] - fStart)*2 + (ornt[0] < 0 ? 1 : 0);
    4780:       orntNew[0] = ornt[0];
    4781:       coneNew[1] = fStartNew + (fMax    - fStart)*2 + (c - cStart)*3 + 2;
    4782:       orntNew[1] = -2;
    4783:       coneNew[2] = fStartNew + (cone[2] - fStart)*2 + (ornt[2] < 0 ? 0 : 1);
    4784:       orntNew[2] = ornt[2];
    4785:       DMPlexSetCone(rdm, newp+0, coneNew);
    4786:       DMPlexSetConeOrientation(rdm, newp+0, orntNew);
    4787: #if 1
    4788:       if ((newp+0 < cStartNew) || (newp+0 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+0, cStartNew, cEndNew);
    4789:       for (p = 0; p < 3; ++p) {
    4790:         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
    4791:       }
    4792: #endif
    4793:       /* B triangle */
    4794:       coneNew[0] = fStartNew + (cone[0] - fStart)*2 + (ornt[0] < 0 ? 0 : 1);
    4795:       orntNew[0] = ornt[0];
    4796:       coneNew[1] = fStartNew + (cone[1] - fStart)*2 + (ornt[1] < 0 ? 1 : 0);
    4797:       orntNew[1] = ornt[1];
    4798:       coneNew[2] = fStartNew + (fMax    - fStart)*2 + (c - cStart)*3 + 0;
    4799:       orntNew[2] = -2;
    4800:       DMPlexSetCone(rdm, newp+1, coneNew);
    4801:       DMPlexSetConeOrientation(rdm, newp+1, orntNew);
    4802: #if 1
    4803:       if ((newp+1 < cStartNew) || (newp+1 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+1, cStartNew, cEndNew);
    4804:       for (p = 0; p < 3; ++p) {
    4805:         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
    4806:       }
    4807: #endif
    4808:       /* C triangle */
    4809:       coneNew[0] = fStartNew + (fMax    - fStart)*2 + (c - cStart)*3 + 1;
    4810:       orntNew[0] = -2;
    4811:       coneNew[1] = fStartNew + (cone[1] - fStart)*2 + (ornt[1] < 0 ? 0 : 1);
    4812:       orntNew[1] = ornt[1];
    4813:       coneNew[2] = fStartNew + (cone[2] - fStart)*2 + (ornt[2] < 0 ? 1 : 0);
    4814:       orntNew[2] = ornt[2];
    4815:       DMPlexSetCone(rdm, newp+2, coneNew);
    4816:       DMPlexSetConeOrientation(rdm, newp+2, orntNew);
    4817: #if 1
    4818:       if ((newp+2 < cStartNew) || (newp+2 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+2, cStartNew, cEndNew);
    4819:       for (p = 0; p < 3; ++p) {
    4820:         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
    4821:       }
    4822: #endif
    4823:       /* D triangle */
    4824:       coneNew[0] = fStartNew + (fMax    - fStart)*2 + (c - cStart)*3 + 0;
    4825:       orntNew[0] = 0;
    4826:       coneNew[1] = fStartNew + (fMax    - fStart)*2 + (c - cStart)*3 + 1;
    4827:       orntNew[1] = 0;
    4828:       coneNew[2] = fStartNew + (fMax    - fStart)*2 + (c - cStart)*3 + 2;
    4829:       orntNew[2] = 0;
    4830:       DMPlexSetCone(rdm, newp+3, coneNew);
    4831:       DMPlexSetConeOrientation(rdm, newp+3, orntNew);
    4832: #if 1
    4833:       if ((newp+3 < cStartNew) || (newp+3 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+3, cStartNew, cEndNew);
    4834:       for (p = 0; p < 3; ++p) {
    4835:         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
    4836:       }
    4837: #endif
    4838:     }
    4839:     /*
    4840:      2----3----3
    4841:      |         |
    4842:      |    B    |
    4843:      |         |
    4844:      0----4--- 1
    4845:      |         |
    4846:      |    A    |
    4847:      |         |
    4848:      0----2----1
    4849:      */
    4850:     /* Hybrid cells have 4 faces */
    4851:     for (c = cMax; c < cEnd; ++c) {
    4852:       const PetscInt  newp = cStartNew + (cMax - cStart)*4 + (c - cMax)*2;
    4853:       const PetscInt *cone, *ornt;
    4854:       PetscInt        coneNew[4], orntNew[4];

    4856:       DMPlexGetCone(dm, c, &cone);
    4857:       DMPlexGetConeOrientation(dm, c, &ornt);
    4858:       /* A quad */
    4859:       coneNew[0] = fStartNew + (cone[0] - fStart)*2 + (ornt[0] < 0 ? 1 : 0);
    4860:       orntNew[0] = ornt[0];
    4861:       coneNew[1] = fStartNew + (cone[1] - fStart)*2 + (ornt[1] < 0 ? 1 : 0);
    4862:       orntNew[1] = ornt[1];
    4863:       coneNew[2] = fStartNew + (fMax    - fStart)*2 + (cMax - cStart)*3 + (cone[2] - fMax);
    4864:       orntNew[2] = 0;
    4865:       coneNew[3] = fStartNew + (fMax    - fStart)*2 + (cMax - cStart)*3 + (fEnd    - fMax) + (c - cMax);
    4866:       orntNew[3] = 0;
    4867:       DMPlexSetCone(rdm, newp+0, coneNew);
    4868:       DMPlexSetConeOrientation(rdm, newp+0, orntNew);
    4869: #if 1
    4870:       if ((newp+0 < cStartNew) || (newp+0 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+0, cStartNew, cEndNew);
    4871:       for (p = 0; p < 4; ++p) {
    4872:         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
    4873:       }
    4874: #endif
    4875:       /* B quad */
    4876:       coneNew[0] = fStartNew + (cone[0] - fStart)*2 + (ornt[0] < 0 ? 0 : 1);
    4877:       orntNew[0] = ornt[0];
    4878:       coneNew[1] = fStartNew + (cone[1] - fStart)*2 + (ornt[1] < 0 ? 0 : 1);
    4879:       orntNew[1] = ornt[1];
    4880:       coneNew[2] = fStartNew + (fMax    - fStart)*2 + (cMax - cStart)*3 + (fEnd    - fMax) + (c - cMax);
    4881:       orntNew[2] = 0;
    4882:       coneNew[3] = fStartNew + (fMax    - fStart)*2 + (cMax - cStart)*3 + (cone[3] - fMax);
    4883:       orntNew[3] = 0;
    4884:       DMPlexSetCone(rdm, newp+1, coneNew);
    4885:       DMPlexSetConeOrientation(rdm, newp+1, orntNew);
    4886: #if 1
    4887:       if ((newp+1 < cStartNew) || (newp+1 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+1, cStartNew, cEndNew);
    4888:       for (p = 0; p < 4; ++p) {
    4889:         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
    4890:       }
    4891: #endif
    4892:     }
    4893:     /* Interior split faces have 2 vertices and the same cells as the parent */
    4894:     DMPlexGetMaxSizes(dm, NULL, &maxSupportSize);
    4895:     PetscMalloc((2 + maxSupportSize*2) * sizeof(PetscInt), &supportRef);
    4896:     for (f = fStart; f < fMax; ++f) {
    4897:       const PetscInt newv = vStartNew + (vEnd - vStart) + (f - fStart);

    4899:       for (r = 0; r < 2; ++r) {
    4900:         const PetscInt  newp = fStartNew + (f - fStart)*2 + r;
    4901:         const PetscInt *cone, *support;
    4902:         PetscInt        coneNew[2], coneSize, c, supportSize, s;

    4904:         DMPlexGetCone(dm, f, &cone);
    4905:         coneNew[0]       = vStartNew + (cone[0] - vStart);
    4906:         coneNew[1]       = vStartNew + (cone[1] - vStart);
    4907:         coneNew[(r+1)%2] = newv;
    4908:         DMPlexSetCone(rdm, newp, coneNew);
    4909: #if 1
    4910:         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
    4911:         for (p = 0; p < 2; ++p) {
    4912:           if ((coneNew[p] < vStartNew) || (coneNew[p] >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", coneNew[p], vStartNew, vEndNew);
    4913:         }
    4914: #endif
    4915:         DMPlexGetSupportSize(dm, f, &supportSize);
    4916:         DMPlexGetSupport(dm, f, &support);
    4917:         for (s = 0; s < supportSize; ++s) {
    4918:           if (support[s] >= cMax) {
    4919:             supportRef[s] = cStartNew + (cMax - cStart)*4 + (support[s] - cMax)*2 + r;
    4920:           } else {
    4921:             DMPlexGetConeSize(dm, support[s], &coneSize);
    4922:             DMPlexGetCone(dm, support[s], &cone);
    4923:             for (c = 0; c < coneSize; ++c) {
    4924:               if (cone[c] == f) break;
    4925:             }
    4926:             supportRef[s] = cStartNew + (support[s] - cStart)*4 + (c+r)%3;
    4927:           }
    4928:         }
    4929:         DMPlexSetSupport(rdm, newp, supportRef);
    4930: #if 1
    4931:         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
    4932:         for (p = 0; p < supportSize; ++p) {
    4933:           if ((supportRef[p] < cStartNew) || (supportRef[p] >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", supportRef[p], cStartNew, cEndNew);
    4934:         }
    4935: #endif
    4936:       }
    4937:     }
    4938:     /* Interior cell faces have 2 vertices and 2 cells */
    4939:     for (c = cStart; c < cMax; ++c) {
    4940:       const PetscInt *cone;

    4942:       DMPlexGetCone(dm, c, &cone);
    4943:       for (r = 0; r < 3; ++r) {
    4944:         const PetscInt newp = fStartNew + (fMax - fStart)*2 + (c - cStart)*3 + r;
    4945:         PetscInt       coneNew[2];
    4946:         PetscInt       supportNew[2];

    4948:         coneNew[0] = vStartNew + (vEnd - vStart) + (cone[r]       - fStart);
    4949:         coneNew[1] = vStartNew + (vEnd - vStart) + (cone[(r+1)%3] - fStart);
    4950:         DMPlexSetCone(rdm, newp, coneNew);
    4951: #if 1
    4952:         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
    4953:         for (p = 0; p < 2; ++p) {
    4954:           if ((coneNew[p] < vStartNew) || (coneNew[p] >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", coneNew[p], vStartNew, vEndNew);
    4955:         }
    4956: #endif
    4957:         supportNew[0] = (c - cStart)*4 + (r+1)%3;
    4958:         supportNew[1] = (c - cStart)*4 + 3;
    4959:         DMPlexSetSupport(rdm, newp, supportNew);
    4960: #if 1
    4961:         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
    4962:         for (p = 0; p < 2; ++p) {
    4963:           if ((supportNew[p] < cStartNew) || (supportNew[p] >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", supportNew[p], cStartNew, cEndNew);
    4964:         }
    4965: #endif
    4966:       }
    4967:     }
    4968:     /* Interior hybrid faces have 2 vertices and the same cells */
    4969:     for (f = fMax; f < fEnd; ++f) {
    4970:       const PetscInt  newp = fStartNew + (fMax - fStart)*2 + (cMax - cStart)*3 + (f - fMax);
    4971:       const PetscInt *cone;
    4972:       const PetscInt *support;
    4973:       PetscInt        coneNew[2];
    4974:       PetscInt        supportNew[2];
    4975:       PetscInt        size, s, r;

    4977:       DMPlexGetCone(dm, f, &cone);
    4978:       coneNew[0] = vStartNew + (cone[0] - vStart);
    4979:       coneNew[1] = vStartNew + (cone[1] - vStart);
    4980:       DMPlexSetCone(rdm, newp, coneNew);
    4981: #if 1
    4982:       if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
    4983:       for (p = 0; p < 2; ++p) {
    4984:         if ((coneNew[p] < vStartNew) || (coneNew[p] >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", coneNew[p], vStartNew, vEndNew);
    4985:       }
    4986: #endif
    4987:       DMPlexGetSupportSize(dm, f, &size);
    4988:       DMPlexGetSupport(dm, f, &support);
    4989:       for (s = 0; s < size; ++s) {
    4990:         DMPlexGetCone(dm, support[s], &cone);
    4991:         for (r = 0; r < 2; ++r) {
    4992:           if (cone[r+2] == f) break;
    4993:         }
    4994:         supportNew[s] = (cMax - cStart)*4 + (support[s] - cMax)*2 + r;
    4995:       }
    4996:       DMPlexSetSupport(rdm, newp, supportNew);
    4997: #if 1
    4998:       if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
    4999:       for (p = 0; p < size; ++p) {
    5000:         if ((supportNew[p] < cStartNew) || (supportNew[p] >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", supportNew[p], cStartNew, cEndNew);
    5001:       }
    5002: #endif
    5003:     }
    5004:     /* Cell hybrid faces have 2 vertices and 2 cells */
    5005:     for (c = cMax; c < cEnd; ++c) {
    5006:       const PetscInt  newp = fStartNew + (fMax - fStart)*2 + (cMax - cStart)*3 + (fEnd - fMax) + (c - cMax);
    5007:       const PetscInt *cone;
    5008:       PetscInt        coneNew[2];
    5009:       PetscInt        supportNew[2];

    5011:       DMPlexGetCone(dm, c, &cone);
    5012:       coneNew[0] = vStartNew + (vEnd - vStart) + (cone[0] - fStart);
    5013:       coneNew[1] = vStartNew + (vEnd - vStart) + (cone[1] - fStart);
    5014:       DMPlexSetCone(rdm, newp, coneNew);
    5015: #if 1
    5016:       if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
    5017:       for (p = 0; p < 2; ++p) {
    5018:         if ((coneNew[p] < vStartNew) || (coneNew[p] >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", coneNew[p], vStartNew, vEndNew);
    5019:       }
    5020: #endif
    5021:       supportNew[0] = (cMax - cStart)*4 + (c - cMax)*2 + 0;
    5022:       supportNew[1] = (cMax - cStart)*4 + (c - cMax)*2 + 1;
    5023:       DMPlexSetSupport(rdm, newp, supportNew);
    5024: #if 1
    5025:       if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
    5026:       for (p = 0; p < 2; ++p) {
    5027:         if ((supportNew[p] < cStartNew) || (supportNew[p] >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", supportNew[p], cStartNew, cEndNew);
    5028:       }
    5029: #endif
    5030:     }
    5031:     /* Old vertices have identical supports */
    5032:     for (v = vStart; v < vEnd; ++v) {
    5033:       const PetscInt  newp = vStartNew + (v - vStart);
    5034:       const PetscInt *support, *cone;
    5035:       PetscInt        size, s;

    5037:       DMPlexGetSupportSize(dm, v, &size);
    5038:       DMPlexGetSupport(dm, v, &support);
    5039:       for (s = 0; s < size; ++s) {
    5040:         if (support[s] >= fMax) {
    5041:           supportRef[s] = fStartNew + (fMax - fStart)*2 + (cMax - cStart)*3 + (support[s] - fMax);
    5042:         } else {
    5043:           PetscInt r = 0;

    5045:           DMPlexGetCone(dm, support[s], &cone);
    5046:           if (cone[1] == v) r = 1;
    5047:           supportRef[s] = fStartNew + (support[s] - fStart)*2 + r;
    5048:         }
    5049:       }
    5050:       DMPlexSetSupport(rdm, newp, supportRef);
    5051: #if 1
    5052:       if ((newp < vStartNew) || (newp >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", newp, vStartNew, vEndNew);
    5053:       for (p = 0; p < size; ++p) {
    5054:         if ((supportRef[p] < fStartNew) || (supportRef[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", supportRef[p], fStartNew, fEndNew);
    5055:       }
    5056: #endif
    5057:     }
    5058:     /* Face vertices have 2 + (2 interior, 1 hybrid) supports */
    5059:     for (f = fStart; f < fMax; ++f) {
    5060:       const PetscInt  newp = vStartNew + (vEnd - vStart) + (f - fStart);
    5061:       const PetscInt *cone, *support;
    5062:       PetscInt        size, newSize = 2, s;

    5064:       DMPlexGetSupportSize(dm, f, &size);
    5065:       DMPlexGetSupport(dm, f, &support);
    5066:       supportRef[0] = fStartNew + (f - fStart)*2 + 0;
    5067:       supportRef[1] = fStartNew + (f - fStart)*2 + 1;
    5068:       for (s = 0; s < size; ++s) {
    5069:         PetscInt r = 0;

    5071:         DMPlexGetCone(dm, support[s], &cone);
    5072:         if (support[s] >= cMax) {
    5073:           supportRef[newSize+0] = fStartNew + (fMax - fStart)*2 + (cMax - cStart)*3 + (fEnd - fMax) + (support[s] - cMax);

    5075:           newSize += 1;
    5076:         } else {
    5077:           if      (cone[1] == f) r = 1;
    5078:           else if (cone[2] == f) r = 2;
    5079:           supportRef[newSize+0] = fStartNew + (fMax - fStart)*2 + (support[s] - cStart)*3 + (r+2)%3;
    5080:           supportRef[newSize+1] = fStartNew + (fMax - fStart)*2 + (support[s] - cStart)*3 + r;

    5082:           newSize += 2;
    5083:         }
    5084:       }
    5085:       DMPlexSetSupport(rdm, newp, supportRef);
    5086: #if 1
    5087:       if ((newp < vStartNew) || (newp >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", newp, vStartNew, vEndNew);
    5088:       for (p = 0; p < newSize; ++p) {
    5089:         if ((supportRef[p] < fStartNew) || (supportRef[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", supportRef[p], fStartNew, fEndNew);
    5090:       }
    5091: #endif
    5092:     }
    5093:     PetscFree(supportRef);
    5094:     break;
    5095:   default:
    5096:     SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown cell refiner %d", refiner);
    5097:   }
    5098:   return(0);
    5099: }

    5103: PetscErrorCode CellRefinerSetCoordinates(CellRefiner refiner, DM dm, PetscInt depthSize[], DM rdm)
    5104: {
    5105:   PetscSection   coordSection, coordSectionNew;
    5106:   Vec            coordinates, coordinatesNew;
    5107:   PetscScalar   *coords, *coordsNew;
    5108:   PetscInt       dim, depth, coordSizeNew, cStart, cEnd, c, vStart, vStartNew, vEnd, v, fStart, fEnd, fMax, f;

    5112:   DMPlexGetDimension(dm, &dim);
    5113:   DMPlexGetDepth(dm, &depth);
    5114:   DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);
    5115:   DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);
    5116:   DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);
    5117:   DMPlexGetHybridBounds(dm, NULL, &fMax, NULL, NULL);
    5118:   GetDepthStart_Private(depth, depthSize, NULL, NULL, NULL, &vStartNew);
    5119:   DMPlexGetCoordinateSection(dm, &coordSection);
    5120:   PetscSectionCreate(PetscObjectComm((PetscObject)dm), &coordSectionNew);
    5121:   PetscSectionSetNumFields(coordSectionNew, 1);
    5122:   PetscSectionSetFieldComponents(coordSectionNew, 0, dim);
    5123:   PetscSectionSetChart(coordSectionNew, vStartNew, vStartNew+depthSize[0]);
    5124:   if (fMax < 0) fMax = fEnd;
    5125:   switch (refiner) {
    5126:   case 1:
    5127:   case 2:
    5128:   case 3:
    5129:     /* Simplicial and Hex 2D */
    5130:     /* All vertices have the dim coordinates */
    5131:     for (v = vStartNew; v < vStartNew+depthSize[0]; ++v) {
    5132:       PetscSectionSetDof(coordSectionNew, v, dim);
    5133:       PetscSectionSetFieldDof(coordSectionNew, v, 0, dim);
    5134:     }
    5135:     PetscSectionSetUp(coordSectionNew);
    5136:     DMPlexSetCoordinateSection(rdm, coordSectionNew);
    5137:     DMGetCoordinatesLocal(dm, &coordinates);
    5138:     PetscSectionGetStorageSize(coordSectionNew, &coordSizeNew);
    5139:     VecCreate(PetscObjectComm((PetscObject)dm), &coordinatesNew);
    5140:     PetscObjectSetName((PetscObject) coordinatesNew, "coordinates");
    5141:     VecSetSizes(coordinatesNew, coordSizeNew, PETSC_DETERMINE);
    5142:     VecSetFromOptions(coordinatesNew);
    5143:     VecGetArray(coordinates, &coords);
    5144:     VecGetArray(coordinatesNew, &coordsNew);
    5145:     /* Old vertices have the same coordinates */
    5146:     for (v = vStart; v < vEnd; ++v) {
    5147:       const PetscInt newv = vStartNew + (v - vStart);
    5148:       PetscInt       off, offnew, d;

    5150:       PetscSectionGetOffset(coordSection, v, &off);
    5151:       PetscSectionGetOffset(coordSectionNew, newv, &offnew);
    5152:       for (d = 0; d < dim; ++d) {
    5153:         coordsNew[offnew+d] = coords[off+d];
    5154:       }
    5155:     }
    5156:     /* Face vertices have the average of endpoint coordinates */
    5157:     for (f = fStart; f < fMax; ++f) {
    5158:       const PetscInt  newv = vStartNew + (vEnd - vStart) + (f - fStart);
    5159:       const PetscInt *cone;
    5160:       PetscInt        coneSize, offA, offB, offnew, d;

    5162:       DMPlexGetConeSize(dm, f, &coneSize);
    5163:       if (coneSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Face %d cone should have two vertices, not %d", f, coneSize);
    5164:       DMPlexGetCone(dm, f, &cone);
    5165:       PetscSectionGetOffset(coordSection, cone[0], &offA);
    5166:       PetscSectionGetOffset(coordSection, cone[1], &offB);
    5167:       PetscSectionGetOffset(coordSectionNew, newv, &offnew);
    5168:       for (d = 0; d < dim; ++d) {
    5169:         coordsNew[offnew+d] = 0.5*(coords[offA+d] + coords[offB+d]);
    5170:       }
    5171:     }
    5172:     /* Just Hex 2D */
    5173:     if (refiner == 2) {
    5174:       /* Cell vertices have the average of corner coordinates */
    5175:       for (c = cStart; c < cEnd; ++c) {
    5176:         const PetscInt newv = vStartNew + (vEnd - vStart) + (fEnd - fStart) + (c - cStart);
    5177:         PetscInt      *cone = NULL;
    5178:         PetscInt       closureSize, coneSize = 0, offA, offB, offC, offD, offnew, p, d;

    5180:         DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &cone);
    5181:         for (p = 0; p < closureSize*2; p += 2) {
    5182:           const PetscInt point = cone[p];
    5183:           if ((point >= vStart) && (point < vEnd)) cone[coneSize++] = point;
    5184:         }
    5185:         if (coneSize != 4) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Quad %d cone should have four vertices, not %d", c, coneSize);
    5186:         PetscSectionGetOffset(coordSection, cone[0], &offA);
    5187:         PetscSectionGetOffset(coordSection, cone[1], &offB);
    5188:         PetscSectionGetOffset(coordSection, cone[2], &offC);
    5189:         PetscSectionGetOffset(coordSection, cone[3], &offD);
    5190:         PetscSectionGetOffset(coordSectionNew, newv, &offnew);
    5191:         for (d = 0; d < dim; ++d) {
    5192:           coordsNew[offnew+d] = 0.25*(coords[offA+d] + coords[offB+d] + coords[offC+d] + coords[offD+d]);
    5193:         }
    5194:         DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &cone);
    5195:       }
    5196:     }
    5197:     VecRestoreArray(coordinates, &coords);
    5198:     VecRestoreArray(coordinatesNew, &coordsNew);
    5199:     DMSetCoordinatesLocal(rdm, coordinatesNew);
    5200:     VecDestroy(&coordinatesNew);
    5201:     PetscSectionDestroy(&coordSectionNew);
    5202:     break;
    5203:   default:
    5204:     SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown cell refiner %d", refiner);
    5205:   }
    5206:   return(0);
    5207: }

    5211: PetscErrorCode DMPlexCreateProcessSF(DM dm, PetscSF sfPoint, IS *processRanks, PetscSF *sfProcess)
    5212: {
    5213:   PetscInt           numRoots, numLeaves, l;
    5214:   const PetscInt    *localPoints;
    5215:   const PetscSFNode *remotePoints;
    5216:   PetscInt          *localPointsNew;
    5217:   PetscSFNode       *remotePointsNew;
    5218:   PetscInt          *ranks, *ranksNew;
    5219:   PetscErrorCode     ierr;

    5222:   PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);
    5223:   PetscMalloc(numLeaves * sizeof(PetscInt), &ranks);
    5224:   for (l = 0; l < numLeaves; ++l) {
    5225:     ranks[l] = remotePoints[l].rank;
    5226:   }
    5227:   PetscSortRemoveDupsInt(&numLeaves, ranks);
    5228:   PetscMalloc(numLeaves * sizeof(PetscInt),    &ranksNew);
    5229:   PetscMalloc(numLeaves * sizeof(PetscInt),    &localPointsNew);
    5230:   PetscMalloc(numLeaves * sizeof(PetscSFNode), &remotePointsNew);
    5231:   for (l = 0; l < numLeaves; ++l) {
    5232:     ranksNew[l]              = ranks[l];
    5233:     localPointsNew[l]        = l;
    5234:     remotePointsNew[l].index = 0;
    5235:     remotePointsNew[l].rank  = ranksNew[l];
    5236:   }
    5237:   PetscFree(ranks);
    5238:   ISCreateGeneral(PetscObjectComm((PetscObject)dm), numLeaves, ranksNew, PETSC_OWN_POINTER, processRanks);
    5239:   PetscSFCreate(PetscObjectComm((PetscObject)dm), sfProcess);
    5240:   PetscSFSetFromOptions(*sfProcess);
    5241:   PetscSFSetGraph(*sfProcess, 1, numLeaves, localPointsNew, PETSC_OWN_POINTER, remotePointsNew, PETSC_OWN_POINTER);
    5242:   return(0);
    5243: }

    5247: PetscErrorCode CellRefinerCreateSF(CellRefiner refiner, DM dm, PetscInt depthSize[], DM rdm)
    5248: {
    5249:   PetscSF            sf, sfNew, sfProcess;
    5250:   IS                 processRanks;
    5251:   MPI_Datatype       depthType;
    5252:   PetscInt           numRoots, numLeaves, numLeavesNew = 0, l, m;
    5253:   const PetscInt    *localPoints, *neighbors;
    5254:   const PetscSFNode *remotePoints;
    5255:   PetscInt          *localPointsNew;
    5256:   PetscSFNode       *remotePointsNew;
    5257:   PetscInt          *depthSizeOld, *rdepthSize, *rdepthSizeOld, *rdepthMaxOld, *rvStart, *rvStartNew, *reStart, *reStartNew, *rfStart, *rfStartNew, *rcStart, *rcStartNew;
    5258:   PetscInt           depth, numNeighbors, pStartNew, pEndNew, cStart, cStartNew, cEnd, cMax, vStart, vStartNew, vEnd, vMax, fStart, fStartNew, fEnd, fMax, eStart, eStartNew, eEnd, eMax, r, n;
    5259:   PetscErrorCode     ierr;

    5262:   DMPlexGetChart(rdm, &pStartNew, &pEndNew);
    5263:   DMPlexGetDepth(dm, &depth);
    5264:   DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);
    5265:   DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);
    5266:   DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);
    5267:   DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);
    5268:   DMPlexGetHybridBounds(dm, &cMax, &fMax, &eMax, &vMax);
    5269:   GetDepthStart_Private(depth, depthSize, &cStartNew, &fStartNew, &eStartNew, &vStartNew);
    5270:   switch (refiner) {
    5271:   case 3:
    5272:     if (cMax < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No cell maximum specified in hybrid mesh");
    5273:     cMax = PetscMin(cEnd, cMax);
    5274:     if (fMax < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No face maximum specified in hybrid mesh");
    5275:     fMax = PetscMin(fEnd, fMax);
    5276:   }
    5277:   DMGetPointSF(dm, &sf);
    5278:   DMGetPointSF(rdm, &sfNew);
    5279:   /* Caculate size of new SF */
    5280:   PetscSFGetGraph(sf, &numRoots, &numLeaves, &localPoints, &remotePoints);
    5281:   if (numRoots < 0) return(0);
    5282:   for (l = 0; l < numLeaves; ++l) {
    5283:     const PetscInt p = localPoints[l];

    5285:     switch (refiner) {
    5286:     case 1:
    5287:       /* Simplicial 2D */
    5288:       if ((p >= vStart) && (p < vEnd)) {
    5289:         /* Old vertices stay the same */
    5290:         ++numLeavesNew;
    5291:       } else if ((p >= fStart) && (p < fEnd)) {
    5292:         /* Old faces add new faces and vertex */
    5293:         numLeavesNew += 1 + 2;
    5294:       } else if ((p >= cStart) && (p < cEnd)) {
    5295:         /* Old cells add new cells and interior faces */
    5296:         numLeavesNew += 4 + 3;
    5297:       }
    5298:       break;
    5299:     case 2:
    5300:       /* Hex 2D */
    5301:       if ((p >= vStart) && (p < vEnd)) {
    5302:         /* Old vertices stay the same */
    5303:         ++numLeavesNew;
    5304:       } else if ((p >= fStart) && (p < fEnd)) {
    5305:         /* Old faces add new faces and vertex */
    5306:         numLeavesNew += 1 + 2;
    5307:       } else if ((p >= cStart) && (p < cEnd)) {
    5308:         /* Old cells add new cells and interior faces */
    5309:         numLeavesNew += 4 + 4;
    5310:       }
    5311:       break;
    5312:     default:
    5313:       SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown cell refiner %d", refiner);
    5314:     }
    5315:   }
    5316:   /* Communicate depthSizes for each remote rank */
    5317:   DMPlexCreateProcessSF(dm, sf, &processRanks, &sfProcess);
    5318:   ISGetLocalSize(processRanks, &numNeighbors);
    5319:   PetscMalloc5((depth+1)*numNeighbors,PetscInt,&rdepthSize,numNeighbors,PetscInt,&rvStartNew,numNeighbors,PetscInt,&reStartNew,numNeighbors,PetscInt,&rfStartNew,numNeighbors,PetscInt,&rcStartNew);
    5320:   PetscMalloc7(depth+1,PetscInt,&depthSizeOld,(depth+1)*numNeighbors,PetscInt,&rdepthSizeOld,(depth+1)*numNeighbors,PetscInt,&rdepthMaxOld,numNeighbors,PetscInt,&rvStart,numNeighbors,PetscInt,&reStart,numNeighbors,PetscInt,&rfStart,numNeighbors,PetscInt,&rcStart);
    5321:   MPI_Type_contiguous(depth+1, MPIU_INT, &depthType);
    5322:   MPI_Type_commit(&depthType);
    5323:   PetscSFBcastBegin(sfProcess, depthType, depthSize, rdepthSize);
    5324:   PetscSFBcastEnd(sfProcess, depthType, depthSize, rdepthSize);
    5325:   for (n = 0; n < numNeighbors; ++n) {
    5326:     GetDepthStart_Private(depth, &rdepthSize[n*(depth+1)], &rcStartNew[n], &rfStartNew[n], &reStartNew[n], &rvStartNew[n]);
    5327:   }
    5328:   depthSizeOld[depth]   = cMax;
    5329:   depthSizeOld[0]       = vMax;
    5330:   depthSizeOld[depth-1] = fMax;
    5331:   depthSizeOld[1]       = eMax;

    5333:   PetscSFBcastBegin(sfProcess, depthType, depthSizeOld, rdepthMaxOld);
    5334:   PetscSFBcastEnd(sfProcess, depthType, depthSizeOld, rdepthMaxOld);

    5336:   depthSizeOld[depth]   = cEnd - cStart;
    5337:   depthSizeOld[0]       = vEnd - vStart;
    5338:   depthSizeOld[depth-1] = fEnd - fStart;
    5339:   depthSizeOld[1]       = eEnd - eStart;

    5341:   PetscSFBcastBegin(sfProcess, depthType, depthSizeOld, rdepthSizeOld);
    5342:   PetscSFBcastEnd(sfProcess, depthType, depthSizeOld, rdepthSizeOld);
    5343:   for (n = 0; n < numNeighbors; ++n) {
    5344:     GetDepthStart_Private(depth, &rdepthSizeOld[n*(depth+1)], &rcStart[n], &rfStart[n], &reStart[n], &rvStart[n]);
    5345:   }
    5346:   MPI_Type_free(&depthType);
    5347:   PetscSFDestroy(&sfProcess);
    5348:   /* Calculate new point SF */
    5349:   PetscMalloc(numLeavesNew * sizeof(PetscInt),    &localPointsNew);
    5350:   PetscMalloc(numLeavesNew * sizeof(PetscSFNode), &remotePointsNew);
    5351:   ISGetIndices(processRanks, &neighbors);
    5352:   for (l = 0, m = 0; l < numLeaves; ++l) {
    5353:     PetscInt    p     = localPoints[l];
    5354:     PetscInt    rp    = remotePoints[l].index, n;
    5355:     PetscMPIInt rrank = remotePoints[l].rank;

    5357:     PetscFindInt(rrank, numNeighbors, neighbors, &n);
    5358:     if (n < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not locate remote rank %d", rrank);
    5359:     switch (refiner) {
    5360:     case 1:
    5361:       /* Simplicial 2D */
    5362:       if ((p >= vStart) && (p < vEnd)) {
    5363:         /* Old vertices stay the same */
    5364:         localPointsNew[m]        = vStartNew     + (p  - vStart);
    5365:         remotePointsNew[m].index = rvStartNew[n] + (rp - rvStart[n]);
    5366:         remotePointsNew[m].rank  = rrank;
    5367:         ++m;
    5368:       } else if ((p >= fStart) && (p < fEnd)) {
    5369:         /* Old faces add new faces and vertex */
    5370:         localPointsNew[m]        = vStartNew     + (vEnd - vStart)              + (p  - fStart);
    5371:         remotePointsNew[m].index = rvStartNew[n] + rdepthSizeOld[n*(depth+1)+0] + (rp - rfStart[n]);
    5372:         remotePointsNew[m].rank  = rrank;
    5373:         ++m;
    5374:         for (r = 0; r < 2; ++r, ++m) {
    5375:           localPointsNew[m]        = fStartNew     + (p  - fStart)*2     + r;
    5376:           remotePointsNew[m].index = rfStartNew[n] + (rp - rfStart[n])*2 + r;
    5377:           remotePointsNew[m].rank  = rrank;
    5378:         }
    5379:       } else if ((p >= cStart) && (p < cEnd)) {
    5380:         /* Old cells add new cells and interior faces */
    5381:         for (r = 0; r < 4; ++r, ++m) {
    5382:           localPointsNew[m]        = cStartNew     + (p  - cStart)*4     + r;
    5383:           remotePointsNew[m].index = rcStartNew[n] + (rp - rcStart[n])*4 + r;
    5384:           remotePointsNew[m].rank  = rrank;
    5385:         }
    5386:         for (r = 0; r < 3; ++r, ++m) {
    5387:           localPointsNew[m]        = fStartNew     + (fEnd - fStart)*2                    + (p  - cStart)*3     + r;
    5388:           remotePointsNew[m].index = rfStartNew[n] + rdepthSizeOld[n*(depth+1)+depth-1]*2 + (rp - rcStart[n])*3 + r;
    5389:           remotePointsNew[m].rank  = rrank;
    5390:         }
    5391:       }
    5392:       break;
    5393:     case 2:
    5394:       /* Hex 2D */
    5395:       if ((p >= vStart) && (p < vEnd)) {
    5396:         /* Old vertices stay the same */
    5397:         localPointsNew[m]        = vStartNew     + (p  - vStart);
    5398:         remotePointsNew[m].index = rvStartNew[n] + (rp - rvStart[n]);
    5399:         remotePointsNew[m].rank  = rrank;
    5400:         ++m;
    5401:       } else if ((p >= fStart) && (p < fEnd)) {
    5402:         /* Old faces add new faces and vertex */
    5403:         localPointsNew[m]        = vStartNew     + (vEnd - vStart)              + (p  - fStart);
    5404:         remotePointsNew[m].index = rvStartNew[n] + rdepthSizeOld[n*(depth+1)+0] + (rp - rfStart[n]);
    5405:         remotePointsNew[m].rank  = rrank;
    5406:         ++m;
    5407:         for (r = 0; r < 2; ++r, ++m) {
    5408:           localPointsNew[m]        = fStartNew     + (p  - fStart)*2     + r;
    5409:           remotePointsNew[m].index = rfStartNew[n] + (rp - rfStart[n])*2 + r;
    5410:           remotePointsNew[m].rank  = rrank;
    5411:         }
    5412:       } else if ((p >= cStart) && (p < cEnd)) {
    5413:         /* Old cells add new cells and interior faces */
    5414:         for (r = 0; r < 4; ++r, ++m) {
    5415:           localPointsNew[m]        = cStartNew     + (p  - cStart)*4     + r;
    5416:           remotePointsNew[m].index = rcStartNew[n] + (rp - rcStart[n])*4 + r;
    5417:           remotePointsNew[m].rank  = rrank;
    5418:         }
    5419:         for (r = 0; r < 4; ++r, ++m) {
    5420:           localPointsNew[m]        = fStartNew     + (fEnd - fStart)*2                    + (p  - cStart)*4     + r;
    5421:           remotePointsNew[m].index = rfStartNew[n] + rdepthSizeOld[n*(depth+1)+depth-1]*2 + (rp - rcStart[n])*4 + r;
    5422:           remotePointsNew[m].rank  = rrank;
    5423:         }
    5424:       }
    5425:       break;
    5426:     case 3:
    5427:       /* Hybrid simplicial 2D */
    5428:       if ((p >= vStart) && (p < vEnd)) {
    5429:         /* Old vertices stay the same */
    5430:         localPointsNew[m]        = vStartNew     + (p  - vStart);
    5431:         remotePointsNew[m].index = rvStartNew[n] + (rp - rvStart[n]);
    5432:         remotePointsNew[m].rank  = rrank;
    5433:         ++m;
    5434:       } else if ((p >= fStart) && (p < fMax)) {
    5435:         /* Old interior faces add new faces and vertex */
    5436:         localPointsNew[m]        = vStartNew     + (vEnd - vStart)              + (p  - fStart);
    5437:         remotePointsNew[m].index = rvStartNew[n] + rdepthSizeOld[n*(depth+1)+0] + (rp - rfStart[n]);
    5438:         remotePointsNew[m].rank  = rrank;
    5439:         ++m;
    5440:         for (r = 0; r < 2; ++r, ++m) {
    5441:           localPointsNew[m]        = fStartNew     + (p  - fStart)*2     + r;
    5442:           remotePointsNew[m].index = rfStartNew[n] + (rp - rfStart[n])*2 + r;
    5443:           remotePointsNew[m].rank  = rrank;
    5444:         }
    5445:       } else if ((p >= fMax) && (p < fEnd)) {
    5446:         /* Old hybrid faces stay the same */
    5447:         localPointsNew[m]        = fStartNew     + (fMax                              - fStart)*2     + (p  - fMax);
    5448:         remotePointsNew[m].index = rfStartNew[n] + (rdepthMaxOld[n*(depth+1)+depth-1] - rfStart[n])*2 + (rp - rdepthMaxOld[n*(depth+1)+depth-1]);
    5449:         remotePointsNew[m].rank  = rrank;
    5450:         ++m;
    5451:       } else if ((p >= cStart) && (p < cMax)) {
    5452:         /* Old interior cells add new cells and interior faces */
    5453:         for (r = 0; r < 4; ++r, ++m) {
    5454:           localPointsNew[m]        = cStartNew     + (p  - cStart)*4     + r;
    5455:           remotePointsNew[m].index = rcStartNew[n] + (rp - rcStart[n])*4 + r;
    5456:           remotePointsNew[m].rank  = rrank;
    5457:         }
    5458:         for (r = 0; r < 3; ++r, ++m) {
    5459:           localPointsNew[m]        = fStartNew     + (fMax                              - fStart)*2     + (p  - cStart)*3     + r;
    5460:           remotePointsNew[m].index = rfStartNew[n] + (rdepthMaxOld[n*(depth+1)+depth-1] - rfStart[n])*2 + (rp - rcStart[n])*3 + r;
    5461:           remotePointsNew[m].rank  = rrank;
    5462:         }
    5463:       } else if ((p >= cStart) && (p < cMax)) {
    5464:         /* Old hybrid cells add new cells and hybrid face */
    5465:         for (r = 0; r < 2; ++r, ++m) {
    5466:           localPointsNew[m]        = cStartNew     + (p  - cStart)*4     + r;
    5467:           remotePointsNew[m].index = rcStartNew[n] + (rp - rcStart[n])*4 + r;
    5468:           remotePointsNew[m].rank  = rrank;
    5469:         }
    5470:         localPointsNew[m]        = fStartNew     + (fMax                              - fStart)*2     + (cMax                            - cStart)*3     + (p  - cMax);
    5471:         remotePointsNew[m].index = rfStartNew[n] + (rdepthMaxOld[n*(depth+1)+depth-1] - rfStart[n])*2 + (rdepthMaxOld[n*(depth+1)+depth] - rcStart[n])*3 + (rp - rdepthMaxOld[n*(depth+1)+depth]);
    5472:         remotePointsNew[m].rank  = rrank;
    5473:         ++m;
    5474:       }
    5475:       break;
    5476:     default:
    5477:       SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown cell refiner %d", refiner);
    5478:     }
    5479:   }
    5480:   ISRestoreIndices(processRanks, &neighbors);
    5481:   ISDestroy(&processRanks);
    5482:   PetscSFSetGraph(sfNew, pEndNew-pStartNew, numLeavesNew, localPointsNew, PETSC_OWN_POINTER, remotePointsNew, PETSC_OWN_POINTER);
    5483:   PetscFree5(rdepthSize,rvStartNew,reStartNew,rfStartNew,rcStartNew);
    5484:   PetscFree6(depthSizeOld,rdepthSizeOld,rvStart,reStart,rfStart,rcStart);
    5485:   return(0);
    5486: }

    5490: PetscErrorCode CellRefinerCreateLabels(CellRefiner refiner, DM dm, PetscInt depthSize[], DM rdm)
    5491: {
    5492:   PetscInt       numLabels, l;
    5493:   PetscInt       newp, cStart, cStartNew, cEnd, cMax, vStart, vStartNew, vEnd, vMax, fStart, fStartNew, fEnd, fMax, eStart, eEnd, eMax, r;

    5497:   DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);
    5498:   DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);
    5499:   DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);
    5500:   DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);

    5502:   cStartNew = 0;
    5503:   vStartNew = depthSize[2];
    5504:   fStartNew = depthSize[2] + depthSize[0];

    5506:   DMPlexGetNumLabels(dm, &numLabels);
    5507:   DMPlexGetHybridBounds(dm, &cMax, &fMax, &eMax, &vMax);
    5508:   switch (refiner) {
    5509:   case 3:
    5510:     if (cMax < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No cell maximum specified in hybrid mesh");
    5511:     cMax = PetscMin(cEnd, cMax);
    5512:     if (fMax < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No face maximum specified in hybrid mesh");
    5513:     fMax = PetscMin(fEnd, fMax);
    5514:   }
    5515:   for (l = 0; l < numLabels; ++l) {
    5516:     DMLabel         label, labelNew;
    5517:     const char     *lname;
    5518:     PetscBool       isDepth;
    5519:     IS              valueIS;
    5520:     const PetscInt *values;
    5521:     PetscInt        numValues, val;

    5523:     DMPlexGetLabelName(dm, l, &lname);
    5524:     PetscStrcmp(lname, "depth", &isDepth);
    5525:     if (isDepth) continue;
    5526:     DMPlexCreateLabel(rdm, lname);
    5527:     DMPlexGetLabel(dm, lname, &label);
    5528:     DMPlexGetLabel(rdm, lname, &labelNew);
    5529:     DMLabelGetValueIS(label, &valueIS);
    5530:     ISGetLocalSize(valueIS, &numValues);
    5531:     ISGetIndices(valueIS, &values);
    5532:     for (val = 0; val < numValues; ++val) {
    5533:       IS              pointIS;
    5534:       const PetscInt *points;
    5535:       PetscInt        numPoints, n;

    5537:       DMLabelGetStratumIS(label, values[val], &pointIS);
    5538:       ISGetLocalSize(pointIS, &numPoints);
    5539:       ISGetIndices(pointIS, &points);
    5540:       for (n = 0; n < numPoints; ++n) {
    5541:         const PetscInt p = points[n];
    5542:         switch (refiner) {
    5543:         case 1:
    5544:           /* Simplicial 2D */
    5545:           if ((p >= vStart) && (p < vEnd)) {
    5546:             /* Old vertices stay the same */
    5547:             newp = vStartNew + (p - vStart);
    5548:             DMLabelSetValue(labelNew, newp, values[val]);
    5549:           } else if ((p >= fStart) && (p < fEnd)) {
    5550:             /* Old faces add new faces and vertex */
    5551:             newp = vStartNew + (vEnd - vStart) + (p - fStart);
    5552:             DMLabelSetValue(labelNew, newp, values[val]);
    5553:             for (r = 0; r < 2; ++r) {
    5554:               newp = fStartNew + (p - fStart)*2 + r;
    5555:               DMLabelSetValue(labelNew, newp, values[val]);
    5556:             }
    5557:           } else if ((p >= cStart) && (p < cEnd)) {
    5558:             /* Old cells add new cells and interior faces */
    5559:             for (r = 0; r < 4; ++r) {
    5560:               newp = cStartNew + (p - cStart)*4 + r;
    5561:               DMLabelSetValue(labelNew, newp, values[val]);
    5562:             }
    5563:             for (r = 0; r < 3; ++r) {
    5564:               newp = fStartNew + (fEnd - fStart)*2 + (p - cStart)*3 + r;
    5565:               DMLabelSetValue(labelNew, newp, values[val]);
    5566:             }
    5567:           }
    5568:           break;
    5569:         case 2:
    5570:           /* Hex 2D */
    5571:           if ((p >= vStart) && (p < vEnd)) {
    5572:             /* Old vertices stay the same */
    5573:             newp = vStartNew + (p - vStart);
    5574:             DMLabelSetValue(labelNew, newp, values[val]);
    5575:           } else if ((p >= fStart) && (p < fEnd)) {
    5576:             /* Old faces add new faces and vertex */
    5577:             newp = vStartNew + (vEnd - vStart) + (p - fStart);
    5578:             DMLabelSetValue(labelNew, newp, values[val]);
    5579:             for (r = 0; r < 2; ++r) {
    5580:               newp = fStartNew + (p - fStart)*2 + r;
    5581:               DMLabelSetValue(labelNew, newp, values[val]);
    5582:             }
    5583:           } else if ((p >= cStart) && (p < cEnd)) {
    5584:             /* Old cells add new cells and interior faces and vertex */
    5585:             for (r = 0; r < 4; ++r) {
    5586:               newp = cStartNew + (p - cStart)*4 + r;
    5587:               DMLabelSetValue(labelNew, newp, values[val]);
    5588:             }
    5589:             for (r = 0; r < 4; ++r) {
    5590:               newp = fStartNew + (fEnd - fStart)*2 + (p - cStart)*4 + r;
    5591:               DMLabelSetValue(labelNew, newp, values[val]);
    5592:             }
    5593:             newp = vStartNew + (vEnd - vStart) + (fEnd - fStart) + (p - cStart);
    5594:             DMLabelSetValue(labelNew, newp, values[val]);
    5595:           }
    5596:           break;
    5597:         case 3:
    5598:           /* Hybrid simplicial 2D */
    5599:           if ((p >= vStart) && (p < vEnd)) {
    5600:             /* Old vertices stay the same */
    5601:             newp = vStartNew + (p - vStart);
    5602:             DMLabelSetValue(labelNew, newp, values[val]);
    5603:           } else if ((p >= fStart) && (p < fMax)) {
    5604:             /* Old interior faces add new faces and vertex */
    5605:             newp = vStartNew + (vEnd - vStart) + (p - fStart);
    5606:             DMLabelSetValue(labelNew, newp, values[val]);
    5607:             for (r = 0; r < 2; ++r) {
    5608:               newp = fStartNew + (p - fStart)*2 + r;
    5609:               DMLabelSetValue(labelNew, newp, values[val]);
    5610:             }
    5611:           } else if ((p >= fMax) && (p < fEnd)) {
    5612:             /* Old hybrid faces stay the same */
    5613:             newp = fStartNew + (fMax - fStart)*2 + (p - fMax);
    5614:             DMLabelSetValue(labelNew, newp, values[val]);
    5615:           } else if ((p >= cStart) && (p < cMax)) {
    5616:             /* Old interior cells add new cells and interior faces */
    5617:             for (r = 0; r < 4; ++r) {
    5618:               newp = cStartNew + (p - cStart)*4 + r;
    5619:               DMLabelSetValue(labelNew, newp, values[val]);
    5620:             }
    5621:             for (r = 0; r < 3; ++r) {
    5622:               newp = fStartNew + (fEnd - fStart)*2 + (p - cStart)*3 + r;
    5623:               DMLabelSetValue(labelNew, newp, values[val]);
    5624:             }
    5625:           } else if ((p >= cMax) && (p < cEnd)) {
    5626:             /* Old hybrid cells add new cells and hybrid face */
    5627:             for (r = 0; r < 2; ++r) {
    5628:               newp = cStartNew + (cMax - cStart)*4 + (p - cMax)*2 + r;
    5629:               DMLabelSetValue(labelNew, newp, values[val]);
    5630:             }
    5631:             newp = fStartNew + (fMax - fStart)*2 + (cMax - cStart)*3 + (p - cMax);
    5632:             DMLabelSetValue(labelNew, newp, values[val]);
    5633:           }
    5634:           break;
    5635:         default:
    5636:           SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown cell refiner %d", refiner);
    5637:         }
    5638:       }
    5639:       ISRestoreIndices(pointIS, &points);
    5640:       ISDestroy(&pointIS);
    5641:     }
    5642:     ISRestoreIndices(valueIS, &values);
    5643:     ISDestroy(&valueIS);
    5644:     if (0) {
    5645:       PetscViewerASCIISynchronizedAllow(PETSC_VIEWER_STDOUT_WORLD, PETSC_TRUE);
    5646:       DMLabelView(labelNew, PETSC_VIEWER_STDOUT_WORLD);
    5647:       PetscViewerFlush(PETSC_VIEWER_STDOUT_WORLD);
    5648:     }
    5649:   }
    5650:   return(0);
    5651: }

    5655: /* This will only work for interpolated meshes */
    5656: PetscErrorCode DMPlexRefine_Uniform(DM dm, CellRefiner cellRefiner, DM *dmRefined)
    5657: {
    5658:   DM             rdm;
    5659:   PetscInt      *depthSize;
    5660:   PetscInt       dim, depth = 0, d, pStart = 0, pEnd = 0;

    5664:   DMCreate(PetscObjectComm((PetscObject)dm), &rdm);
    5665:   DMSetType(rdm, DMPLEX);
    5666:   DMPlexGetDimension(dm, &dim);
    5667:   DMPlexSetDimension(rdm, dim);
    5668:   /* Calculate number of new points of each depth */
    5669:   DMPlexGetDepth(dm, &depth);
    5670:   PetscMalloc((depth+1) * sizeof(PetscInt), &depthSize);
    5671:   PetscMemzero(depthSize, (depth+1) * sizeof(PetscInt));
    5672:   CellRefinerGetSizes(cellRefiner, dm, depthSize);
    5673:   /* Step 1: Set chart */
    5674:   for (d = 0; d <= depth; ++d) pEnd += depthSize[d];
    5675:   DMPlexSetChart(rdm, pStart, pEnd);
    5676:   /* Step 2: Set cone/support sizes */
    5677:   CellRefinerSetConeSizes(cellRefiner, dm, depthSize, rdm);
    5678:   /* Step 3: Setup refined DM */
    5679:   DMSetUp(rdm);
    5680:   /* Step 4: Set cones and supports */
    5681:   CellRefinerSetCones(cellRefiner, dm, depthSize, rdm);
    5682:   /* Step 5: Stratify */
    5683:   DMPlexStratify(rdm);
    5684:   /* Step 6: Set coordinates for vertices */
    5685:   CellRefinerSetCoordinates(cellRefiner, dm, depthSize, rdm);
    5686:   /* Step 7: Create pointSF */
    5687:   CellRefinerCreateSF(cellRefiner, dm, depthSize, rdm);
    5688:   /* Step 8: Create labels */
    5689:   CellRefinerCreateLabels(cellRefiner, dm, depthSize, rdm);
    5690:   PetscFree(depthSize);

    5692:   *dmRefined = rdm;
    5693:   return(0);
    5694: }

    5698: PetscErrorCode DMPlexSetRefinementUniform(DM dm, PetscBool refinementUniform)
    5699: {
    5700:   DM_Plex *mesh = (DM_Plex*) dm->data;

    5704:   mesh->refinementUniform = refinementUniform;
    5705:   return(0);
    5706: }

    5710: PetscErrorCode DMPlexGetRefinementUniform(DM dm, PetscBool *refinementUniform)
    5711: {
    5712:   DM_Plex *mesh = (DM_Plex*) dm->data;

    5717:   *refinementUniform = mesh->refinementUniform;
    5718:   return(0);
    5719: }

    5723: PetscErrorCode DMPlexSetRefinementLimit(DM dm, PetscReal refinementLimit)
    5724: {
    5725:   DM_Plex *mesh = (DM_Plex*) dm->data;

    5729:   mesh->refinementLimit = refinementLimit;
    5730:   return(0);
    5731: }

    5735: PetscErrorCode DMPlexGetRefinementLimit(DM dm, PetscReal *refinementLimit)
    5736: {
    5737:   DM_Plex *mesh = (DM_Plex*) dm->data;

    5742:   /* if (mesh->refinementLimit < 0) = getMaxVolume()/2.0; */
    5743:   *refinementLimit = mesh->refinementLimit;
    5744:   return(0);
    5745: }

    5749: PetscErrorCode DMPlexGetCellRefiner_Private(DM dm, CellRefiner *cellRefiner)
    5750: {
    5751:   PetscInt       dim, cStart, coneSize, cMax;

    5755:   DMPlexGetDimension(dm, &dim);
    5756:   DMPlexGetHeightStratum(dm, 0, &cStart, NULL);
    5757:   DMPlexGetConeSize(dm, cStart, &coneSize);
    5758:   DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);
    5759:   switch (dim) {
    5760:   case 2:
    5761:     switch (coneSize) {
    5762:     case 3:
    5763:       if (cMax >= 0) *cellRefiner = 3; /* Hybrid */
    5764:       else *cellRefiner = 1; /* Triangular */
    5765:       break;
    5766:     case 4:
    5767:       if (cMax >= 0) *cellRefiner = 4; /* Hybrid */
    5768:       else *cellRefiner = 2; /* Quadrilateral */
    5769:       break;
    5770:     default:
    5771:       SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown coneSize %d in dimension %d for cell refiner", coneSize, dim);
    5772:     }
    5773:     break;
    5774:   default:
    5775:     SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown dimension %d for cell refiner", dim);
    5776:   }
    5777:   return(0);
    5778: }

    5782: PetscErrorCode DMRefine_Plex(DM dm, MPI_Comm comm, DM *dmRefined)
    5783: {
    5784:   PetscReal      refinementLimit;
    5785:   PetscInt       dim, cStart, cEnd;
    5786:   char           genname[1024], *name = NULL;
    5787:   PetscBool      isUniform, isTriangle = PETSC_FALSE, isTetgen = PETSC_FALSE, isCTetgen = PETSC_FALSE, flg;

    5791:   DMPlexGetRefinementUniform(dm, &isUniform);
    5792:   if (isUniform) {
    5793:     CellRefiner cellRefiner;

    5795:     DMPlexGetCellRefiner_Private(dm, &cellRefiner);
    5796:     DMPlexRefine_Uniform(dm, cellRefiner, dmRefined);
    5797:     return(0);
    5798:   }
    5799:   DMPlexGetRefinementLimit(dm, &refinementLimit);
    5800:   if (refinementLimit == 0.0) return(0);
    5801:   DMPlexGetDimension(dm, &dim);
    5802:   DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);
    5803:   PetscOptionsGetString(((PetscObject) dm)->prefix, "-dm_plex_generator", genname, 1024, &flg);
    5804:   if (flg) name = genname;
    5805:   if (name) {
    5806:     PetscStrcmp(name, "triangle", &isTriangle);
    5807:     PetscStrcmp(name, "tetgen",   &isTetgen);
    5808:     PetscStrcmp(name, "ctetgen",  &isCTetgen);
    5809:   }
    5810:   switch (dim) {
    5811:   case 2:
    5812:     if (!name || isTriangle) {
    5813: #if defined(PETSC_HAVE_TRIANGLE)
    5814:       double  *maxVolumes;
    5815:       PetscInt c;

    5817:       PetscMalloc((cEnd - cStart) * sizeof(double), &maxVolumes);
    5818:       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
    5819:       DMPlexRefine_Triangle(dm, maxVolumes, dmRefined);
    5820: #else
    5821:       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Mesh refinement needs external package support.\nPlease reconfigure with --download-triangle.");
    5822: #endif
    5823:     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unknown 2D mesh generation package %s", name);
    5824:     break;
    5825:   case 3:
    5826:     if (!name || isCTetgen) {
    5827: #if defined(PETSC_HAVE_CTETGEN)
    5828:       PetscReal *maxVolumes;
    5829:       PetscInt   c;

    5831:       PetscMalloc((cEnd - cStart) * sizeof(PetscReal), &maxVolumes);
    5832:       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
    5833:       DMPlexRefine_CTetgen(dm, maxVolumes, dmRefined);
    5834: #else
    5835:       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "CTetgen needs external package support.\nPlease reconfigure with --download-ctetgen.");
    5836: #endif
    5837:     } else if (isTetgen) {
    5838: #if defined(PETSC_HAVE_TETGEN)
    5839:       double  *maxVolumes;
    5840:       PetscInt c;

    5842:       PetscMalloc((cEnd - cStart) * sizeof(double), &maxVolumes);
    5843:       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
    5844:       DMPlexRefine_Tetgen(dm, maxVolumes, dmRefined);
    5845: #else
    5846:       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Tetgen needs external package support.\nPlease reconfigure with --with-c-language=cxx --download-tetgen.");
    5847: #endif
    5848:     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unknown 3D mesh generation package %s", name);
    5849:     break;
    5850:   default:
    5851:     SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Mesh refinement in dimension %d is not supported.", dim);
    5852:   }
    5853:   return(0);
    5854: }

    5858: /*@
    5859:   DMPlexGetDepth - get the number of strata

    5861:   Not Collective

    5863:   Input Parameters:
    5864: . dm           - The DMPlex object

    5866:   Output Parameters:
    5867: . depth - number of strata

    5869:   Level: developer

    5871:   Notes:
    5872:   DMPlexGetHeightStratum(dm,0,..) should return the same points as DMPlexGetDepthStratum(dm,depth,..).

    5874: .keywords: mesh, points
    5875: .seealso: DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
    5876: @*/
    5877: PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
    5878: {
    5879:   PetscInt       d;

    5885:   DMPlexGetLabelSize(dm, "depth", &d);
    5886:   *depth = d-1;
    5887:   return(0);
    5888: }

    5892: /*@
    5893:   DMPlexGetDepthStratum - Get the bounds [start, end) for all points at a certain depth.

    5895:   Not Collective

    5897:   Input Parameters:
    5898: + dm           - The DMPlex object
    5899: - stratumValue - The requested depth

    5901:   Output Parameters:
    5902: + start - The first point at this depth
    5903: - end   - One beyond the last point at this depth

    5905:   Level: developer

    5907: .keywords: mesh, points
    5908: .seealso: DMPlexGetHeightStratum(), DMPlexGetDepth()
    5909: @*/
    5910: PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
    5911: {
    5912:   DM_Plex       *mesh = (DM_Plex*) dm->data;
    5913:   DMLabel        next  = mesh->labels;
    5914:   PetscBool      flg   = PETSC_FALSE;
    5915:   PetscInt       depth;

    5920:   if (stratumValue < 0) {
    5921:     DMPlexGetChart(dm, start, end);
    5922:     return(0);
    5923:   } else {
    5924:     PetscInt pStart, pEnd;

    5926:     if (start) *start = 0;
    5927:     if (end)   *end   = 0;
    5928:     DMPlexGetChart(dm, &pStart, &pEnd);
    5929:     if (pStart == pEnd) return(0);
    5930:   }
    5931:   DMPlexHasLabel(dm, "depth", &flg);
    5932:   if (!flg) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
    5933:   /* We should have a generic GetLabel() and a Label class */
    5934:   while (next) {
    5935:     PetscStrcmp("depth", next->name, &flg);
    5936:     if (flg) break;
    5937:     next = next->next;
    5938:   }
    5939:   /* Strata are sorted and contiguous -- In addition, depth/height is either full or 1-level */
    5940:   depth = stratumValue;
    5941:   if ((depth < 0) || (depth >= next->numStrata)) {
    5942:     if (start) *start = 0;
    5943:     if (end)   *end   = 0;
    5944:   } else {
    5945:     if (start) *start = next->points[next->stratumOffsets[depth]];
    5946:     if (end)   *end   = next->points[next->stratumOffsets[depth]+next->stratumSizes[depth]-1]+1;
    5947:   }
    5948:   return(0);
    5949: }

    5953: /*@
    5954:   DMPlexGetHeightStratum - Get the bounds [start, end) for all points at a certain height.

    5956:   Not Collective

    5958:   Input Parameters:
    5959: + dm           - The DMPlex object
    5960: - stratumValue - The requested height

    5962:   Output Parameters:
    5963: + start - The first point at this height
    5964: - end   - One beyond the last point at this height

    5966:   Level: developer

    5968: .keywords: mesh, points
    5969: .seealso: DMPlexGetDepthStratum(), DMPlexGetDepth()
    5970: @*/
    5971: PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
    5972: {
    5973:   DM_Plex       *mesh = (DM_Plex*) dm->data;
    5974:   DMLabel        next  = mesh->labels;
    5975:   PetscBool      flg   = PETSC_FALSE;
    5976:   PetscInt       depth;

    5981:   if (stratumValue < 0) {
    5982:     DMPlexGetChart(dm, start, end);
    5983:   } else {
    5984:     PetscInt pStart, pEnd;

    5986:     if (start) *start = 0;
    5987:     if (end)   *end   = 0;
    5988:     DMPlexGetChart(dm, &pStart, &pEnd);
    5989:     if (pStart == pEnd) return(0);
    5990:   }
    5991:   DMPlexHasLabel(dm, "depth", &flg);
    5992:   if (!flg) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
    5993:   /* We should have a generic GetLabel() and a Label class */
    5994:   while (next) {
    5995:     PetscStrcmp("depth", next->name, &flg);
    5996:     if (flg) break;
    5997:     next = next->next;
    5998:   }
    5999:   /* Strata are sorted and contiguous -- In addition, depth/height is either full or 1-level */
    6000:   depth = next->stratumValues[next->numStrata-1] - stratumValue;
    6001:   if ((depth < 0) || (depth >= next->numStrata)) {
    6002:     if (start) *start = 0;
    6003:     if (end)   *end   = 0;
    6004:   } else {
    6005:     if (start) *start = next->points[next->stratumOffsets[depth]];
    6006:     if (end)   *end   = next->points[next->stratumOffsets[depth]+next->stratumSizes[depth]-1]+1;
    6007:   }
    6008:   return(0);
    6009: }

    6013: /* Set the number of dof on each point and separate by fields */
    6014: PetscErrorCode DMPlexCreateSectionInitial(DM dm, PetscInt dim, PetscInt numFields,const PetscInt numComp[],const PetscInt numDof[], PetscSection *section)
    6015: {
    6016:   PetscInt      *numDofTot;
    6017:   PetscInt       pStart = 0, pEnd = 0;
    6018:   PetscInt       p, d, f;

    6022:   PetscMalloc((dim+1) * sizeof(PetscInt), &numDofTot);
    6023:   for (d = 0; d <= dim; ++d) {
    6024:     numDofTot[d] = 0;
    6025:     for (f = 0; f < numFields; ++f) numDofTot[d] += numDof[f*(dim+1)+d];
    6026:   }
    6027:   PetscSectionCreate(PetscObjectComm((PetscObject)dm), section);
    6028:   if (numFields > 0) {
    6029:     PetscSectionSetNumFields(*section, numFields);
    6030:     if (numComp) {
    6031:       for (f = 0; f < numFields; ++f) {
    6032:         PetscSectionSetFieldComponents(*section, f, numComp[f]);
    6033:       }
    6034:     }
    6035:   }
    6036:   DMPlexGetChart(dm, &pStart, &pEnd);
    6037:   PetscSectionSetChart(*section, pStart, pEnd);
    6038:   for (d = 0; d <= dim; ++d) {
    6039:     DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);
    6040:     for (p = pStart; p < pEnd; ++p) {
    6041:       for (f = 0; f < numFields; ++f) {
    6042:         PetscSectionSetFieldDof(*section, p, f, numDof[f*(dim+1)+d]);
    6043:       }
    6044:       PetscSectionSetDof(*section, p, numDofTot[d]);
    6045:     }
    6046:   }
    6047:   PetscFree(numDofTot);
    6048:   return(0);
    6049: }

    6053: /* Set the number of dof on each point and separate by fields
    6054:    If constDof is PETSC_DETERMINE, constrain every dof on the point
    6055: */
    6056: PetscErrorCode DMPlexCreateSectionBCDof(DM dm, PetscInt numBC,const PetscInt bcField[],const IS bcPoints[], PetscInt constDof, PetscSection section)
    6057: {
    6058:   PetscInt       numFields;
    6059:   PetscInt       bc;

    6063:   PetscSectionGetNumFields(section, &numFields);
    6064:   for (bc = 0; bc < numBC; ++bc) {
    6065:     PetscInt        field = 0;
    6066:     const PetscInt *idx;
    6067:     PetscInt        n, i;

    6069:     if (numFields) field = bcField[bc];
    6070:     ISGetLocalSize(bcPoints[bc], &n);
    6071:     ISGetIndices(bcPoints[bc], &idx);
    6072:     for (i = 0; i < n; ++i) {
    6073:       const PetscInt p        = idx[i];
    6074:       PetscInt       numConst = constDof;

    6076:       /* Constrain every dof on the point */
    6077:       if (numConst < 0) {
    6078:         if (numFields) {
    6079:           PetscSectionGetFieldDof(section, p, field, &numConst);
    6080:         } else {
    6081:           PetscSectionGetDof(section, p, &numConst);
    6082:         }
    6083:       }
    6084:       if (numFields) {
    6085:         PetscSectionAddFieldConstraintDof(section, p, field, numConst);
    6086:       }
    6087:       PetscSectionAddConstraintDof(section, p, numConst);
    6088:     }
    6089:     ISRestoreIndices(bcPoints[bc], &idx);
    6090:   }
    6091:   return(0);
    6092: }

    6096: /* Set the constrained indices on each point and separate by fields */
    6097: PetscErrorCode DMPlexCreateSectionBCIndicesAll(DM dm, PetscSection section)
    6098: {
    6099:   PetscInt      *maxConstraints;
    6100:   PetscInt       numFields, f, pStart = 0, pEnd = 0, p;

    6104:   PetscSectionGetNumFields(section, &numFields);
    6105:   PetscSectionGetChart(section, &pStart, &pEnd);
    6106:   PetscMalloc((numFields+1) * sizeof(PetscInt), &maxConstraints);
    6107:   for (f = 0; f <= numFields; ++f) maxConstraints[f] = 0;
    6108:   for (p = pStart; p < pEnd; ++p) {
    6109:     PetscInt cdof;

    6111:     if (numFields) {
    6112:       for (f = 0; f < numFields; ++f) {
    6113:         PetscSectionGetFieldConstraintDof(section, p, f, &cdof);
    6114:         maxConstraints[f] = PetscMax(maxConstraints[f], cdof);
    6115:       }
    6116:     } else {
    6117:       PetscSectionGetConstraintDof(section, p, &cdof);
    6118:       maxConstraints[0] = PetscMax(maxConstraints[0], cdof);
    6119:     }
    6120:   }
    6121:   for (f = 0; f < numFields; ++f) {
    6122:     maxConstraints[numFields] += maxConstraints[f];
    6123:   }
    6124:   if (maxConstraints[numFields]) {
    6125:     PetscInt *indices;

    6127:     PetscMalloc(maxConstraints[numFields] * sizeof(PetscInt), &indices);
    6128:     for (p = pStart; p < pEnd; ++p) {
    6129:       PetscInt cdof, d;

    6131:       PetscSectionGetConstraintDof(section, p, &cdof);
    6132:       if (cdof) {
    6133:         if (cdof > maxConstraints[numFields]) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_LIB, "Likely memory corruption, point %D cDof %D > maxConstraints %D", p, cdof, maxConstraints[numFields]);
    6134:         if (numFields) {
    6135:           PetscInt numConst = 0, foff = 0;

    6137:           for (f = 0; f < numFields; ++f) {
    6138:             PetscInt cfdof, fdof;

    6140:             PetscSectionGetFieldDof(section, p, f, &fdof);
    6141:             PetscSectionGetFieldConstraintDof(section, p, f, &cfdof);
    6142:             /* Change constraint numbering from absolute local dof number to field relative local dof number */
    6143:             for (d = 0; d < cfdof; ++d) indices[numConst+d] = d;
    6144:             PetscSectionSetFieldConstraintIndices(section, p, f, &indices[numConst]);
    6145:             for (d = 0; d < cfdof; ++d) indices[numConst+d] += foff;
    6146:             numConst += cfdof;
    6147:             foff     += fdof;
    6148:           }
    6149:           if (cdof != numConst) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_LIB, "Total number of field constraints %D should be %D", numConst, cdof);
    6150:         } else {
    6151:           for (d = 0; d < cdof; ++d) indices[d] = d;
    6152:         }
    6153:         PetscSectionSetConstraintIndices(section, p, indices);
    6154:       }
    6155:     }
    6156:     PetscFree(indices);
    6157:   }
    6158:   PetscFree(maxConstraints);
    6159:   return(0);
    6160: }

    6164: /* Set the constrained field indices on each point */
    6165: PetscErrorCode DMPlexCreateSectionBCIndicesField(DM dm, PetscInt field, IS bcPoints, IS constraintIndices, PetscSection section)
    6166: {
    6167:   const PetscInt *points, *indices;
    6168:   PetscInt        numFields, maxDof, numPoints, p, numConstraints;
    6169:   PetscErrorCode  ierr;

    6172:   PetscSectionGetNumFields(section, &numFields);
    6173:   if ((field < 0) || (field >= numFields)) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Section field %d should be in [%d, %d)", field, 0, numFields);

    6175:   ISGetLocalSize(bcPoints, &numPoints);
    6176:   ISGetIndices(bcPoints, &points);
    6177:   if (!constraintIndices) {
    6178:     PetscInt *idx, i;

    6180:     PetscSectionGetMaxDof(section, &maxDof);
    6181:     PetscMalloc(maxDof * sizeof(PetscInt), &idx);
    6182:     for (i = 0; i < maxDof; ++i) idx[i] = i;
    6183:     for (p = 0; p < numPoints; ++p) {
    6184:       PetscSectionSetFieldConstraintIndices(section, points[p], field, idx);
    6185:     }
    6186:     PetscFree(idx);
    6187:   } else {
    6188:     ISGetLocalSize(constraintIndices, &numConstraints);
    6189:     ISGetIndices(constraintIndices, &indices);
    6190:     for (p = 0; p < numPoints; ++p) {
    6191:       PetscInt fcdof;

    6193:       PetscSectionGetFieldConstraintDof(section, points[p], field, &fcdof);
    6194:       if (fcdof != numConstraints) SETERRQ4(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Section point %d field %d has %d constraints, but yo ugave %d indices", p, field, fcdof, numConstraints);
    6195:       PetscSectionSetFieldConstraintIndices(section, points[p], field, indices);
    6196:     }
    6197:     ISRestoreIndices(constraintIndices, &indices);
    6198:   }
    6199:   ISRestoreIndices(bcPoints, &points);
    6200:   return(0);
    6201: }

    6205: /* Set the constrained indices on each point and separate by fields */
    6206: PetscErrorCode DMPlexCreateSectionBCIndices(DM dm, PetscSection section)
    6207: {
    6208:   PetscInt      *indices;
    6209:   PetscInt       numFields, maxDof, f, pStart = 0, pEnd = 0, p;

    6213:   PetscSectionGetMaxDof(section, &maxDof);
    6214:   PetscMalloc(maxDof * sizeof(PetscInt), &indices);
    6215:   PetscSectionGetNumFields(section, &numFields);
    6216:   if (!numFields) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "This function only works after users have set field constraint indices.");
    6217:   PetscSectionGetChart(section, &pStart, &pEnd);
    6218:   for (p = pStart; p < pEnd; ++p) {
    6219:     PetscInt cdof, d;

    6221:     PetscSectionGetConstraintDof(section, p, &cdof);
    6222:     if (cdof) {
    6223:       PetscInt numConst = 0, foff = 0;

    6225:       for (f = 0; f < numFields; ++f) {
    6226:         const PetscInt *fcind;
    6227:         PetscInt        fdof, fcdof;

    6229:         PetscSectionGetFieldDof(section, p, f, &fdof);
    6230:         PetscSectionGetFieldConstraintDof(section, p, f, &fcdof);
    6231:         if (fcdof) {PetscSectionGetFieldConstraintIndices(section, p, f, &fcind);}
    6232:         /* Change constraint numbering from field relative local dof number to absolute local dof number */
    6233:         for (d = 0; d < fcdof; ++d) indices[numConst+d] = fcind[d]+foff;
    6234:         foff     += fdof;
    6235:         numConst += fcdof;
    6236:       }
    6237:       if (cdof != numConst) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_LIB, "Total number of field constraints %D should be %D", numConst, cdof);
    6238:       PetscSectionSetConstraintIndices(section, p, indices);
    6239:     }
    6240:   }
    6241:   PetscFree(indices);
    6242:   return(0);
    6243: }

    6247: /*@C
    6248:   DMPlexCreateSection - Create a PetscSection based upon the dof layout specification provided.

    6250:   Not Collective

    6252:   Input Parameters:
    6253: + dm        - The DMPlex object
    6254: . dim       - The spatial dimension of the problem
    6255: . numFields - The number of fields in the problem
    6256: . numComp   - An array of size numFields that holds the number of components for each field
    6257: . numDof    - An array of size numFields*(dim+1) which holds the number of dof for each field on a mesh piece of dimension d
    6258: . numBC     - The number of boundary conditions
    6259: . bcField   - An array of size numBC giving the field number for each boundry condition
    6260: - bcPoints  - An array of size numBC giving an IS holding the sieve points to which each boundary condition applies

    6262:   Output Parameter:
    6263: . section - The PetscSection object

    6265:   Notes: numDof[f*(dim+1)+d] gives the number of dof for field f on sieve points of dimension d. For instance, numDof[1] is the
    6266:   nubmer of dof for field 0 on each edge.

    6268:   Level: developer

    6270:   Fortran Notes:
    6271:   A Fortran 90 version is available as DMPlexCreateSectionF90()

    6273: .keywords: mesh, elements
    6274: .seealso: DMPlexCreate(), PetscSectionCreate()
    6275: @*/
    6276: PetscErrorCode DMPlexCreateSection(DM dm, PetscInt dim, PetscInt numFields,const PetscInt numComp[],const PetscInt numDof[], PetscInt numBC,const PetscInt bcField[],const IS bcPoints[], PetscSection *section)
    6277: {

    6281:   DMPlexCreateSectionInitial(dm, dim, numFields, numComp, numDof, section);
    6282:   DMPlexCreateSectionBCDof(dm, numBC, bcField, bcPoints, PETSC_DETERMINE, *section);
    6283:   PetscSectionSetUp(*section);
    6284:   if (numBC) {DMPlexCreateSectionBCIndicesAll(dm, *section);}
    6285:   {
    6286:     PetscBool view = PETSC_FALSE;

    6288:     PetscOptionsHasName(((PetscObject) dm)->prefix, "-section_view", &view);
    6289:     if (view) {PetscSectionView(*section, PETSC_VIEWER_STDOUT_WORLD);}
    6290:   }
    6291:   return(0);
    6292: }

    6296: PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
    6297: {
    6298:   PetscSection   section;

    6302:   DMPlexClone(dm, cdm);
    6303:   PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);
    6304:   DMSetDefaultSection(*cdm, section);
    6305:   PetscSectionDestroy(&section);
    6306:   return(0);
    6307: }

    6311: /*@
    6312:   DMPlexGetCoordinateSection - Retrieve the layout of coordinate values over the mesh.

    6314:   Not Collective

    6316:   Input Parameter:
    6317: . dm - The DMPlex object

    6319:   Output Parameter:
    6320: . section - The PetscSection object

    6322:   Level: intermediate

    6324: .keywords: mesh, coordinates
    6325: .seealso: DMGetCoordinateDM(), DMPlexGetDefaultSection(), DMPlexSetDefaultSection()
    6326: @*/
    6327: PetscErrorCode DMPlexGetCoordinateSection(DM dm, PetscSection *section)
    6328: {
    6329:   DM             cdm;

    6335:   DMGetCoordinateDM(dm, &cdm);
    6336:   DMGetDefaultSection(cdm, section);
    6337:   return(0);
    6338: }

    6342: /*@
    6343:   DMPlexSetCoordinateSection - Set the layout of coordinate values over the mesh.

    6345:   Not Collective

    6347:   Input Parameters:
    6348: + dm      - The DMPlex object
    6349: - section - The PetscSection object

    6351:   Level: intermediate

    6353: .keywords: mesh, coordinates
    6354: .seealso: DMPlexGetCoordinateSection(), DMPlexGetDefaultSection(), DMPlexSetDefaultSection()
    6355: @*/
    6356: PetscErrorCode DMPlexSetCoordinateSection(DM dm, PetscSection section)
    6357: {
    6358:   DM             cdm;

    6364:   DMGetCoordinateDM(dm, &cdm);
    6365:   DMSetDefaultSection(cdm, section);
    6366:   return(0);
    6367: }

    6371: PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
    6372: {
    6373:   DM_Plex *mesh = (DM_Plex*) dm->data;

    6377:   if (section) *section = mesh->coneSection;
    6378:   return(0);
    6379: }

    6383: PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
    6384: {
    6385:   DM_Plex *mesh = (DM_Plex*) dm->data;

    6389:   if (cones) *cones = mesh->cones;
    6390:   return(0);
    6391: }

    6395: PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
    6396: {
    6397:   DM_Plex *mesh = (DM_Plex*) dm->data;

    6401:   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
    6402:   return(0);
    6403: }

    6405: /******************************** FEM Support **********************************/

    6409: /*@C
    6410:   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'

    6412:   Not collective

    6414:   Input Parameters:
    6415: + dm - The DM
    6416: . section - The section describing the layout in v, or NULL to use the default section
    6417: . v - The local vector
    6418: - point - The sieve point in the DM

    6420:   Output Parameters:
    6421: + csize - The number of values in the closure, or NULL
    6422: - values - The array of values, which is a borrowed array and should not be freed

    6424:   Fortran Notes:
    6425:   Since it returns an array, this routine is only available in Fortran 90, and you must
    6426:   include petsc.h90 in your code.

    6428:   The csize argument is not present in the Fortran 90 binding since it is internal to the array.

    6430:   Level: intermediate

    6432: .seealso DMPlexVecRestoreClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
    6433: @*/
    6434: PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
    6435: {
    6436:   PetscScalar   *array, *vArray;
    6437:   PetscInt      *points = NULL;
    6438:   PetscInt       offsets[32];
    6439:   PetscInt       numFields, size, numPoints, pStart, pEnd, p, q, f;

    6445:   if (!section) {
    6446:     DMGetDefaultSection(dm, &section);
    6447:   }
    6448:   PetscSectionGetNumFields(section, &numFields);
    6449:   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
    6450:   PetscMemzero(offsets, 32 * sizeof(PetscInt));
    6451:   DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);
    6452:   /* Compress out points not in the section */
    6453:   PetscSectionGetChart(section, &pStart, &pEnd);
    6454:   for (p = 0, q = 0; p < numPoints*2; p += 2) {
    6455:     if ((points[p] >= pStart) && (points[p] < pEnd)) {
    6456:       points[q*2]   = points[p];
    6457:       points[q*2+1] = points[p+1];
    6458:       ++q;
    6459:     }
    6460:   }
    6461:   numPoints = q;
    6462:   for (p = 0, size = 0; p < numPoints*2; p += 2) {
    6463:     PetscInt dof, fdof;

    6465:     PetscSectionGetDof(section, points[p], &dof);
    6466:     for (f = 0; f < numFields; ++f) {
    6467:       PetscSectionGetFieldDof(section, points[p], f, &fdof);
    6468:       offsets[f+1] += fdof;
    6469:     }
    6470:     size += dof;
    6471:   }
    6472:   for (f = 1; f < numFields; ++f) offsets[f+1] += offsets[f];
    6473:   if (numFields && offsets[numFields] != size) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", offsets[numFields], size);
    6474:   DMGetWorkArray(dm, size, PETSC_SCALAR, &array);
    6475:   VecGetArray(v, &vArray);
    6476:   for (p = 0; p < numPoints*2; p += 2) {
    6477:     PetscInt     o = points[p+1];
    6478:     PetscInt     dof, off, d;
    6479:     PetscScalar *varr;

    6481:     PetscSectionGetDof(section, points[p], &dof);
    6482:     PetscSectionGetOffset(section, points[p], &off);
    6483:     varr = &vArray[off];
    6484:     if (numFields) {
    6485:       PetscInt fdof, foff, fcomp, f, c;

    6487:       for (f = 0, foff = 0; f < numFields; ++f) {
    6488:         PetscSectionGetFieldDof(section, points[p], f, &fdof);
    6489:         if (o >= 0) {
    6490:           for (d = 0; d < fdof; ++d, ++offsets[f]) {
    6491:             array[offsets[f]] = varr[foff+d];
    6492:           }
    6493:         } else {
    6494:           PetscSectionGetFieldComponents(section, f, &fcomp);
    6495:           for (d = fdof/fcomp-1; d >= 0; --d) {
    6496:             for (c = 0; c < fcomp; ++c, ++offsets[f]) {
    6497:               array[offsets[f]] = varr[foff+d*fcomp+c];
    6498:             }
    6499:           }
    6500:         }
    6501:         foff += fdof;
    6502:       }
    6503:     } else {
    6504:       if (o >= 0) {
    6505:         for (d = 0; d < dof; ++d, ++offsets[0]) {
    6506:           array[offsets[0]] = varr[d];
    6507:         }
    6508:       } else {
    6509:         for (d = dof-1; d >= 0; --d, ++offsets[0]) {
    6510:           array[offsets[0]] = varr[d];
    6511:         }
    6512:       }
    6513:     }
    6514:   }
    6515:   DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);
    6516:   VecRestoreArray(v, &vArray);
    6517:   if (csize) *csize = size;
    6518:   *values = array;
    6519:   return(0);
    6520: }

    6524: /*@C
    6525:   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point'

    6527:   Not collective

    6529:   Input Parameters:
    6530: + dm - The DM
    6531: . section - The section describing the layout in v, or NULL to use the default section
    6532: . v - The local vector
    6533: . point - The sieve point in the DM
    6534: . csize - The number of values in the closure, or NULL
    6535: - values - The array of values, which is a borrowed array and should not be freed

    6537:   Fortran Notes:
    6538:   Since it returns an array, this routine is only available in Fortran 90, and you must
    6539:   include petsc.h90 in your code.

    6541:   The csize argument is not present in the Fortran 90 binding since it is internal to the array.

    6543:   Level: intermediate

    6545: .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
    6546: @*/
    6547: PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
    6548: {
    6549:   PetscInt       size = 0;

    6553:   /* Should work without recalculating size */
    6554:   DMRestoreWorkArray(dm, size, PETSC_SCALAR, (void*) values);
    6555:   return(0);
    6556: }

    6558: PETSC_STATIC_INLINE void add   (PetscScalar *x, PetscScalar y) {*x += y;}
    6559: PETSC_STATIC_INLINE void insert(PetscScalar *x, PetscScalar y) {*x  = y;}

    6563: PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, PetscInt orientation, const PetscScalar values[], PetscScalar array[])
    6564: {
    6565:   PetscInt        cdof;   /* The number of constraints on this point */
    6566:   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
    6567:   PetscScalar    *a;
    6568:   PetscInt        off, cind = 0, k;
    6569:   PetscErrorCode  ierr;

    6572:   PetscSectionGetConstraintDof(section, point, &cdof);
    6573:   PetscSectionGetOffset(section, point, &off);
    6574:   a    = &array[off];
    6575:   if (!cdof || setBC) {
    6576:     if (orientation >= 0) {
    6577:       for (k = 0; k < dof; ++k) {
    6578:         fuse(&a[k], values[k]);
    6579:       }
    6580:     } else {
    6581:       for (k = 0; k < dof; ++k) {
    6582:         fuse(&a[k], values[dof-k-1]);
    6583:       }
    6584:     }
    6585:   } else {
    6586:     PetscSectionGetConstraintIndices(section, point, &cdofs);
    6587:     if (orientation >= 0) {
    6588:       for (k = 0; k < dof; ++k) {
    6589:         if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
    6590:         fuse(&a[k], values[k]);
    6591:       }
    6592:     } else {
    6593:       for (k = 0; k < dof; ++k) {
    6594:         if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
    6595:         fuse(&a[k], values[dof-k-1]);
    6596:       }
    6597:     }
    6598:   }
    6599:   return(0);
    6600: }

    6604: PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, PetscInt foffs[], void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, PetscInt orientation, const PetscScalar values[], PetscScalar array[])
    6605: {
    6606:   PetscScalar   *a;
    6607:   PetscInt       numFields, off, foff, f;

    6611:   PetscSectionGetNumFields(section, &numFields);
    6612:   PetscSectionGetOffset(section, point, &off);
    6613:   a    = &array[off];
    6614:   for (f = 0, foff = 0; f < numFields; ++f) {
    6615:     PetscInt        fdof, fcomp, fcdof;
    6616:     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
    6617:     PetscInt        cind = 0, k, c;

    6619:     PetscSectionGetFieldComponents(section, f, &fcomp);
    6620:     PetscSectionGetFieldDof(section, point, f, &fdof);
    6621:     PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);
    6622:     if (!fcdof || setBC) {
    6623:       if (orientation >= 0) {
    6624:         for (k = 0; k < fdof; ++k) {
    6625:           fuse(&a[foff+k], values[foffs[f]+k]);
    6626:         }
    6627:       } else {
    6628:         for (k = fdof/fcomp-1; k >= 0; --k) {
    6629:           for (c = 0; c < fcomp; ++c) {
    6630:             fuse(&a[foff+(fdof/fcomp-1-k)*fcomp+c], values[foffs[f]+k*fcomp+c]);
    6631:           }
    6632:         }
    6633:       }
    6634:     } else {
    6635:       PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);
    6636:       if (orientation >= 0) {
    6637:         for (k = 0; k < fdof; ++k) {
    6638:           if ((cind < fcdof) && (k == fcdofs[cind])) {++cind; continue;}
    6639:           fuse(&a[foff+k], values[foffs[f]+k]);
    6640:         }
    6641:       } else {
    6642:         for (k = fdof/fcomp-1; k >= 0; --k) {
    6643:           for (c = 0; c < fcomp; ++c) {
    6644:             if ((cind < fcdof) && (k*fcomp+c == fcdofs[cind])) {++cind; continue;}
    6645:             fuse(&a[foff+(fdof/fcomp-1-k)*fcomp+c], values[foffs[f]+k*fcomp+c]);
    6646:           }
    6647:         }
    6648:       }
    6649:     }
    6650:     foff     += fdof;
    6651:     foffs[f] += fdof;
    6652:   }
    6653:   return(0);
    6654: }

    6658: /*@C
    6659:   DMPlexVecSetClosure - Set an array of the values on the closure of 'point'

    6661:   Not collective

    6663:   Input Parameters:
    6664: + dm - The DM
    6665: . section - The section describing the layout in v, or NULL to use the default section
    6666: . v - The local vector
    6667: . point - The sieve point in the DM
    6668: . values - The array of values
    6669: - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions

    6671:   Fortran Notes:
    6672:   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.

    6674:   Level: intermediate

    6676: .seealso DMPlexVecGetClosure(), DMPlexMatSetClosure()
    6677: @*/
    6678: PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
    6679: {
    6680:   PetscScalar   *array;
    6681:   PetscInt      *points = NULL;
    6682:   PetscInt       offsets[32];
    6683:   PetscInt       numFields, numPoints, off, dof, pStart, pEnd, p, q, f;

    6689:   if (!section) {
    6690:     DMGetDefaultSection(dm, &section);
    6691:   }
    6692:   PetscSectionGetNumFields(section, &numFields);
    6693:   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
    6694:   PetscMemzero(offsets, 32 * sizeof(PetscInt));
    6695:   DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);
    6696:   /* Compress out points not in the section */
    6697:   PetscSectionGetChart(section, &pStart, &pEnd);
    6698:   for (p = 0, q = 0; p < numPoints*2; p += 2) {
    6699:     if ((points[p] >= pStart) && (points[p] < pEnd)) {
    6700:       points[q*2]   = points[p];
    6701:       points[q*2+1] = points[p+1];
    6702:       ++q;
    6703:     }
    6704:   }
    6705:   numPoints = q;
    6706:   for (p = 0; p < numPoints*2; p += 2) {
    6707:     PetscInt fdof;

    6709:     for (f = 0; f < numFields; ++f) {
    6710:       PetscSectionGetFieldDof(section, points[p], f, &fdof);
    6711:       offsets[f+1] += fdof;
    6712:     }
    6713:   }
    6714:   for (f = 1; f < numFields; ++f) offsets[f+1] += offsets[f];
    6715:   VecGetArray(v, &array);
    6716:   if (numFields) {
    6717:     switch (mode) {
    6718:     case INSERT_VALUES:
    6719:       for (p = 0; p < numPoints*2; p += 2) {
    6720:         PetscInt o = points[p+1];
    6721:         updatePointFields_private(section, points[p], offsets, insert, PETSC_FALSE, o, values, array);
    6722:       } break;
    6723:     case INSERT_ALL_VALUES:
    6724:       for (p = 0; p < numPoints*2; p += 2) {
    6725:         PetscInt o = points[p+1];
    6726:         updatePointFields_private(section, points[p], offsets, insert, PETSC_TRUE,  o, values, array);
    6727:       } break;
    6728:     case ADD_VALUES:
    6729:       for (p = 0; p < numPoints*2; p += 2) {
    6730:         PetscInt o = points[p+1];
    6731:         updatePointFields_private(section, points[p], offsets, add,    PETSC_FALSE, o, values, array);
    6732:       } break;
    6733:     case ADD_ALL_VALUES:
    6734:       for (p = 0; p < numPoints*2; p += 2) {
    6735:         PetscInt o = points[p+1];
    6736:         updatePointFields_private(section, points[p], offsets, add,    PETSC_TRUE,  o, values, array);
    6737:       } break;
    6738:     default:
    6739:       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %D", mode);
    6740:     }
    6741:   } else {
    6742:     switch (mode) {
    6743:     case INSERT_VALUES:
    6744:       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
    6745:         PetscInt o = points[p+1];
    6746:         PetscSectionGetDof(section, points[p], &dof);
    6747:         updatePoint_private(section, points[p], dof, insert, PETSC_FALSE, o, &values[off], array);
    6748:       } break;
    6749:     case INSERT_ALL_VALUES:
    6750:       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
    6751:         PetscInt o = points[p+1];
    6752:         PetscSectionGetDof(section, points[p], &dof);
    6753:         updatePoint_private(section, points[p], dof, insert, PETSC_TRUE,  o, &values[off], array);
    6754:       } break;
    6755:     case ADD_VALUES:
    6756:       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
    6757:         PetscInt o = points[p+1];
    6758:         PetscSectionGetDof(section, points[p], &dof);
    6759:         updatePoint_private(section, points[p], dof, add,    PETSC_FALSE, o, &values[off], array);
    6760:       } break;
    6761:     case ADD_ALL_VALUES:
    6762:       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
    6763:         PetscInt o = points[p+1];
    6764:         PetscSectionGetDof(section, points[p], &dof);
    6765:         updatePoint_private(section, points[p], dof, add,    PETSC_TRUE,  o, &values[off], array);
    6766:       } break;
    6767:     default:
    6768:       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %D", mode);
    6769:     }
    6770:   }
    6771:   DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);
    6772:   VecRestoreArray(v, &array);
    6773:   return(0);
    6774: }

    6778: PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numIndices, const PetscInt indices[], const PetscScalar values[])
    6779: {
    6780:   PetscMPIInt    rank;
    6781:   PetscInt       i, j;

    6785:   MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);
    6786:   PetscViewerASCIIPrintf(viewer, "[%D]mat for sieve point %D\n", rank, point);
    6787:   for (i = 0; i < numIndices; i++) {
    6788:     PetscViewerASCIIPrintf(viewer, "[%D]mat indices[%D] = %D\n", rank, i, indices[i]);
    6789:   }
    6790:   for (i = 0; i < numIndices; i++) {
    6791:     PetscViewerASCIIPrintf(viewer, "[%D]", rank);
    6792:     for (j = 0; j < numIndices; j++) {
    6793: #if defined(PETSC_USE_COMPLEX)
    6794:       PetscViewerASCIIPrintf(viewer, " (%G,%G)", PetscRealPart(values[i*numIndices+j]), PetscImaginaryPart(values[i*numIndices+j]));
    6795: #else
    6796:       PetscViewerASCIIPrintf(viewer, " %G", values[i*numIndices+j]);
    6797: #endif
    6798:     }
    6799:     PetscViewerASCIIPrintf(viewer, "\n");
    6800:   }
    6801:   return(0);
    6802: }

    6806: /* . off - The global offset of this point */
    6807: PetscErrorCode indicesPoint_private(PetscSection section, PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, PetscInt orientation, PetscInt indices[])
    6808: {
    6809:   PetscInt        dof;    /* The number of unknowns on this point */
    6810:   PetscInt        cdof;   /* The number of constraints on this point */
    6811:   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
    6812:   PetscInt        cind = 0, k;
    6813:   PetscErrorCode  ierr;

    6816:   PetscSectionGetDof(section, point, &dof);
    6817:   PetscSectionGetConstraintDof(section, point, &cdof);
    6818:   if (!cdof || setBC) {
    6819:     if (orientation >= 0) {
    6820:       for (k = 0; k < dof; ++k) indices[*loff+k] = off+k;
    6821:     } else {
    6822:       for (k = 0; k < dof; ++k) indices[*loff+dof-k-1] = off+k;
    6823:     }
    6824:   } else {
    6825:     PetscSectionGetConstraintIndices(section, point, &cdofs);
    6826:     if (orientation >= 0) {
    6827:       for (k = 0; k < dof; ++k) {
    6828:         if ((cind < cdof) && (k == cdofs[cind])) {
    6829:           /* Insert check for returning constrained indices */
    6830:           indices[*loff+k] = -(off+k+1);
    6831:           ++cind;
    6832:         } else {
    6833:           indices[*loff+k] = off+k-cind;
    6834:         }
    6835:       }
    6836:     } else {
    6837:       for (k = 0; k < dof; ++k) {
    6838:         if ((cind < cdof) && (k == cdofs[cind])) {
    6839:           /* Insert check for returning constrained indices */
    6840:           indices[*loff+dof-k-1] = -(off+k+1);
    6841:           ++cind;
    6842:         } else {
    6843:           indices[*loff+dof-k-1] = off+k-cind;
    6844:         }
    6845:       }
    6846:     }
    6847:   }
    6848:   *loff += dof;
    6849:   return(0);
    6850: }

    6854: /* . off - The global offset of this point */
    6855: PetscErrorCode indicesPointFields_private(PetscSection section, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, PetscInt orientation, PetscInt indices[])
    6856: {
    6857:   PetscInt       numFields, foff, f;

    6861:   PetscSectionGetNumFields(section, &numFields);
    6862:   for (f = 0, foff = 0; f < numFields; ++f) {
    6863:     PetscInt        fdof, fcomp, cfdof;
    6864:     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
    6865:     PetscInt        cind = 0, k, c;

    6867:     PetscSectionGetFieldComponents(section, f, &fcomp);
    6868:     PetscSectionGetFieldDof(section, point, f, &fdof);
    6869:     PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);
    6870:     if (!cfdof || setBC) {
    6871:       if (orientation >= 0) {
    6872:         for (k = 0; k < fdof; ++k) indices[foffs[f]+k] = off+foff+k;
    6873:       } else {
    6874:         for (k = fdof/fcomp-1; k >= 0; --k) {
    6875:           for (c = 0; c < fcomp; ++c) {
    6876:             indices[foffs[f]+k*fcomp+c] = off+foff+(fdof/fcomp-1-k)*fcomp+c;
    6877:           }
    6878:         }
    6879:       }
    6880:     } else {
    6881:       PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);
    6882:       if (orientation >= 0) {
    6883:         for (k = 0; k < fdof; ++k) {
    6884:           if ((cind < cfdof) && (k == fcdofs[cind])) {
    6885:             indices[foffs[f]+k] = -(off+foff+k+1);
    6886:             ++cind;
    6887:           } else {
    6888:             indices[foffs[f]+k] = off+foff+k-cind;
    6889:           }
    6890:         }
    6891:       } else {
    6892:         for (k = fdof/fcomp-1; k >= 0; --k) {
    6893:           for (c = 0; c < fcomp; ++c) {
    6894:             if ((cind < cfdof) && ((fdof/fcomp-1-k)*fcomp+c == fcdofs[cind])) {
    6895:               indices[foffs[f]+k*fcomp+c] = -(off+foff+(fdof/fcomp-1-k)*fcomp+c+1);
    6896:               ++cind;
    6897:             } else {
    6898:               indices[foffs[f]+k*fcomp+c] = off+foff+(fdof/fcomp-1-k)*fcomp+c-cind;
    6899:             }
    6900:           }
    6901:         }
    6902:       }
    6903:     }
    6904:     foff     += fdof - cfdof;
    6905:     foffs[f] += fdof;
    6906:   }
    6907:   return(0);
    6908: }

    6912: /*@C
    6913:   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'

    6915:   Not collective

    6917:   Input Parameters:
    6918: + dm - The DM
    6919: . section - The section describing the layout in v, or NULL to use the default section
    6920: . globalSection - The section describing the layout in v, or NULL to use the default section
    6921: . A - The matrix
    6922: . point - The sieve point in the DM
    6923: . values - The array of values
    6924: - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions

    6926:   Fortran Notes:
    6927:   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.

    6929:   Level: intermediate

    6931: .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure()
    6932: @*/
    6933: PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
    6934: {
    6935:   DM_Plex       *mesh   = (DM_Plex*) dm->data;
    6936:   PetscInt      *points = NULL;
    6937:   PetscInt      *indices;
    6938:   PetscInt       offsets[32];
    6939:   PetscInt       numFields, numPoints, numIndices, dof, off, globalOff, pStart, pEnd, p, q, f;
    6940:   PetscBool      useDefault       =       !section ? PETSC_TRUE : PETSC_FALSE;
    6941:   PetscBool      useGlobalDefault = !globalSection ? PETSC_TRUE : PETSC_FALSE;

    6947:   if (useDefault) {
    6948:     DMGetDefaultSection(dm, &section);
    6949:   }
    6950:   if (useGlobalDefault) {
    6951:     if (useDefault) {
    6952:       DMGetDefaultGlobalSection(dm, &globalSection);
    6953:     } else {
    6954:       PetscSectionCreateGlobalSection(section, dm->sf, PETSC_FALSE, &globalSection);
    6955:     }
    6956:   }
    6957:   PetscSectionGetNumFields(section, &numFields);
    6958:   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
    6959:   PetscMemzero(offsets, 32 * sizeof(PetscInt));
    6960:   DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);
    6961:   /* Compress out points not in the section */
    6962:   PetscSectionGetChart(section, &pStart, &pEnd);
    6963:   for (p = 0, q = 0; p < numPoints*2; p += 2) {
    6964:     if ((points[p] >= pStart) && (points[p] < pEnd)) {
    6965:       points[q*2]   = points[p];
    6966:       points[q*2+1] = points[p+1];
    6967:       ++q;
    6968:     }
    6969:   }
    6970:   numPoints = q;
    6971:   for (p = 0, numIndices = 0; p < numPoints*2; p += 2) {
    6972:     PetscInt fdof;

    6974:     PetscSectionGetDof(section, points[p], &dof);
    6975:     for (f = 0; f < numFields; ++f) {
    6976:       PetscSectionGetFieldDof(section, points[p], f, &fdof);
    6977:       offsets[f+1] += fdof;
    6978:     }
    6979:     numIndices += dof;
    6980:   }
    6981:   for (f = 1; f < numFields; ++f) offsets[f+1] += offsets[f];

    6983:   if (numFields && offsets[numFields] != numIndices) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", offsets[numFields], numIndices);
    6984:   DMGetWorkArray(dm, numIndices, PETSC_INT, &indices);
    6985:   if (numFields) {
    6986:     for (p = 0; p < numPoints*2; p += 2) {
    6987:       PetscInt o = points[p+1];
    6988:       PetscSectionGetOffset(globalSection, points[p], &globalOff);
    6989:       indicesPointFields_private(section, points[p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, o, indices);
    6990:     }
    6991:   } else {
    6992:     for (p = 0, off = 0; p < numPoints*2; p += 2) {
    6993:       PetscInt o = points[p+1];
    6994:       PetscSectionGetOffset(globalSection, points[p], &globalOff);
    6995:       indicesPoint_private(section, points[p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, o, indices);
    6996:     }
    6997:   }
    6998:   if (useGlobalDefault && !useDefault) {
    6999:     PetscSectionDestroy(&globalSection);
    7000:   }
    7001:   if (mesh->printSetValues) {DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, values);}
    7002:   MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);
    7003:   if (ierr) {
    7004:     PetscMPIInt    rank;
    7005:     PetscErrorCode ierr2;

    7007:     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
    7008:     ierr2 = (*PetscErrorPrintf)("[%D]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
    7009:     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, values);CHKERRQ(ierr2);
    7010:     ierr2 = DMRestoreWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr2);
    7011:
    7012:   }
    7013:   DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);
    7014:   DMRestoreWorkArray(dm, numIndices, PETSC_INT, &indices);
    7015:   return(0);
    7016: }

    7020: PetscErrorCode DMPlexGetHybridBounds(DM dm, PetscInt *cMax, PetscInt *fMax, PetscInt *eMax, PetscInt *vMax)
    7021: {
    7022:   DM_Plex       *mesh = (DM_Plex*) dm->data;
    7023:   PetscInt       dim;

    7028:   DMPlexGetDimension(dm, &dim);
    7029:   if (cMax) *cMax = mesh->hybridPointMax[dim];
    7030:   if (fMax) *fMax = mesh->hybridPointMax[dim-1];
    7031:   if (eMax) *eMax = mesh->hybridPointMax[1];
    7032:   if (vMax) *vMax = mesh->hybridPointMax[0];
    7033:   return(0);
    7034: }

    7038: PetscErrorCode DMPlexSetHybridBounds(DM dm, PetscInt cMax, PetscInt fMax, PetscInt eMax, PetscInt vMax)
    7039: {
    7040:   DM_Plex       *mesh = (DM_Plex*) dm->data;
    7041:   PetscInt       dim;

    7046:   DMPlexGetDimension(dm, &dim);
    7047:   if (cMax >= 0) mesh->hybridPointMax[dim]   = cMax;
    7048:   if (fMax >= 0) mesh->hybridPointMax[dim-1] = fMax;
    7049:   if (eMax >= 0) mesh->hybridPointMax[1]     = eMax;
    7050:   if (vMax >= 0) mesh->hybridPointMax[0]     = vMax;
    7051:   return(0);
    7052: }

    7056: PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
    7057: {
    7058:   DM_Plex *mesh = (DM_Plex*) dm->data;

    7063:   *cellHeight = mesh->vtkCellHeight;
    7064:   return(0);
    7065: }

    7069: PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
    7070: {
    7071:   DM_Plex *mesh = (DM_Plex*) dm->data;

    7075:   mesh->vtkCellHeight = cellHeight;
    7076:   return(0);
    7077: }

    7081: /* We can easily have a form that takes an IS instead */
    7082: PetscErrorCode DMPlexCreateNumbering_Private(DM dm, PetscInt pStart, PetscInt pEnd, PetscSF sf, IS *numbering)
    7083: {
    7084:   PetscSection   section, globalSection;
    7085:   PetscInt      *numbers, p;

    7089:   PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);
    7090:   PetscSectionSetChart(section, pStart, pEnd);
    7091:   for (p = pStart; p < pEnd; ++p) {
    7092:     PetscSectionSetDof(section, p, 1);
    7093:   }
    7094:   PetscSectionSetUp(section);
    7095:   PetscSectionCreateGlobalSection(section, sf, PETSC_FALSE, &globalSection);
    7096:   PetscMalloc((pEnd - pStart) * sizeof(PetscInt), &numbers);
    7097:   for (p = pStart; p < pEnd; ++p) {
    7098:     PetscSectionGetOffset(globalSection, p, &numbers[p-pStart]);
    7099:   }
    7100:   ISCreateGeneral(PetscObjectComm((PetscObject)dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering);
    7101:   PetscSectionDestroy(&section);
    7102:   PetscSectionDestroy(&globalSection);
    7103:   return(0);
    7104: }

    7108: PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
    7109: {
    7110:   DM_Plex       *mesh = (DM_Plex*) dm->data;
    7111:   PetscInt       cellHeight, cStart, cEnd, cMax;

    7116:   if (!mesh->globalCellNumbers) {
    7117:     DMPlexGetVTKCellHeight(dm, &cellHeight);
    7118:     DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);
    7119:     DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);
    7120:     if (cMax >= 0) cEnd = PetscMin(cEnd, cMax);
    7121:     DMPlexCreateNumbering_Private(dm, cStart, cEnd, dm->sf, &mesh->globalCellNumbers);
    7122:   }
    7123:   *globalCellNumbers = mesh->globalCellNumbers;
    7124:   return(0);
    7125: }

    7129: PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
    7130: {
    7131:   DM_Plex       *mesh = (DM_Plex*) dm->data;
    7132:   PetscInt       vStart, vEnd, vMax;

    7137:   if (!mesh->globalVertexNumbers) {
    7138:     DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);
    7139:     DMPlexGetHybridBounds(dm, NULL, NULL, NULL, &vMax);
    7140:     if (vMax >= 0) vEnd = PetscMin(vEnd, vMax);
    7141:     DMPlexCreateNumbering_Private(dm, vStart, vEnd, dm->sf, &mesh->globalVertexNumbers);
    7142:   }
    7143:   *globalVertexNumbers = mesh->globalVertexNumbers;
    7144:   return(0);
    7145: }

    7150: /*@C
    7151:   PetscSectionCreateGlobalSectionLabel - Create a section describing the global field layout using
    7152:   the local section and an SF describing the section point overlap.

    7154:   Input Parameters:
    7155:   + s - The PetscSection for the local field layout
    7156:   . sf - The SF describing parallel layout of the section points
    7157:   . includeConstraints - By default this is PETSC_FALSE, meaning that the global field vector will not possess constrained dofs
    7158:   . label - The label specifying the points
    7159:   - labelValue - The label stratum specifying the points

    7161:   Output Parameter:
    7162:   . gsection - The PetscSection for the global field layout

    7164:   Note: This gives negative sizes and offsets to points not owned by this process

    7166:   Level: developer

    7168: .seealso: PetscSectionCreate()
    7169: @*/
    7170: PetscErrorCode PetscSectionCreateGlobalSectionLabel(PetscSection s, PetscSF sf, PetscBool includeConstraints, DMLabel label, PetscInt labelValue, PetscSection *gsection)
    7171: {
    7172:   PetscInt      *neg;
    7173:   PetscInt       pStart, pEnd, p, dof, cdof, off, globalOff = 0, nroots;

    7177:   PetscSectionCreate(s->atlasLayout.comm, gsection);
    7178:   PetscSectionGetChart(s, &pStart, &pEnd);
    7179:   PetscSectionSetChart(*gsection, pStart, pEnd);
    7180:   PetscMalloc((pEnd - pStart) * sizeof(PetscInt), &neg);
    7181:   /* Mark ghost points with negative dof */
    7182:   for (p = pStart; p < pEnd; ++p) {
    7183:     PetscInt value;

    7185:     DMLabelGetValue(label, p, &value);
    7186:     if (value != labelValue) continue;
    7187:     PetscSectionGetDof(s, p, &dof);
    7188:     PetscSectionSetDof(*gsection, p, dof);
    7189:     PetscSectionGetConstraintDof(s, p, &cdof);
    7190:     if (!includeConstraints && cdof > 0) {PetscSectionSetConstraintDof(*gsection, p, cdof);}
    7191:     neg[p-pStart] = -(dof+1);
    7192:   }
    7193:   PetscSectionSetUpBC(*gsection);
    7194:   PetscSFGetGraph(sf, &nroots, NULL, NULL, NULL);
    7195:   if (nroots >= 0) {
    7196:     if (nroots > pEnd - pStart) {
    7197:       PetscInt *tmpDof;
    7198:       /* Help Jed: HAVE TO MAKE A BUFFER HERE THE SIZE OF THE COMPLETE SPACE AND THEN COPY INTO THE atlasDof FOR THIS SECTION */
    7199:       PetscMalloc(nroots * sizeof(PetscInt), &tmpDof);
    7200:       PetscSFBcastBegin(sf, MPIU_INT, &neg[-pStart], tmpDof);
    7201:       PetscSFBcastEnd(sf, MPIU_INT, &neg[-pStart], tmpDof);
    7202:       for (p = pStart; p < pEnd; ++p) {
    7203:         if (tmpDof[p] < 0) (*gsection)->atlasDof[p-pStart] = tmpDof[p];
    7204:       }
    7205:       PetscFree(tmpDof);
    7206:     } else {
    7207:       PetscSFBcastBegin(sf, MPIU_INT, &neg[-pStart], &(*gsection)->atlasDof[-pStart]);
    7208:       PetscSFBcastEnd(sf, MPIU_INT, &neg[-pStart], &(*gsection)->atlasDof[-pStart]);
    7209:     }
    7210:   }
    7211:   /* Calculate new sizes, get proccess offset, and calculate point offsets */
    7212:   for (p = 0, off = 0; p < pEnd-pStart; ++p) {
    7213:     cdof = (!includeConstraints && s->bc) ? s->bc->atlasDof[p] : 0;

    7215:     (*gsection)->atlasOff[p] = off;

    7217:     off += (*gsection)->atlasDof[p] > 0 ? (*gsection)->atlasDof[p]-cdof : 0;
    7218:   }
    7219:   MPI_Scan(&off, &globalOff, 1, MPIU_INT, MPI_SUM, s->atlasLayout.comm);
    7220:   globalOff -= off;
    7221:   for (p = 0, off = 0; p < pEnd-pStart; ++p) {
    7222:     (*gsection)->atlasOff[p] += globalOff;

    7224:     neg[p] = -((*gsection)->atlasOff[p]+1);
    7225:   }
    7226:   /* Put in negative offsets for ghost points */
    7227:   if (nroots >= 0) {
    7228:     if (nroots > pEnd - pStart) {
    7229:       PetscInt *tmpOff;
    7230:       /* Help Jed: HAVE TO MAKE A BUFFER HERE THE SIZE OF THE COMPLETE SPACE AND THEN COPY INTO THE atlasDof FOR THIS SECTION */
    7231:       PetscMalloc(nroots * sizeof(PetscInt), &tmpOff);
    7232:       PetscSFBcastBegin(sf, MPIU_INT, &neg[-pStart], tmpOff);
    7233:       PetscSFBcastEnd(sf, MPIU_INT, &neg[-pStart], tmpOff);
    7234:       for (p = pStart; p < pEnd; ++p) {
    7235:         if (tmpOff[p] < 0) (*gsection)->atlasOff[p-pStart] = tmpOff[p];
    7236:       }
    7237:       PetscFree(tmpOff);
    7238:     } else {
    7239:       PetscSFBcastBegin(sf, MPIU_INT, &neg[-pStart], &(*gsection)->atlasOff[-pStart]);
    7240:       PetscSFBcastEnd(sf, MPIU_INT, &neg[-pStart], &(*gsection)->atlasOff[-pStart]);
    7241:     }
    7242:   }
    7243:   PetscFree(neg);
    7244:   return(0);
    7245: }