Date: 2013-07-02T14:45:11+00:00

Actual source code: dadd.c
petsc-3.4.2 2013-07-02

      1: #include <petsc-private/dmdaimpl.h>  /*I   "petscdmda.h"   I*/

      5: /*@
      6:   DMDACreatePatchIS - Creates an index set corresponding to a patch of the DA.

      8:   Not Collective

     10:   Input Parameters:
     11: +  da - the DMDA
     12: .  lower - a matstencil with i, j and k corresponding to the lower corner of the patch
     13: -  upper - a matstencil with i, j and k corresponding to the upper corner of the patch

     15:   Output Parameters:
     16: .  is - the IS corresponding to the patch

     18:   Level: developer

     20: .seealso: DMDACreateDomainDecomposition(), DMDACreateDomainDecompositionScatters()
     21: @*/
     22: PetscErrorCode DMDACreatePatchIS(DM da,MatStencil *lower,MatStencil *upper,IS *is)
     23: {
     24:   PetscInt       ms=0,ns=0,ps=0;
     25:   PetscInt       me=1,ne=1,pe=1;
     26:   PetscInt       mr=0,nr=0,pr=0;
     27:   PetscInt       ii,jj,kk;
     28:   PetscInt       si,sj,sk;
     29:   PetscInt       i,j,k,l,idx;
     30:   PetscInt       base;
     31:   PetscInt       xm=1,ym=1,zm=1;
     32:   const PetscInt *lx,*ly,*lz;
     33:   PetscInt       ox,oy,oz;
     34:   PetscInt       m,n,p,M,N,P,dof;
     35:   PetscInt       nindices;
     36:   PetscInt       *indices;
     37:   DM_DA          *dd = (DM_DA*)da->data;

     41:   /* need to get the sizes of the actual DM rather than the "global" space of a subdomain DM */
     42:   M = dd->M;N = dd->N;P=dd->P;
     43:   m = dd->m;n = dd->n;p=dd->p;
     44:   dof = dd->w;
     45:   DMDAGetOffset(da,&ox,&oy,&oz,NULL,NULL,NULL);
     46:   DMDAGetOwnershipRanges(da,&lx,&ly,&lz);
     47:   nindices = (upper->i - lower->i)*(upper->j - lower->j)*(upper->k - lower->k)*dof;
     48:   PetscMalloc(sizeof(PetscInt)*nindices,&indices);
     49:   /* start at index 0 on processor 0 */
     50:   mr = 0;
     51:   nr = 0;
     52:   pr = 0;
     53:   ms = 0;
     54:   ns = 0;
     55:   ps = 0;
     56:   if (lx) me = lx[0];
     57:   if (ly) ne = ly[0];
     58:   if (lz) pe = lz[0];
     59:   idx = 0;
     60:   for (k=lower->k-oz;k<upper->k-oz;k++) {
     61:     for (j=lower->j-oy;j < upper->j-oy;j++) {
     62:       for (i=lower->i-ox;i < upper->i-ox;i++) {
     63:         /* "actual" indices rather than ones outside of the domain */
     64:         ii = i;
     65:         jj = j;
     66:         kk = k;
     67:         if (ii < 0) ii = M + ii;
     68:         if (jj < 0) jj = N + jj;
     69:         if (kk < 0) kk = P + kk;
     70:         if (ii > M-1) ii = ii - M;
     71:         if (jj > N-1) jj = jj - N;
     72:         if (kk > P-1) kk = kk - P;
     73:         /* gone out of processor range on x axis */
     74:         while(ii > me-1 || ii < ms) {
     75:           if (mr == m-1) {
     76:             ms = 0;
     77:             me = lx[0];
     78:             mr = 0;
     79:           } else {
     80:             mr++;
     81:             ms = me;
     82:             me += lx[mr];
     83:           }
     84:         }
     85:         /* gone out of processor range on y axis */
     86:         while(jj > ne-1 || jj < ns) {
     87:           if (nr == n-1) {
     88:             ns = 0;
     89:             ne = ly[0];
     90:             nr = 0;
     91:           } else {
     92:             nr++;
     93:             ns = ne;
     94:             ne += ly[nr];
     95:           }
     96:         }
     97:         /* gone out of processor range on z axis */
     98:         while(kk > pe-1 || kk < ps) {
     99:           if (pr == p-1) {
    100:             ps = 0;
    101:             pe = lz[0];
    102:             pr = 0;
    103:           } else {
    104:             pr++;
    105:             ps = pe;
    106:             pe += lz[pr];
    107:           }
    108:         }
    109:         /* compute the vector base on owning processor */
    110:         xm = me - ms;
    111:         ym = ne - ns;
    112:         zm = pe - ps;
    113:         base = ms*ym*zm + ns*M + ps*M*N;
    114:         /* compute the local coordinates on owning processor */
    115:         si = ii - ms;
    116:         sj = jj - ns;
    117:         sk = kk - ps;
    118:         for (l=0;l<dof;l++) {
    119:           indices[idx] = l + dof*(base + si + xm*sj + xm*ym*sk);
    120:           idx++;
    121:         }
    122:       }
    123:     }
    124:   }
    125:   ISCreateGeneral(PETSC_COMM_SELF,idx,indices,PETSC_OWN_POINTER,is);
    126:   return(0);
    127: }

    131: PetscErrorCode DMDASubDomainDA_Private(DM dm, PetscInt *nlocal, DM **sdm)
    132: {
    133:   DM             *da;
    134:   PetscInt       dim,size,i,j,k,idx;
    136:   DMDALocalInfo  info;
    137:   PetscInt       xsize,ysize,zsize;
    138:   PetscInt       xo,yo,zo;
    139:   PetscInt       xs,ys,zs;
    140:   PetscInt       xm=1,ym=1,zm=1;
    141:   PetscInt       xol,yol,zol;
    142:   PetscInt       m=1,n=1,p=1;
    143:   PetscInt       M,N,P;
    144:   PetscInt       pm,mtmp;

    147:   DMDAGetLocalInfo(dm,&info);
    148:   DMDAGetOverlap(dm,&xol,&yol,&zol);
    149:   DMDAGetNumLocalSubDomains(dm,&size);
    150:   PetscMalloc(size*sizeof(DM),&da);

    152:   if (nlocal) *nlocal = size;

    154:   dim = info.dim;

    156:   M = info.xm;
    157:   N = info.ym;
    158:   P = info.zm;

    160:   if (dim == 1) {
    161:     m = size;
    162:   } else if (dim == 2) {
    163:     m = (PetscInt)(0.5 + PetscSqrtReal(((PetscReal)M)*((PetscReal)size)/((PetscReal)N)));
    164:     while (m > 0) {
    165:       n = size/m;
    166:       if (m*n*p == size) break;
    167:       m--;
    168:     }
    169:   } else if (dim == 3) {
    170:     n = (PetscInt)(0.5 + PetscPowReal(((PetscReal)N*N)*((PetscReal)size)/((PetscReal)P*M),(PetscReal)(1./3.)));    if (!n) n = 1;
    171:     while (n > 0) {
    172:       pm = size/n;
    173:       if (n*pm == size) break;
    174:       n--;
    175:     }
    176:     if (!n) n = 1;
    177:     m = (PetscInt)(0.5 + PetscSqrtReal(((PetscReal)M)*((PetscReal)size)/((PetscReal)P*n)));
    178:     if (!m) m = 1;
    179:     while (m > 0) {
    180:       p = size/(m*n);
    181:       if (m*n*p == size) break;
    182:       m--;
    183:     }
    184:     if (M > P && m < p) {mtmp = m; m = p; p = mtmp;}
    185:   }

    187:   zs = info.zs;
    188:   idx = 0;
    189:   for (k = 0; k < p; k++) {
    190:     ys = info.ys;
    191:     for (j = 0; j < n; j++) {
    192:       xs = info.xs;
    193:       for (i = 0; i < m; i++) {
    194:         if (dim == 1) {
    195:           xm = M/m + ((M % m) > i);
    196:         } else if (dim == 2) {
    197:           xm = M/m + ((M % m) > i);
    198:           ym = N/n + ((N % n) > j);
    199:         } else if (dim == 3) {
    200:           xm = M/m + ((M % m) > i);
    201:           ym = N/n + ((N % n) > j);
    202:           zm = P/p + ((P % p) > k);
    203:         }

    205:         xsize = xm;
    206:         ysize = ym;
    207:         zsize = zm;
    208:         xo = xs;
    209:         yo = ys;
    210:         zo = zs;

    212:         DMDACreate(PETSC_COMM_SELF,&(da[idx]));
    213:         DMSetOptionsPrefix(da[idx],"sub_");
    214:         DMDASetDim(da[idx], info.dim);
    215:         DMDASetDof(da[idx], info.dof);

    217:         DMDASetStencilType(da[idx],info.st);
    218:         DMDASetStencilWidth(da[idx],info.sw);

    220:         if (info.bx == DMDA_BOUNDARY_PERIODIC || (xs != 0)) {
    221:           xsize += xol;
    222:           xo    -= xol;
    223:         }
    224:         if (info.by == DMDA_BOUNDARY_PERIODIC || (ys != 0)) {
    225:           ysize += yol;
    226:           yo    -= yol;
    227:         }
    228:         if (info.bz == DMDA_BOUNDARY_PERIODIC || (zs != 0)) {
    229:           zsize += zol;
    230:           zo    -= zol;
    231:         }

    233:         if (info.bx == DMDA_BOUNDARY_PERIODIC || (xs+xm != info.mx)) xsize += xol;
    234:         if (info.by == DMDA_BOUNDARY_PERIODIC || (ys+ym != info.my)) ysize += yol;
    235:         if (info.bz == DMDA_BOUNDARY_PERIODIC || (zs+zm != info.mz)) zsize += zol;

    237:         if (info.bx != DMDA_BOUNDARY_PERIODIC) {
    238:           if (xo < 0) {
    239:             xsize += xo;
    240:             xo = 0;
    241:           }
    242:           if (xo+xsize > info.mx-1) {
    243:             xsize -= xo+xsize - info.mx;
    244:           }
    245:         }
    246:         if (info.by != DMDA_BOUNDARY_PERIODIC) {
    247:           if (yo < 0) {
    248:             ysize += yo;
    249:             yo = 0;
    250:           }
    251:           if (yo+ysize > info.my-1) {
    252:             ysize -= yo+ysize - info.my;
    253:           }
    254:         }
    255:         if (info.bz != DMDA_BOUNDARY_PERIODIC) {
    256:           if (zo < 0) {
    257:             zsize += zo;
    258:             zo = 0;
    259:           }
    260:           if (zo+zsize > info.mz-1) {
    261:             zsize -= zo+zsize - info.mz;
    262:           }
    263:         }

    265:         DMDASetSizes(da[idx], xsize, ysize, zsize);
    266:         DMDASetNumProcs(da[idx], 1, 1, 1);
    267:         DMDASetBoundaryType(da[idx], DMDA_BOUNDARY_GHOSTED, DMDA_BOUNDARY_GHOSTED, DMDA_BOUNDARY_GHOSTED);

    269:         /* set up as a block instead */
    270:         DMSetUp(da[idx]);

    272:         /* nonoverlapping region */
    273:         DMDASetNonOverlappingRegion(da[idx],xs,ys,zs,xm,ym,zm);

    275:         /* this alters the behavior of DMDAGetInfo, DMDAGetLocalInfo, DMDAGetCorners, and DMDAGetGhostedCorners and should be used with care */
    276:         DMDASetOffset(da[idx],xo,yo,zo,info.mx,info.my,info.mz);
    277:         xs += xm;
    278:         idx++;
    279:       }
    280:       ys += ym;
    281:     }
    282:     zs += zm;
    283:   }
    284:   *sdm = da;
    285:   return(0);
    286: }

    290: /*
    291:  Fills the local vector problem on the subdomain from the global problem.

    293:  Right now this assumes one subdomain per processor.

    295:  */
    296: PetscErrorCode DMCreateDomainDecompositionScatters_DA(DM dm,PetscInt nsubdms,DM *subdms,VecScatter **iscat,VecScatter **oscat, VecScatter **lscat)
    297: {
    299:   DMDALocalInfo  info,subinfo;
    300:   DM             subdm;
    301:   MatStencil     upper,lower;
    302:   IS             idis,isis,odis,osis,gdis;
    303:   Vec            svec,dvec,slvec;
    304:   PetscInt       xm,ym,zm,xs,ys,zs;
    305:   PetscInt       i;

    309:   /* allocate the arrays of scatters */
    310:   if (iscat) {PetscMalloc(nsubdms*sizeof(VecScatter*),iscat);}
    311:   if (oscat) {PetscMalloc(nsubdms*sizeof(VecScatter*),oscat);}
    312:   if (lscat) {PetscMalloc(nsubdms*sizeof(VecScatter*),lscat);}

    314:   DMDAGetLocalInfo(dm,&info);
    315:   for (i = 0; i < nsubdms; i++) {
    316:     subdm = subdms[i];
    317:     DMDAGetLocalInfo(subdm,&subinfo);
    318:     DMDAGetNonOverlappingRegion(subdm,&xs,&ys,&zs,&xm,&ym,&zm);

    320:     /* create the global and subdomain index sets for the inner domain */
    321:     lower.i = xs;
    322:     lower.j = ys;
    323:     lower.k = zs;
    324:     upper.i = xs+xm;
    325:     upper.j = ys+ym;
    326:     upper.k = zs+zm;
    327:     DMDACreatePatchIS(dm,&lower,&upper,&idis);
    328:     DMDACreatePatchIS(subdm,&lower,&upper,&isis);

    330:     /* create the global and subdomain index sets for the outer subdomain */
    331:     lower.i = subinfo.xs;
    332:     lower.j = subinfo.ys;
    333:     lower.k = subinfo.zs;
    334:     upper.i = subinfo.xs+subinfo.xm;
    335:     upper.j = subinfo.ys+subinfo.ym;
    336:     upper.k = subinfo.zs+subinfo.zm;
    337:     DMDACreatePatchIS(dm,&lower,&upper,&odis);
    338:     DMDACreatePatchIS(subdm,&lower,&upper,&osis);

    340:     /* global and subdomain ISes for the local indices of the subdomain */
    341:     /* todo - make this not loop over at nonperiodic boundaries, which will be more involved */
    342:     lower.i = subinfo.gxs;
    343:     lower.j = subinfo.gys;
    344:     lower.k = subinfo.gzs;
    345:     upper.i = subinfo.gxs+subinfo.gxm;
    346:     upper.j = subinfo.gys+subinfo.gym;
    347:     upper.k = subinfo.gzs+subinfo.gzm;

    349:     DMDACreatePatchIS(dm,&lower,&upper,&gdis);

    351:     /* form the scatter */
    352:     DMGetGlobalVector(dm,&dvec);
    353:     DMGetGlobalVector(subdm,&svec);
    354:     DMGetLocalVector(subdm,&slvec);

    356:     if (iscat) {VecScatterCreate(dvec,idis,svec,isis,&(*iscat)[i]);}
    357:     if (oscat) {VecScatterCreate(dvec,odis,svec,osis,&(*oscat)[i]);}
    358:     if (lscat) {VecScatterCreate(dvec,gdis,slvec,NULL,&(*lscat)[i]);}

    360:     DMRestoreGlobalVector(dm,&dvec);
    361:     DMRestoreGlobalVector(subdm,&svec);
    362:     DMRestoreLocalVector(subdm,&slvec);

    364:     ISDestroy(&idis);
    365:     ISDestroy(&isis);

    367:     ISDestroy(&odis);
    368:     ISDestroy(&osis);

    370:     ISDestroy(&gdis);
    371:   }
    372:   return(0);
    373: }

    377: PetscErrorCode DMDASubDomainIS_Private(DM dm,PetscInt n,DM *subdm,IS **iis,IS **ois)
    378: {
    380:   PetscInt       i;
    381:   DMDALocalInfo  info,subinfo;
    382:   MatStencil     lower,upper;

    385:   DMDAGetLocalInfo(dm,&info);
    386:   if (iis) {PetscMalloc(n*sizeof(IS*),iis);}
    387:   if (ois) {PetscMalloc(n*sizeof(IS*),ois);}

    389:   for (i = 0;i < n; i++) {
    390:     DMDAGetLocalInfo(subdm[i],&subinfo);
    391:     if (iis) {
    392:       /* create the inner IS */
    393:       lower.i = info.xs;
    394:       lower.j = info.ys;
    395:       lower.k = info.zs;
    396:       upper.i = info.xs+info.xm;
    397:       upper.j = info.ys+info.ym;
    398:       upper.k = info.zs+info.zm;
    399:       DMDACreatePatchIS(dm,&lower,&upper,&(*iis)[i]);
    400:     }

    402:     if (ois) {
    403:       /* create the outer IS */
    404:       lower.i = subinfo.xs;
    405:       lower.j = subinfo.ys;
    406:       lower.k = subinfo.zs;
    407:       upper.i = subinfo.xs+subinfo.xm;
    408:       upper.j = subinfo.ys+subinfo.ym;
    409:       upper.k = subinfo.zs+subinfo.zm;
    410:       DMDACreatePatchIS(dm,&lower,&upper,&(*ois)[i]);
    411:     }
    412:   }
    413:   return(0);
    414: }

    418: PetscErrorCode DMCreateDomainDecomposition_DA(DM dm,PetscInt *len,char ***names,IS **iis,IS **ois,DM **subdm)
    419: {
    421:   DM             *sdm;
    422:   PetscInt       n,i;

    425:   DMDASubDomainDA_Private(dm,&n,&sdm);
    426:   if (names) {
    427:     PetscMalloc(n*sizeof(char*),names);
    428:     for (i=0;i<n;i++) (*names)[i] = 0;
    429:   }
    430:   DMDASubDomainIS_Private(dm,n,sdm,iis,ois);
    431:   if (subdm) *subdm = sdm;
    432:   else {
    433:     for (i=0;i<n;i++) {
    434:       DMDestroy(&sdm[i]);
    435:     }
    436:   }
    437:   if (len) *len = n;
    438:   return(0);
    439: }