Date: 2015-10-02T22:17:53+00:00

Actual source code: bjacobi.c
petsc-3.6.2 2015-10-02
Report Typos and Errors

      2: /*
      3:    Defines a block Jacobi preconditioner.
      4: */
      5: #include <petsc/private/pcimpl.h>              /*I "petscpc.h" I*/
      6: #include <../src/ksp/pc/impls/bjacobi/bjacobi.h>

      8: static PetscErrorCode PCSetUp_BJacobi_Singleblock(PC,Mat,Mat);
      9: static PetscErrorCode PCSetUp_BJacobi_Multiblock(PC,Mat,Mat);
     10: static PetscErrorCode PCSetUp_BJacobi_Multiproc(PC);

     14: static PetscErrorCode PCSetUp_BJacobi(PC pc)
     15: {
     16:   PC_BJacobi     *jac = (PC_BJacobi*)pc->data;
     17:   Mat            mat  = pc->mat,pmat = pc->pmat;
     18:   PetscErrorCode ierr,(*f)(Mat,Mat*);
     19:   PetscInt       N,M,start,i,sum,end;
     20:   PetscInt       bs,i_start=-1,i_end=-1;
     21:   PetscMPIInt    rank,size;
     22:   const char     *pprefix,*mprefix;

     25:   MPI_Comm_rank(PetscObjectComm((PetscObject)pc),&rank);
     26:   MPI_Comm_size(PetscObjectComm((PetscObject)pc),&size);
     27:   MatGetLocalSize(pc->pmat,&M,&N);
     28:   MatGetBlockSize(pc->pmat,&bs);

     30:   if (jac->n > 0 && jac->n < size) {
     31:     PCSetUp_BJacobi_Multiproc(pc);
     32:     return(0);
     33:   }

     35:   /* --------------------------------------------------------------------------
     36:       Determines the number of blocks assigned to each processor
     37:   -----------------------------------------------------------------------------*/

     39:   /*   local block count  given */
     40:   if (jac->n_local > 0 && jac->n < 0) {
     41:     MPI_Allreduce(&jac->n_local,&jac->n,1,MPIU_INT,MPI_SUM,PetscObjectComm((PetscObject)pc));
     42:     if (jac->l_lens) { /* check that user set these correctly */
     43:       sum = 0;
     44:       for (i=0; i<jac->n_local; i++) {
     45:         if (jac->l_lens[i]/bs*bs !=jac->l_lens[i]) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"Mat blocksize doesn't match block Jacobi layout");
     46:         sum += jac->l_lens[i];
     47:       }
     48:       if (sum != M) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"Local lens set incorrectly");
     49:     } else {
     50:       PetscMalloc1(jac->n_local,&jac->l_lens);
     51:       for (i=0; i<jac->n_local; i++) jac->l_lens[i] = bs*((M/bs)/jac->n_local + (((M/bs) % jac->n_local) > i));
     52:     }
     53:   } else if (jac->n > 0 && jac->n_local < 0) { /* global block count given */
     54:     /* global blocks given: determine which ones are local */
     55:     if (jac->g_lens) {
     56:       /* check if the g_lens is has valid entries */
     57:       for (i=0; i<jac->n; i++) {
     58:         if (!jac->g_lens[i]) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"Zero block not allowed");
     59:         if (jac->g_lens[i]/bs*bs != jac->g_lens[i]) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"Mat blocksize doesn't match block Jacobi layout");
     60:       }
     61:       if (size == 1) {
     62:         jac->n_local = jac->n;
     63:         PetscMalloc1(jac->n_local,&jac->l_lens);
     64:         PetscMemcpy(jac->l_lens,jac->g_lens,jac->n_local*sizeof(PetscInt));
     65:         /* check that user set these correctly */
     66:         sum = 0;
     67:         for (i=0; i<jac->n_local; i++) sum += jac->l_lens[i];
     68:         if (sum != M) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"Global lens set incorrectly");
     69:       } else {
     70:         MatGetOwnershipRange(pc->pmat,&start,&end);
     71:         /* loop over blocks determing first one owned by me */
     72:         sum = 0;
     73:         for (i=0; i<jac->n+1; i++) {
     74:           if (sum == start) { i_start = i; goto start_1;}
     75:           if (i < jac->n) sum += jac->g_lens[i];
     76:         }
     77:         SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"Block sizes used in PCBJacobiSetTotalBlocks()\nare not compatible with parallel matrix layout");
     78: start_1:
     79:         for (i=i_start; i<jac->n+1; i++) {
     80:           if (sum == end) { i_end = i; goto end_1; }
     81:           if (i < jac->n) sum += jac->g_lens[i];
     82:         }
     83:         SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"Block sizes used in PCBJacobiSetTotalBlocks()\nare not compatible with parallel matrix layout");
     84: end_1:
     85:         jac->n_local = i_end - i_start;
     86:         PetscMalloc1(jac->n_local,&jac->l_lens);
     87:         PetscMemcpy(jac->l_lens,jac->g_lens+i_start,jac->n_local*sizeof(PetscInt));
     88:       }
     89:     } else { /* no global blocks given, determine then using default layout */
     90:       jac->n_local = jac->n/size + ((jac->n % size) > rank);
     91:       PetscMalloc1(jac->n_local,&jac->l_lens);
     92:       for (i=0; i<jac->n_local; i++) {
     93:         jac->l_lens[i] = ((M/bs)/jac->n_local + (((M/bs) % jac->n_local) > i))*bs;
     94:         if (!jac->l_lens[i]) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"Too many blocks given");
     95:       }
     96:     }
     97:   } else if (jac->n < 0 && jac->n_local < 0) { /* no blocks given */
     98:     jac->n         = size;
     99:     jac->n_local   = 1;
    100:     PetscMalloc1(1,&jac->l_lens);
    101:     jac->l_lens[0] = M;
    102:   } else { /* jac->n > 0 && jac->n_local > 0 */
    103:     if (!jac->l_lens) {
    104:       PetscMalloc1(jac->n_local,&jac->l_lens);
    105:       for (i=0; i<jac->n_local; i++) jac->l_lens[i] = bs*((M/bs)/jac->n_local + (((M/bs) % jac->n_local) > i));
    106:     }
    107:   }
    108:   if (jac->n_local < 1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Number of blocks is less than number of processors");

    110:   /* -------------------------
    111:       Determines mat and pmat
    112:   ---------------------------*/
    113:   PetscObjectQueryFunction((PetscObject)pc->mat,"MatGetDiagonalBlock_C",&f);
    114:   if (!f && size == 1) {
    115:     mat  = pc->mat;
    116:     pmat = pc->pmat;
    117:   } else {
    118:     if (pc->useAmat) {
    119:       /* use block from Amat matrix, not Pmat for local MatMult() */
    120:       MatGetDiagonalBlock(pc->mat,&mat);
    121:       /* make submatrix have same prefix as entire matrix */
    122:       PetscObjectGetOptionsPrefix((PetscObject)pc->mat,&mprefix);
    123:       PetscObjectSetOptionsPrefix((PetscObject)mat,mprefix);
    124:     }
    125:     if (pc->pmat != pc->mat || !pc->useAmat) {
    126:       MatGetDiagonalBlock(pc->pmat,&pmat);
    127:       /* make submatrix have same prefix as entire matrix */
    128:       PetscObjectGetOptionsPrefix((PetscObject)pc->pmat,&pprefix);
    129:       PetscObjectSetOptionsPrefix((PetscObject)pmat,pprefix);
    130:     } else pmat = mat;
    131:   }

    133:   /* ------
    134:      Setup code depends on the number of blocks
    135:   */
    136:   if (jac->n_local == 1) {
    137:     PCSetUp_BJacobi_Singleblock(pc,mat,pmat);
    138:   } else {
    139:     PCSetUp_BJacobi_Multiblock(pc,mat,pmat);
    140:   }
    141:   return(0);
    142: }

    144: /* Default destroy, if it has never been setup */
    147: static PetscErrorCode PCDestroy_BJacobi(PC pc)
    148: {
    149:   PC_BJacobi     *jac = (PC_BJacobi*)pc->data;

    153:   PetscFree(jac->g_lens);
    154:   PetscFree(jac->l_lens);
    155:   PetscFree(pc->data);
    156:   return(0);
    157: }

    162: static PetscErrorCode PCSetFromOptions_BJacobi(PetscOptions *PetscOptionsObject,PC pc)
    163: {
    164:   PC_BJacobi     *jac = (PC_BJacobi*)pc->data;
    166:   PetscInt       blocks,i;
    167:   PetscBool      flg;

    170:   PetscOptionsHead(PetscOptionsObject,"Block Jacobi options");
    171:   PetscOptionsInt("-pc_bjacobi_blocks","Total number of blocks","PCBJacobiSetTotalBlocks",jac->n,&blocks,&flg);
    172:   if (flg) {
    173:     PCBJacobiSetTotalBlocks(pc,blocks,NULL);
    174:   }
    175:   if (jac->ksp) {
    176:     /* The sub-KSP has already been set up (e.g., PCSetUp_BJacobi_Singleblock), but KSPSetFromOptions was not called
    177:      * unless we had already been called. */
    178:     for (i=0; i<jac->n_local; i++) {
    179:       KSPSetFromOptions(jac->ksp[i]);
    180:     }
    181:   }
    182:   PetscOptionsTail();
    183:   return(0);
    184: }

    186: #include <petscdraw.h>
    189: static PetscErrorCode PCView_BJacobi(PC pc,PetscViewer viewer)
    190: {
    191:   PC_BJacobi           *jac   = (PC_BJacobi*)pc->data;
    192:   PC_BJacobi_Multiproc *mpjac = (PC_BJacobi_Multiproc*)jac->data;
    193:   PetscErrorCode       ierr;
    194:   PetscMPIInt          rank;
    195:   PetscInt             i;
    196:   PetscBool            iascii,isstring,isdraw;
    197:   PetscViewer          sviewer;

    200:   PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERASCII,&iascii);
    201:   PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERSTRING,&isstring);
    202:   PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERDRAW,&isdraw);
    203:   if (iascii) {
    204:     if (pc->useAmat) {
    205:       PetscViewerASCIIPrintf(viewer,"  block Jacobi: using Amat local matrix, number of blocks = %D\n",jac->n);
    206:     }
    207:     PetscViewerASCIIPrintf(viewer,"  block Jacobi: number of blocks = %D\n",jac->n);
    208:     MPI_Comm_rank(PetscObjectComm((PetscObject)pc),&rank);
    209:     if (jac->same_local_solves) {
    210:       PetscViewerASCIIPrintf(viewer,"  Local solve is same for all blocks, in the following KSP and PC objects:\n");
    211:       if (jac->ksp && !jac->psubcomm) {
    212:         PetscViewerGetSingleton(viewer,&sviewer);
    213:         if (!rank) {
    214:           PetscViewerASCIIPushTab(viewer);
    215:           KSPView(jac->ksp[0],sviewer);
    216:           PetscViewerASCIIPopTab(viewer);
    217:         }
    218:         PetscViewerRestoreSingleton(viewer,&sviewer);
    219:       } else if (jac->psubcomm && !jac->psubcomm->color) {
    220:         PetscViewerASCIIGetStdout(PetscSubcommChild(mpjac->psubcomm),&sviewer);
    221:         PetscViewerASCIIPushTab(viewer);
    222:         KSPView(*(jac->ksp),sviewer);
    223:         PetscViewerASCIIPopTab(viewer);
    224:       }
    225:     } else {
    226:       PetscInt n_global;
    227:       MPI_Allreduce(&jac->n_local,&n_global,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)pc));
    228:       PetscViewerASCIISynchronizedAllow(viewer,PETSC_TRUE);
    229:       PetscViewerASCIIPrintf(viewer,"  Local solve info for each block is in the following KSP and PC objects:\n");
    230:       PetscViewerASCIISynchronizedPrintf(viewer,"[%d] number of local blocks = %D, first local block number = %D\n",
    231:                                                 rank,jac->n_local,jac->first_local);
    232:       PetscViewerASCIIPushTab(viewer);
    233:       PetscViewerGetSingleton(viewer,&sviewer);
    234:       for (i=0; i<jac->n_local; i++) {
    235:         PetscViewerASCIISynchronizedPrintf(viewer,"[%d] local block number %D\n",rank,i);
    236:         KSPView(jac->ksp[i],sviewer);
    237:         PetscViewerASCIISynchronizedPrintf(viewer,"- - - - - - - - - - - - - - - - - -\n");
    238:       }
    239:       PetscViewerRestoreSingleton(viewer,&sviewer);
    240:       PetscViewerASCIIPopTab(viewer);
    241:       PetscViewerFlush(viewer);
    242:       PetscViewerASCIISynchronizedAllow(viewer,PETSC_FALSE);
    243:     }
    244:   } else if (isstring) {
    245:     PetscViewerStringSPrintf(viewer," blks=%D",jac->n);
    246:     PetscViewerGetSingleton(viewer,&sviewer);
    247:     if (jac->ksp) {KSPView(jac->ksp[0],sviewer);}
    248:     PetscViewerRestoreSingleton(viewer,&sviewer);
    249:   } else if (isdraw) {
    250:     PetscDraw draw;
    251:     char      str[25];
    252:     PetscReal x,y,bottom,h;

    254:     PetscViewerDrawGetDraw(viewer,0,&draw);
    255:     PetscDrawGetCurrentPoint(draw,&x,&y);
    256:     PetscSNPrintf(str,25,"Number blocks %D",jac->n);
    257:     PetscDrawStringBoxed(draw,x,y,PETSC_DRAW_RED,PETSC_DRAW_BLACK,str,NULL,&h);
    258:     bottom = y - h;
    259:     PetscDrawPushCurrentPoint(draw,x,bottom);
    260:     /* warning the communicator on viewer is different then on ksp in parallel */
    261:     if (jac->ksp) {KSPView(jac->ksp[0],viewer);}
    262:     PetscDrawPopCurrentPoint(draw);
    263:   }
    264:   return(0);
    265: }

    267: /* -------------------------------------------------------------------------------------*/

    271: static PetscErrorCode  PCBJacobiGetSubKSP_BJacobi(PC pc,PetscInt *n_local,PetscInt *first_local,KSP **ksp)
    272: {
    273:   PC_BJacobi *jac = (PC_BJacobi*)pc->data;;

    276:   if (!pc->setupcalled) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_WRONGSTATE,"Must call KSPSetUp() or PCSetUp() first");

    278:   if (n_local) *n_local = jac->n_local;
    279:   if (first_local) *first_local = jac->first_local;
    280:   *ksp                   = jac->ksp;
    281:   jac->same_local_solves = PETSC_FALSE;        /* Assume that local solves are now different;
    282:                                                   not necessarily true though!  This flag is
    283:                                                   used only for PCView_BJacobi() */
    284:   return(0);
    285: }

    289: static PetscErrorCode  PCBJacobiSetTotalBlocks_BJacobi(PC pc,PetscInt blocks,PetscInt *lens)
    290: {
    291:   PC_BJacobi     *jac = (PC_BJacobi*)pc->data;

    295:   if (pc->setupcalled > 0 && jac->n!=blocks) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_ORDER,"Cannot alter number of blocks after PCSetUp()/KSPSetUp() has been called");
    296:   jac->n = blocks;
    297:   if (!lens) jac->g_lens = 0;
    298:   else {
    299:     PetscMalloc1(blocks,&jac->g_lens);
    300:     PetscLogObjectMemory((PetscObject)pc,blocks*sizeof(PetscInt));
    301:     PetscMemcpy(jac->g_lens,lens,blocks*sizeof(PetscInt));
    302:   }
    303:   return(0);
    304: }

    308: static PetscErrorCode  PCBJacobiGetTotalBlocks_BJacobi(PC pc, PetscInt *blocks, const PetscInt *lens[])
    309: {
    310:   PC_BJacobi *jac = (PC_BJacobi*) pc->data;

    313:   *blocks = jac->n;
    314:   if (lens) *lens = jac->g_lens;
    315:   return(0);
    316: }

    320: static PetscErrorCode  PCBJacobiSetLocalBlocks_BJacobi(PC pc,PetscInt blocks,const PetscInt lens[])
    321: {
    322:   PC_BJacobi     *jac;

    326:   jac = (PC_BJacobi*)pc->data;

    328:   jac->n_local = blocks;
    329:   if (!lens) jac->l_lens = 0;
    330:   else {
    331:     PetscMalloc1(blocks,&jac->l_lens);
    332:     PetscLogObjectMemory((PetscObject)pc,blocks*sizeof(PetscInt));
    333:     PetscMemcpy(jac->l_lens,lens,blocks*sizeof(PetscInt));
    334:   }
    335:   return(0);
    336: }

    340: static PetscErrorCode  PCBJacobiGetLocalBlocks_BJacobi(PC pc, PetscInt *blocks, const PetscInt *lens[])
    341: {
    342:   PC_BJacobi *jac = (PC_BJacobi*) pc->data;

    345:   *blocks = jac->n_local;
    346:   if (lens) *lens = jac->l_lens;
    347:   return(0);
    348: }

    350: /* -------------------------------------------------------------------------------------*/

    354: /*@C
    355:    PCBJacobiGetSubKSP - Gets the local KSP contexts for all blocks on
    356:    this processor.

    358:    Note Collective

    360:    Input Parameter:
    361: .  pc - the preconditioner context

    363:    Output Parameters:
    364: +  n_local - the number of blocks on this processor, or NULL
    365: .  first_local - the global number of the first block on this processor, or NULL
    366: -  ksp - the array of KSP contexts

    368:    Notes:
    369:    After PCBJacobiGetSubKSP() the array of KSP contexts is not to be freed.

    371:    Currently for some matrix implementations only 1 block per processor
    372:    is supported.

    374:    You must call KSPSetUp() or PCSetUp() before calling PCBJacobiGetSubKSP().

    376:    Fortran Usage: You must pass in a KSP array that is large enough to contain all the local KSPs.
    377:       You can call PCBJacobiGetSubKSP(pc,nlocal,firstlocal,NULL_OBJECT,ierr) to determine how large the
    378:       KSP array must be.

    380:    Level: advanced

    382: .keywords:  block, Jacobi, get, sub, KSP, context

    384: .seealso: PCBJacobiGetSubKSP()
    385: @*/
    386: PetscErrorCode  PCBJacobiGetSubKSP(PC pc,PetscInt *n_local,PetscInt *first_local,KSP *ksp[])
    387: {

    392:   PetscUseMethod(pc,"PCBJacobiGetSubKSP_C",(PC,PetscInt*,PetscInt*,KSP **),(pc,n_local,first_local,ksp));
    393:   return(0);
    394: }

    398: /*@
    399:    PCBJacobiSetTotalBlocks - Sets the global number of blocks for the block
    400:    Jacobi preconditioner.

    402:    Collective on PC

    404:    Input Parameters:
    405: +  pc - the preconditioner context
    406: .  blocks - the number of blocks
    407: -  lens - [optional] integer array containing the size of each block

    409:    Options Database Key:
    410: .  -pc_bjacobi_blocks <blocks> - Sets the number of global blocks

    412:    Notes:
    413:    Currently only a limited number of blocking configurations are supported.
    414:    All processors sharing the PC must call this routine with the same data.

    416:    Level: intermediate

    418: .keywords:  set, number, Jacobi, global, total, blocks

    420: .seealso: PCSetUseAmat(), PCBJacobiSetLocalBlocks()
    421: @*/
    422: PetscErrorCode  PCBJacobiSetTotalBlocks(PC pc,PetscInt blocks,const PetscInt lens[])
    423: {

    428:   if (blocks <= 0) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_OUTOFRANGE,"Must have positive blocks");
    429:   PetscTryMethod(pc,"PCBJacobiSetTotalBlocks_C",(PC,PetscInt,const PetscInt[]),(pc,blocks,lens));
    430:   return(0);
    431: }

    435: /*@C
    436:    PCBJacobiGetTotalBlocks - Gets the global number of blocks for the block
    437:    Jacobi preconditioner.

    439:    Not Collective

    441:    Input Parameter:
    442: .  pc - the preconditioner context

    444:    Output parameters:
    445: +  blocks - the number of blocks
    446: -  lens - integer array containing the size of each block

    448:    Level: intermediate

    450: .keywords:  get, number, Jacobi, global, total, blocks

    452: .seealso: PCSetUseAmat(), PCBJacobiGetLocalBlocks()
    453: @*/
    454: PetscErrorCode  PCBJacobiGetTotalBlocks(PC pc, PetscInt *blocks, const PetscInt *lens[])
    455: {

    461:   PetscUseMethod(pc,"PCBJacobiGetTotalBlocks_C",(PC,PetscInt*, const PetscInt *[]),(pc,blocks,lens));
    462:   return(0);
    463: }

    467: /*@
    468:    PCBJacobiSetLocalBlocks - Sets the local number of blocks for the block
    469:    Jacobi preconditioner.

    471:    Not Collective

    473:    Input Parameters:
    474: +  pc - the preconditioner context
    475: .  blocks - the number of blocks
    476: -  lens - [optional] integer array containing size of each block

    478:    Note:
    479:    Currently only a limited number of blocking configurations are supported.

    481:    Level: intermediate

    483: .keywords: PC, set, number, Jacobi, local, blocks

    485: .seealso: PCSetUseAmat(), PCBJacobiSetTotalBlocks()
    486: @*/
    487: PetscErrorCode  PCBJacobiSetLocalBlocks(PC pc,PetscInt blocks,const PetscInt lens[])
    488: {

    493:   if (blocks < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Must have nonegative blocks");
    494:   PetscTryMethod(pc,"PCBJacobiSetLocalBlocks_C",(PC,PetscInt,const PetscInt []),(pc,blocks,lens));
    495:   return(0);
    496: }

    500: /*@C
    501:    PCBJacobiGetLocalBlocks - Gets the local number of blocks for the block
    502:    Jacobi preconditioner.

    504:    Not Collective

    506:    Input Parameters:
    507: +  pc - the preconditioner context
    508: .  blocks - the number of blocks
    509: -  lens - [optional] integer array containing size of each block

    511:    Note:
    512:    Currently only a limited number of blocking configurations are supported.

    514:    Level: intermediate

    516: .keywords: PC, get, number, Jacobi, local, blocks

    518: .seealso: PCSetUseAmat(), PCBJacobiGetTotalBlocks()
    519: @*/
    520: PetscErrorCode  PCBJacobiGetLocalBlocks(PC pc, PetscInt *blocks, const PetscInt *lens[])
    521: {

    527:   PetscUseMethod(pc,"PCBJacobiGetLocalBlocks_C",(PC,PetscInt*, const PetscInt *[]),(pc,blocks,lens));
    528:   return(0);
    529: }

    531: /* -----------------------------------------------------------------------------------*/

    533: /*MC
    534:    PCBJACOBI - Use block Jacobi preconditioning, each block is (approximately) solved with
    535:            its own KSP object.

    537:    Options Database Keys:
    538: .  -pc_use_amat - use Amat to apply block of operator in inner Krylov method

    540:    Notes: Each processor can have one or more blocks, but a block cannot be shared by more
    541:      than one processor. Defaults to one block per processor.

    543:      To set options on the solvers for each block append -sub_ to all the KSP, KSP, and PC
    544:         options database keys. For example, -sub_pc_type ilu -sub_pc_factor_levels 1 -sub_ksp_type preonly

    546:      To set the options on the solvers separate for each block call PCBJacobiGetSubKSP()
    547:          and set the options directly on the resulting KSP object (you can access its PC
    548:          KSPGetPC())

    550:    Level: beginner

    552:    Concepts: block Jacobi

    554:    Developer Notes: This preconditioner does not currently work with CUDA/CUSP for a couple of reasons.
    555:        (1) It creates seq vectors as work vectors that should be cusp
    556:        (2) The use of VecPlaceArray() is not handled properly by CUSP (that is it will not know where
    557:            the ownership of the vector is so may use wrong values) even if it did know the ownership
    558:            it may induce extra copy ups and downs. Satish suggests a VecTransplantArray() to handle two
    559:            vectors sharing the same pointer and handling the CUSP side as well instead of VecGetArray()/VecPlaceArray().

    562: .seealso:  PCCreate(), PCSetType(), PCType (for list of available types), PC,
    563:            PCASM, PCSetUseAmat(), PCGetUseAmat(), PCBJacobiGetSubKSP(), PCBJacobiSetTotalBlocks(),
    564:            PCBJacobiSetLocalBlocks(), PCSetModifySubmatrices()
    565: M*/

    569: PETSC_EXTERN PetscErrorCode PCCreate_BJacobi(PC pc)
    570: {
    572:   PetscMPIInt    rank;
    573:   PC_BJacobi     *jac;

    576:   PetscNewLog(pc,&jac);
    577:   MPI_Comm_rank(PetscObjectComm((PetscObject)pc),&rank);

    579:   pc->ops->apply           = 0;
    580:   pc->ops->applytranspose  = 0;
    581:   pc->ops->setup           = PCSetUp_BJacobi;
    582:   pc->ops->destroy         = PCDestroy_BJacobi;
    583:   pc->ops->setfromoptions  = PCSetFromOptions_BJacobi;
    584:   pc->ops->view            = PCView_BJacobi;
    585:   pc->ops->applyrichardson = 0;

    587:   pc->data               = (void*)jac;
    588:   jac->n                 = -1;
    589:   jac->n_local           = -1;
    590:   jac->first_local       = rank;
    591:   jac->ksp               = 0;
    592:   jac->same_local_solves = PETSC_TRUE;
    593:   jac->g_lens            = 0;
    594:   jac->l_lens            = 0;
    595:   jac->psubcomm          = 0;

    597:   PetscObjectComposeFunction((PetscObject)pc,"PCBJacobiGetSubKSP_C",PCBJacobiGetSubKSP_BJacobi);
    598:   PetscObjectComposeFunction((PetscObject)pc,"PCBJacobiSetTotalBlocks_C",PCBJacobiSetTotalBlocks_BJacobi);
    599:   PetscObjectComposeFunction((PetscObject)pc,"PCBJacobiGetTotalBlocks_C",PCBJacobiGetTotalBlocks_BJacobi);
    600:   PetscObjectComposeFunction((PetscObject)pc,"PCBJacobiSetLocalBlocks_C",PCBJacobiSetLocalBlocks_BJacobi);
    601:   PetscObjectComposeFunction((PetscObject)pc,"PCBJacobiGetLocalBlocks_C",PCBJacobiGetLocalBlocks_BJacobi);
    602:   return(0);
    603: }

    605: /* --------------------------------------------------------------------------------------------*/
    606: /*
    607:         These are for a single block per processor; works for AIJ, BAIJ; Seq and MPI
    608: */
    611: PetscErrorCode PCReset_BJacobi_Singleblock(PC pc)
    612: {
    613:   PC_BJacobi             *jac  = (PC_BJacobi*)pc->data;
    614:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
    615:   PetscErrorCode         ierr;

    618:   KSPReset(jac->ksp[0]);
    619:   VecDestroy(&bjac->x);
    620:   VecDestroy(&bjac->y);
    621:   return(0);
    622: }

    626: PetscErrorCode PCDestroy_BJacobi_Singleblock(PC pc)
    627: {
    628:   PC_BJacobi             *jac  = (PC_BJacobi*)pc->data;
    629:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
    630:   PetscErrorCode         ierr;

    633:   PCReset_BJacobi_Singleblock(pc);
    634:   KSPDestroy(&jac->ksp[0]);
    635:   PetscFree(jac->ksp);
    636:   PetscFree(jac->l_lens);
    637:   PetscFree(jac->g_lens);
    638:   PetscFree(bjac);
    639:   PetscFree(pc->data);
    640:   return(0);
    641: }

    645: PetscErrorCode PCSetUpOnBlocks_BJacobi_Singleblock(PC pc)
    646: {
    648:   PC_BJacobi     *jac = (PC_BJacobi*)pc->data;

    651:   KSPSetUp(jac->ksp[0]);
    652:   return(0);
    653: }

    657: PetscErrorCode PCApply_BJacobi_Singleblock(PC pc,Vec x,Vec y)
    658: {
    659:   PetscErrorCode         ierr;
    660:   PC_BJacobi             *jac  = (PC_BJacobi*)pc->data;
    661:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;

    664:   VecGetLocalVectorRead(x, bjac->x);
    665:   VecGetLocalVector(y, bjac->y);
    666:  /* Since the inner KSP matrix may point directly to the diagonal block of an MPI matrix the inner
    667:      matrix may change even if the outter KSP/PC has not updated the preconditioner, this will trigger a rebuild
    668:      of the inner preconditioner automatically unless we pass down the outter preconditioners reuse flag.*/
    669:   KSPSetReusePreconditioner(jac->ksp[0],pc->reusepreconditioner);
    670:   KSPSolve(jac->ksp[0],bjac->x,bjac->y);
    671:   VecRestoreLocalVectorRead(x, bjac->x);
    672:   VecRestoreLocalVector(y, bjac->y);
    673:   return(0);
    674: }

    678: PetscErrorCode PCApplySymmetricLeft_BJacobi_Singleblock(PC pc,Vec x,Vec y)
    679: {
    680:   PetscErrorCode         ierr;
    681:   PC_BJacobi             *jac  = (PC_BJacobi*)pc->data;
    682:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
    683:   PetscScalar            *y_array;
    684:   const PetscScalar      *x_array;
    685:   PC                     subpc;

    688:   /*
    689:       The VecPlaceArray() is to avoid having to copy the
    690:     y vector into the bjac->x vector. The reason for
    691:     the bjac->x vector is that we need a sequential vector
    692:     for the sequential solve.
    693:   */
    694:   VecGetArrayRead(x,&x_array);
    695:   VecGetArray(y,&y_array);
    696:   VecPlaceArray(bjac->x,x_array);
    697:   VecPlaceArray(bjac->y,y_array);
    698:   /* apply the symmetric left portion of the inner PC operator */
    699:   /* note this by-passes the inner KSP and its options completely */
    700:   KSPGetPC(jac->ksp[0],&subpc);
    701:   PCApplySymmetricLeft(subpc,bjac->x,bjac->y);
    702:   VecResetArray(bjac->x);
    703:   VecResetArray(bjac->y);
    704:   VecRestoreArrayRead(x,&x_array);
    705:   VecRestoreArray(y,&y_array);
    706:   return(0);
    707: }

    711: PetscErrorCode PCApplySymmetricRight_BJacobi_Singleblock(PC pc,Vec x,Vec y)
    712: {
    713:   PetscErrorCode         ierr;
    714:   PC_BJacobi             *jac  = (PC_BJacobi*)pc->data;
    715:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
    716:   PetscScalar            *y_array;
    717:   const PetscScalar      *x_array;
    718:   PC                     subpc;

    721:   /*
    722:       The VecPlaceArray() is to avoid having to copy the
    723:     y vector into the bjac->x vector. The reason for
    724:     the bjac->x vector is that we need a sequential vector
    725:     for the sequential solve.
    726:   */
    727:   VecGetArrayRead(x,&x_array);
    728:   VecGetArray(y,&y_array);
    729:   VecPlaceArray(bjac->x,x_array);
    730:   VecPlaceArray(bjac->y,y_array);

    732:   /* apply the symmetric right portion of the inner PC operator */
    733:   /* note this by-passes the inner KSP and its options completely */

    735:   KSPGetPC(jac->ksp[0],&subpc);
    736:   PCApplySymmetricRight(subpc,bjac->x,bjac->y);

    738:   VecRestoreArrayRead(x,&x_array);
    739:   VecRestoreArray(y,&y_array);
    740:   return(0);
    741: }

    745: PetscErrorCode PCApplyTranspose_BJacobi_Singleblock(PC pc,Vec x,Vec y)
    746: {
    747:   PetscErrorCode         ierr;
    748:   PC_BJacobi             *jac  = (PC_BJacobi*)pc->data;
    749:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
    750:   PetscScalar            *y_array;
    751:   const PetscScalar      *x_array;

    754:   /*
    755:       The VecPlaceArray() is to avoid having to copy the
    756:     y vector into the bjac->x vector. The reason for
    757:     the bjac->x vector is that we need a sequential vector
    758:     for the sequential solve.
    759:   */
    760:   VecGetArrayRead(x,&x_array);
    761:   VecGetArray(y,&y_array);
    762:   VecPlaceArray(bjac->x,x_array);
    763:   VecPlaceArray(bjac->y,y_array);
    764:   KSPSolveTranspose(jac->ksp[0],bjac->x,bjac->y);
    765:   VecResetArray(bjac->x);
    766:   VecResetArray(bjac->y);
    767:   VecRestoreArrayRead(x,&x_array);
    768:   VecRestoreArray(y,&y_array);
    769:   return(0);
    770: }

    774: static PetscErrorCode PCSetUp_BJacobi_Singleblock(PC pc,Mat mat,Mat pmat)
    775: {
    776:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
    777:   PetscErrorCode         ierr;
    778:   PetscInt               m;
    779:   KSP                    ksp;
    780:   PC_BJacobi_Singleblock *bjac;
    781:   PetscBool              wasSetup = PETSC_TRUE;

    784:   if (!pc->setupcalled) {
    785:     const char *prefix;

    787:     if (!jac->ksp) {
    788:       wasSetup = PETSC_FALSE;

    790:       KSPCreate(PETSC_COMM_SELF,&ksp);
    791:       KSPSetErrorIfNotConverged(ksp,pc->erroriffailure);
    792:       PetscObjectIncrementTabLevel((PetscObject)ksp,(PetscObject)pc,1);
    793:       PetscLogObjectParent((PetscObject)pc,(PetscObject)ksp);
    794:       KSPSetType(ksp,KSPPREONLY);
    795:       PCGetOptionsPrefix(pc,&prefix);
    796:       KSPSetOptionsPrefix(ksp,prefix);
    797:       KSPAppendOptionsPrefix(ksp,"sub_");

    799:       pc->ops->reset               = PCReset_BJacobi_Singleblock;
    800:       pc->ops->destroy             = PCDestroy_BJacobi_Singleblock;
    801:       pc->ops->apply               = PCApply_BJacobi_Singleblock;
    802:       pc->ops->applysymmetricleft  = PCApplySymmetricLeft_BJacobi_Singleblock;
    803:       pc->ops->applysymmetricright = PCApplySymmetricRight_BJacobi_Singleblock;
    804:       pc->ops->applytranspose      = PCApplyTranspose_BJacobi_Singleblock;
    805:       pc->ops->setuponblocks       = PCSetUpOnBlocks_BJacobi_Singleblock;

    807:       PetscMalloc1(1,&jac->ksp);
    808:       jac->ksp[0] = ksp;

    810:       PetscNewLog(pc,&bjac);
    811:       jac->data = (void*)bjac;
    812:     } else {
    813:       ksp  = jac->ksp[0];
    814:       bjac = (PC_BJacobi_Singleblock*)jac->data;
    815:     }

    817:     /*
    818:       The reason we need to generate these vectors is to serve
    819:       as the right-hand side and solution vector for the solve on the
    820:       block. We do not need to allocate space for the vectors since
    821:       that is provided via VecPlaceArray() just before the call to
    822:       KSPSolve() on the block.
    823:     */
    824:     MatGetSize(pmat,&m,&m);
    825:     VecCreateSeqWithArray(PETSC_COMM_SELF,1,m,NULL,&bjac->x);
    826:     VecCreateSeqWithArray(PETSC_COMM_SELF,1,m,NULL,&bjac->y);
    827: #ifdef PETSC_HAVE_CUSP
    828:     VecSetType(bjac->x,VECCUSP);
    829:     VecSetType(bjac->y,VECCUSP);
    830: #endif
    831:     PetscLogObjectParent((PetscObject)pc,(PetscObject)bjac->x);
    832:     PetscLogObjectParent((PetscObject)pc,(PetscObject)bjac->y);
    833:   } else {
    834:     ksp  = jac->ksp[0];
    835:     bjac = (PC_BJacobi_Singleblock*)jac->data;
    836:   }
    837:   if (pc->useAmat) {
    838:     KSPSetOperators(ksp,mat,pmat);
    839:   } else {
    840:     KSPSetOperators(ksp,pmat,pmat);
    841:   }
    842:   if (!wasSetup && pc->setfromoptionscalled) {
    843:     /* If PCSetFromOptions_BJacobi is called later, KSPSetFromOptions will be called at that time. */
    844:     KSPSetFromOptions(ksp);
    845:   }
    846:   return(0);
    847: }

    849: /* ---------------------------------------------------------------------------------------------*/
    852: PetscErrorCode PCReset_BJacobi_Multiblock(PC pc)
    853: {
    854:   PC_BJacobi            *jac  = (PC_BJacobi*)pc->data;
    855:   PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
    856:   PetscErrorCode        ierr;
    857:   PetscInt              i;

    860:   if (bjac && bjac->pmat) {
    861:     MatDestroyMatrices(jac->n_local,&bjac->pmat);
    862:     if (pc->useAmat) {
    863:       MatDestroyMatrices(jac->n_local,&bjac->mat);
    864:     }
    865:   }

    867:   for (i=0; i<jac->n_local; i++) {
    868:     KSPReset(jac->ksp[i]);
    869:     if (bjac && bjac->x) {
    870:       VecDestroy(&bjac->x[i]);
    871:       VecDestroy(&bjac->y[i]);
    872:       ISDestroy(&bjac->is[i]);
    873:     }
    874:   }
    875:   PetscFree(jac->l_lens);
    876:   PetscFree(jac->g_lens);
    877:   return(0);
    878: }

    882: PetscErrorCode PCDestroy_BJacobi_Multiblock(PC pc)
    883: {
    884:   PC_BJacobi            *jac  = (PC_BJacobi*)pc->data;
    885:   PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
    886:   PetscErrorCode        ierr;
    887:   PetscInt              i;

    890:   PCReset_BJacobi_Multiblock(pc);
    891:   if (bjac) {
    892:     PetscFree2(bjac->x,bjac->y);
    893:     PetscFree(bjac->starts);
    894:     PetscFree(bjac->is);
    895:   }
    896:   PetscFree(jac->data);
    897:   for (i=0; i<jac->n_local; i++) {
    898:     KSPDestroy(&jac->ksp[i]);
    899:   }
    900:   PetscFree(jac->ksp);
    901:   PetscFree(pc->data);
    902:   return(0);
    903: }

    907: PetscErrorCode PCSetUpOnBlocks_BJacobi_Multiblock(PC pc)
    908: {
    909:   PC_BJacobi     *jac = (PC_BJacobi*)pc->data;
    911:   PetscInt       i,n_local = jac->n_local;

    914:   for (i=0; i<n_local; i++) {
    915:     KSPSetUp(jac->ksp[i]);
    916:   }
    917:   return(0);
    918: }

    920: /*
    921:       Preconditioner for block Jacobi
    922: */
    925: PetscErrorCode PCApply_BJacobi_Multiblock(PC pc,Vec x,Vec y)
    926: {
    927:   PC_BJacobi            *jac = (PC_BJacobi*)pc->data;
    928:   PetscErrorCode        ierr;
    929:   PetscInt              i,n_local = jac->n_local;
    930:   PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
    931:   PetscScalar           *yin;
    932:   const PetscScalar     *xin;

    935:   VecGetArrayRead(x,&xin);
    936:   VecGetArray(y,&yin);
    937:   for (i=0; i<n_local; i++) {
    938:     /*
    939:        To avoid copying the subvector from x into a workspace we instead
    940:        make the workspace vector array point to the subpart of the array of
    941:        the global vector.
    942:     */
    943:     VecPlaceArray(bjac->x[i],xin+bjac->starts[i]);
    944:     VecPlaceArray(bjac->y[i],yin+bjac->starts[i]);

    946:     PetscLogEventBegin(PC_ApplyOnBlocks,jac->ksp[i],bjac->x[i],bjac->y[i],0);
    947:     KSPSolve(jac->ksp[i],bjac->x[i],bjac->y[i]);
    948:     PetscLogEventEnd(PC_ApplyOnBlocks,jac->ksp[i],bjac->x[i],bjac->y[i],0);

    950:     VecResetArray(bjac->x[i]);
    951:     VecResetArray(bjac->y[i]);
    952:   }
    953:   VecRestoreArrayRead(x,&xin);
    954:   VecRestoreArray(y,&yin);
    955:   return(0);
    956: }

    958: /*
    959:       Preconditioner for block Jacobi
    960: */
    963: PetscErrorCode PCApplyTranspose_BJacobi_Multiblock(PC pc,Vec x,Vec y)
    964: {
    965:   PC_BJacobi            *jac = (PC_BJacobi*)pc->data;
    966:   PetscErrorCode        ierr;
    967:   PetscInt              i,n_local = jac->n_local;
    968:   PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
    969:   PetscScalar           *yin;
    970:   const PetscScalar     *xin;

    973:   VecGetArrayRead(x,&xin);
    974:   VecGetArray(y,&yin);
    975:   for (i=0; i<n_local; i++) {
    976:     /*
    977:        To avoid copying the subvector from x into a workspace we instead
    978:        make the workspace vector array point to the subpart of the array of
    979:        the global vector.
    980:     */
    981:     VecPlaceArray(bjac->x[i],xin+bjac->starts[i]);
    982:     VecPlaceArray(bjac->y[i],yin+bjac->starts[i]);

    984:     PetscLogEventBegin(PC_ApplyTransposeOnBlocks,jac->ksp[i],bjac->x[i],bjac->y[i],0);
    985:     KSPSolveTranspose(jac->ksp[i],bjac->x[i],bjac->y[i]);
    986:     PetscLogEventEnd(PC_ApplyTransposeOnBlocks,jac->ksp[i],bjac->x[i],bjac->y[i],0);

    988:     VecResetArray(bjac->x[i]);
    989:     VecResetArray(bjac->y[i]);
    990:   }
    991:   VecRestoreArrayRead(x,&xin);
    992:   VecRestoreArray(y,&yin);
    993:   return(0);
    994: }

    998: static PetscErrorCode PCSetUp_BJacobi_Multiblock(PC pc,Mat mat,Mat pmat)
    999: {
    1000:   PC_BJacobi            *jac = (PC_BJacobi*)pc->data;
    1001:   PetscErrorCode        ierr;
    1002:   PetscInt              m,n_local,N,M,start,i;
    1003:   const char            *prefix,*pprefix,*mprefix;
    1004:   KSP                   ksp;
    1005:   Vec                   x,y;
    1006:   PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
    1007:   PC                    subpc;
    1008:   IS                    is;
    1009:   MatReuse              scall;

    1012:   MatGetLocalSize(pc->pmat,&M,&N);

    1014:   n_local = jac->n_local;

    1016:   if (pc->useAmat) {
    1017:     PetscBool same;
    1018:     PetscObjectTypeCompare((PetscObject)mat,((PetscObject)pmat)->type_name,&same);
    1019:     if (!same) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_INCOMP,"Matrices not of same type");
    1020:   }

    1022:   if (!pc->setupcalled) {
    1023:     scall = MAT_INITIAL_MATRIX;

    1025:     if (!jac->ksp) {
    1026:       pc->ops->reset         = PCReset_BJacobi_Multiblock;
    1027:       pc->ops->destroy       = PCDestroy_BJacobi_Multiblock;
    1028:       pc->ops->apply         = PCApply_BJacobi_Multiblock;
    1029:       pc->ops->applytranspose= PCApplyTranspose_BJacobi_Multiblock;
    1030:       pc->ops->setuponblocks = PCSetUpOnBlocks_BJacobi_Multiblock;

    1032:       PetscNewLog(pc,&bjac);
    1033:       PetscMalloc1(n_local,&jac->ksp);
    1034:       PetscLogObjectMemory((PetscObject)pc,sizeof(n_local*sizeof(KSP)));
    1035:       PetscMalloc2(n_local,&bjac->x,n_local,&bjac->y);
    1036:       PetscMalloc1(n_local,&bjac->starts);
    1037:       PetscLogObjectMemory((PetscObject)pc,sizeof(n_local*sizeof(PetscScalar)));

    1039:       jac->data = (void*)bjac;
    1040:       PetscMalloc1(n_local,&bjac->is);
    1041:       PetscLogObjectMemory((PetscObject)pc,sizeof(n_local*sizeof(IS)));

    1043:       for (i=0; i<n_local; i++) {
    1044:         KSPCreate(PETSC_COMM_SELF,&ksp);
    1045:         KSPSetErrorIfNotConverged(ksp,pc->erroriffailure);
    1046:         PetscObjectIncrementTabLevel((PetscObject)ksp,(PetscObject)pc,1);
    1047:         PetscLogObjectParent((PetscObject)pc,(PetscObject)ksp);
    1048:         KSPSetType(ksp,KSPPREONLY);
    1049:         KSPGetPC(ksp,&subpc);
    1050:         PCGetOptionsPrefix(pc,&prefix);
    1051:         KSPSetOptionsPrefix(ksp,prefix);
    1052:         KSPAppendOptionsPrefix(ksp,"sub_");

    1054:         jac->ksp[i] = ksp;
    1055:       }
    1056:     } else {
    1057:       bjac = (PC_BJacobi_Multiblock*)jac->data;
    1058:     }

    1060:     start = 0;
    1061:     for (i=0; i<n_local; i++) {
    1062:       m = jac->l_lens[i];
    1063:       /*
    1064:       The reason we need to generate these vectors is to serve
    1065:       as the right-hand side and solution vector for the solve on the
    1066:       block. We do not need to allocate space for the vectors since
    1067:       that is provided via VecPlaceArray() just before the call to
    1068:       KSPSolve() on the block.

    1070:       */
    1071:       VecCreateSeq(PETSC_COMM_SELF,m,&x);
    1072:       VecCreateSeqWithArray(PETSC_COMM_SELF,1,m,NULL,&y);
    1073: #ifdef PETSC_HAVE_CUSP
    1074:       VecSetType(x,VECCUSP);
    1075:       VecSetType(y,VECCUSP);
    1076: #endif
    1077:       PetscLogObjectParent((PetscObject)pc,(PetscObject)x);
    1078:       PetscLogObjectParent((PetscObject)pc,(PetscObject)y);

    1080:       bjac->x[i]      = x;
    1081:       bjac->y[i]      = y;
    1082:       bjac->starts[i] = start;

    1084:       ISCreateStride(PETSC_COMM_SELF,m,start,1,&is);
    1085:       bjac->is[i] = is;
    1086:       PetscLogObjectParent((PetscObject)pc,(PetscObject)is);

    1088:       start += m;
    1089:     }
    1090:   } else {
    1091:     bjac = (PC_BJacobi_Multiblock*)jac->data;
    1092:     /*
    1093:        Destroy the blocks from the previous iteration
    1094:     */
    1095:     if (pc->flag == DIFFERENT_NONZERO_PATTERN) {
    1096:       MatDestroyMatrices(n_local,&bjac->pmat);
    1097:       if (pc->useAmat) {
    1098:         MatDestroyMatrices(n_local,&bjac->mat);
    1099:       }
    1100:       scall = MAT_INITIAL_MATRIX;
    1101:     } else scall = MAT_REUSE_MATRIX;
    1102:   }

    1104:   MatGetSubMatrices(pmat,n_local,bjac->is,bjac->is,scall,&bjac->pmat);
    1105:   if (pc->useAmat) {
    1106:     PetscObjectGetOptionsPrefix((PetscObject)mat,&mprefix);
    1107:     MatGetSubMatrices(mat,n_local,bjac->is,bjac->is,scall,&bjac->mat);
    1108:   }
    1109:   /* Return control to the user so that the submatrices can be modified (e.g., to apply
    1110:      different boundary conditions for the submatrices than for the global problem) */
    1111:   PCModifySubMatrices(pc,n_local,bjac->is,bjac->is,bjac->pmat,pc->modifysubmatricesP);

    1113:   PetscObjectGetOptionsPrefix((PetscObject)pmat,&pprefix);
    1114:   for (i=0; i<n_local; i++) {
    1115:     PetscLogObjectParent((PetscObject)pc,(PetscObject)bjac->pmat[i]);
    1116:     PetscObjectSetOptionsPrefix((PetscObject)bjac->pmat[i],pprefix);
    1117:     if (pc->useAmat) {
    1118:       PetscLogObjectParent((PetscObject)pc,(PetscObject)bjac->mat[i]);
    1119:       PetscObjectSetOptionsPrefix((PetscObject)bjac->mat[i],mprefix);
    1120:       KSPSetOperators(jac->ksp[i],bjac->mat[i],bjac->pmat[i]);
    1121:     } else {
    1122:       KSPSetOperators(jac->ksp[i],bjac->pmat[i],bjac->pmat[i]);
    1123:     }
    1124:     if (pc->setfromoptionscalled) {
    1125:       KSPSetFromOptions(jac->ksp[i]);
    1126:     }
    1127:   }
    1128:   return(0);
    1129: }

    1131: /* ---------------------------------------------------------------------------------------------*/
    1132: /*
    1133:       These are for a single block with multiple processes;
    1134: */
    1137: static PetscErrorCode PCReset_BJacobi_Multiproc(PC pc)
    1138: {
    1139:   PC_BJacobi           *jac   = (PC_BJacobi*)pc->data;
    1140:   PC_BJacobi_Multiproc *mpjac = (PC_BJacobi_Multiproc*)jac->data;
    1141:   PetscErrorCode       ierr;

    1144:   VecDestroy(&mpjac->ysub);
    1145:   VecDestroy(&mpjac->xsub);
    1146:   MatDestroy(&mpjac->submats);
    1147:   if (jac->ksp) {KSPReset(jac->ksp[0]);}
    1148:   return(0);
    1149: }

    1153: static PetscErrorCode PCDestroy_BJacobi_Multiproc(PC pc)
    1154: {
    1155:   PC_BJacobi           *jac   = (PC_BJacobi*)pc->data;
    1156:   PC_BJacobi_Multiproc *mpjac = (PC_BJacobi_Multiproc*)jac->data;
    1157:   PetscErrorCode       ierr;

    1160:   PCReset_BJacobi_Multiproc(pc);
    1161:   KSPDestroy(&jac->ksp[0]);
    1162:   PetscFree(jac->ksp);
    1163:   PetscSubcommDestroy(&mpjac->psubcomm);

    1165:   PetscFree(mpjac);
    1166:   PetscFree(pc->data);
    1167:   return(0);
    1168: }

    1172: static PetscErrorCode PCApply_BJacobi_Multiproc(PC pc,Vec x,Vec y)
    1173: {
    1174:   PC_BJacobi           *jac   = (PC_BJacobi*)pc->data;
    1175:   PC_BJacobi_Multiproc *mpjac = (PC_BJacobi_Multiproc*)jac->data;
    1176:   PetscErrorCode       ierr;
    1177:   PetscScalar          *yarray;
    1178:   const PetscScalar    *xarray;

    1181:   /* place x's and y's local arrays into xsub and ysub */
    1182:   VecGetArrayRead(x,&xarray);
    1183:   VecGetArray(y,&yarray);
    1184:   VecPlaceArray(mpjac->xsub,xarray);
    1185:   VecPlaceArray(mpjac->ysub,yarray);

    1187:   /* apply preconditioner on each matrix block */
    1188:   PetscLogEventBegin(PC_ApplyOnMproc,jac->ksp[0],mpjac->xsub,mpjac->ysub,0);
    1189:   KSPSolve(jac->ksp[0],mpjac->xsub,mpjac->ysub);
    1190:   PetscLogEventEnd(PC_ApplyOnMproc,jac->ksp[0],mpjac->xsub,mpjac->ysub,0);

    1192:   VecResetArray(mpjac->xsub);
    1193:   VecResetArray(mpjac->ysub);
    1194:   VecRestoreArrayRead(x,&xarray);
    1195:   VecRestoreArray(y,&yarray);
    1196:   return(0);
    1197: }

    1199: #include <petsc/private/matimpl.h>
    1202: static PetscErrorCode PCSetUp_BJacobi_Multiproc(PC pc)
    1203: {
    1204:   PC_BJacobi           *jac   = (PC_BJacobi*)pc->data;
    1205:   PC_BJacobi_Multiproc *mpjac = (PC_BJacobi_Multiproc*)jac->data;
    1206:   PetscErrorCode       ierr;
    1207:   PetscInt             m,n;
    1208:   MPI_Comm             comm,subcomm=0;
    1209:   const char           *prefix;
    1210:   PetscBool            wasSetup = PETSC_TRUE;

    1213:   PetscObjectGetComm((PetscObject)pc,&comm);
    1214:   if (jac->n_local > 1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Only a single block in a subcommunicator is supported");
    1215:   jac->n_local = 1; /* currently only a single block is supported for a subcommunicator */
    1216:   if (!pc->setupcalled) {
    1217:     wasSetup  = PETSC_FALSE;
    1218:     PetscNewLog(pc,&mpjac);
    1219:     jac->data = (void*)mpjac;

    1221:     /* initialize datastructure mpjac */
    1222:     if (!jac->psubcomm) {
    1223:       /* Create default contiguous subcommunicatiors if user does not provide them */
    1224:       PetscSubcommCreate(comm,&jac->psubcomm);
    1225:       PetscSubcommSetNumber(jac->psubcomm,jac->n);
    1226:       PetscSubcommSetType(jac->psubcomm,PETSC_SUBCOMM_CONTIGUOUS);
    1227:       PetscLogObjectMemory((PetscObject)pc,sizeof(PetscSubcomm));
    1228:     }
    1229:     mpjac->psubcomm = jac->psubcomm;
    1230:     subcomm         = PetscSubcommChild(mpjac->psubcomm);

    1232:     /* Get matrix blocks of pmat */
    1233:     if (!pc->pmat->ops->getmultiprocblock) SETERRQ(PetscObjectComm((PetscObject)pc->pmat),PETSC_ERR_SUP,"No support for the requested operation");
    1234:     (*pc->pmat->ops->getmultiprocblock)(pc->pmat,subcomm,MAT_INITIAL_MATRIX,&mpjac->submats);

    1236:     /* create a new PC that processors in each subcomm have copy of */
    1237:     PetscMalloc1(1,&jac->ksp);
    1238:     KSPCreate(subcomm,&jac->ksp[0]);
    1239:     KSPSetErrorIfNotConverged(jac->ksp[0],pc->erroriffailure);
    1240:     PetscObjectIncrementTabLevel((PetscObject)jac->ksp[0],(PetscObject)pc,1);
    1241:     PetscLogObjectParent((PetscObject)pc,(PetscObject)jac->ksp[0]);
    1242:     KSPSetOperators(jac->ksp[0],mpjac->submats,mpjac->submats);
    1243:     KSPGetPC(jac->ksp[0],&mpjac->pc);

    1245:     PCGetOptionsPrefix(pc,&prefix);
    1246:     KSPSetOptionsPrefix(jac->ksp[0],prefix);
    1247:     KSPAppendOptionsPrefix(jac->ksp[0],"sub_");
    1248:     /*
    1249:       PetscMPIInt rank,subsize,subrank;
    1250:       MPI_Comm_rank(comm,&rank);
    1251:       MPI_Comm_size(subcomm,&subsize);
    1252:       MPI_Comm_rank(subcomm,&subrank);

    1254:       MatGetLocalSize(mpjac->submats,&m,NULL);
    1255:       MatGetSize(mpjac->submats,&n,NULL);
    1256:       PetscSynchronizedPrintf(comm,"[%d], sub-size %d,sub-rank %d\n",rank,subsize,subrank);
    1257:       PetscSynchronizedFlush(comm,PETSC_STDOUT);
    1258:     */

    1260:     /* create dummy vectors xsub and ysub */
    1261:     MatGetLocalSize(mpjac->submats,&m,&n);
    1262:     VecCreateMPIWithArray(subcomm,1,n,PETSC_DECIDE,NULL,&mpjac->xsub);
    1263:     VecCreateMPIWithArray(subcomm,1,m,PETSC_DECIDE,NULL,&mpjac->ysub);
    1264: #ifdef PETSC_HAVE_CUSP
    1265:     VecSetType(mpjac->xsub,VECMPICUSP);
    1266:     VecSetType(mpjac->ysub,VECMPICUSP);
    1267: #endif
    1268:     PetscLogObjectParent((PetscObject)pc,(PetscObject)mpjac->xsub);
    1269:     PetscLogObjectParent((PetscObject)pc,(PetscObject)mpjac->ysub);

    1271:     pc->ops->reset   = PCReset_BJacobi_Multiproc;
    1272:     pc->ops->destroy = PCDestroy_BJacobi_Multiproc;
    1273:     pc->ops->apply   = PCApply_BJacobi_Multiproc;
    1274:   } else { /* pc->setupcalled */
    1275:     subcomm = PetscSubcommChild(mpjac->psubcomm);
    1276:     if (pc->flag == DIFFERENT_NONZERO_PATTERN) {
    1277:       /* destroy old matrix blocks, then get new matrix blocks */
    1278:       if (mpjac->submats) {MatDestroy(&mpjac->submats);}
    1279:       (*pc->pmat->ops->getmultiprocblock)(pc->pmat,subcomm,MAT_INITIAL_MATRIX,&mpjac->submats);
    1280:     } else {
    1281:       (*pc->pmat->ops->getmultiprocblock)(pc->pmat,subcomm,MAT_REUSE_MATRIX,&mpjac->submats);
    1282:     }
    1283:     KSPSetOperators(jac->ksp[0],mpjac->submats,mpjac->submats);
    1284:   }

    1286:   if (!wasSetup && pc->setfromoptionscalled) {
    1287:     KSPSetFromOptions(jac->ksp[0]);
    1288:   }
    1289:   return(0);
    1290: }