Date: 2013-07-02T14:46:36+00:00

Actual source code: bjacobi.c
petsc-3.4.2 2013-07-02

      2: /*
      3:    Defines a block Jacobi preconditioner.
      4: */
      5: #include <petsc-private/pcimpl.h>              /*I "petscpc.h" I*/
      6: #include <../src/ksp/pc/impls/bjacobi/bjacobi.h>

      8: static PetscErrorCode PCSetUp_BJacobi_Singleblock(PC,Mat,Mat);
      9: static PetscErrorCode PCSetUp_BJacobi_Multiblock(PC,Mat,Mat);
     10: static PetscErrorCode PCSetUp_BJacobi_Multiproc(PC);

     14: static PetscErrorCode PCSetUp_BJacobi(PC pc)
     15: {
     16:   PC_BJacobi     *jac = (PC_BJacobi*)pc->data;
     17:   Mat            mat  = pc->mat,pmat = pc->pmat;
     18:   PetscErrorCode ierr,(*f)(Mat,Mat*);
     19:   PetscInt       N,M,start,i,sum,end;
     20:   PetscInt       bs,i_start=-1,i_end=-1;
     21:   PetscMPIInt    rank,size;
     22:   const char     *pprefix,*mprefix;

     25:   MPI_Comm_rank(PetscObjectComm((PetscObject)pc),&rank);
     26:   MPI_Comm_size(PetscObjectComm((PetscObject)pc),&size);
     27:   MatGetLocalSize(pc->pmat,&M,&N);
     28:   MatGetBlockSize(pc->pmat,&bs);

     30:   if (jac->n > 0 && jac->n < size) {
     31:     PCSetUp_BJacobi_Multiproc(pc);
     32:     return(0);
     33:   }

     35:   /* --------------------------------------------------------------------------
     36:       Determines the number of blocks assigned to each processor
     37:   -----------------------------------------------------------------------------*/

     39:   /*   local block count  given */
     40:   if (jac->n_local > 0 && jac->n < 0) {
     41:     MPI_Allreduce(&jac->n_local,&jac->n,1,MPIU_INT,MPI_SUM,PetscObjectComm((PetscObject)pc));
     42:     if (jac->l_lens) { /* check that user set these correctly */
     43:       sum = 0;
     44:       for (i=0; i<jac->n_local; i++) {
     45:         if (jac->l_lens[i]/bs*bs !=jac->l_lens[i]) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"Mat blocksize doesn't match block Jacobi layout");
     46:         sum += jac->l_lens[i];
     47:       }
     48:       if (sum != M) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"Local lens set incorrectly");
     49:     } else {
     50:       PetscMalloc(jac->n_local*sizeof(PetscInt),&jac->l_lens);
     51:       for (i=0; i<jac->n_local; i++) jac->l_lens[i] = bs*((M/bs)/jac->n_local + (((M/bs) % jac->n_local) > i));
     52:     }
     53:   } else if (jac->n > 0 && jac->n_local < 0) { /* global block count given */
     54:     /* global blocks given: determine which ones are local */
     55:     if (jac->g_lens) {
     56:       /* check if the g_lens is has valid entries */
     57:       for (i=0; i<jac->n; i++) {
     58:         if (!jac->g_lens[i]) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"Zero block not allowed");
     59:         if (jac->g_lens[i]/bs*bs != jac->g_lens[i]) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"Mat blocksize doesn't match block Jacobi layout");
     60:       }
     61:       if (size == 1) {
     62:         jac->n_local = jac->n;
     63:         PetscMalloc(jac->n_local*sizeof(PetscInt),&jac->l_lens);
     64:         PetscMemcpy(jac->l_lens,jac->g_lens,jac->n_local*sizeof(PetscInt));
     65:         /* check that user set these correctly */
     66:         sum = 0;
     67:         for (i=0; i<jac->n_local; i++) sum += jac->l_lens[i];
     68:         if (sum != M) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"Global lens set incorrectly");
     69:       } else {
     70:         MatGetOwnershipRange(pc->pmat,&start,&end);
     71:         /* loop over blocks determing first one owned by me */
     72:         sum = 0;
     73:         for (i=0; i<jac->n+1; i++) {
     74:           if (sum == start) { i_start = i; goto start_1;}
     75:           if (i < jac->n) sum += jac->g_lens[i];
     76:         }
     77:         SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"Block sizes used in PCBJacobiSetTotalBlocks()\nare not compatible with parallel matrix layout");
     78: start_1:
     79:         for (i=i_start; i<jac->n+1; i++) {
     80:           if (sum == end) { i_end = i; goto end_1; }
     81:           if (i < jac->n) sum += jac->g_lens[i];
     82:         }
     83:         SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"Block sizes used in PCBJacobiSetTotalBlocks()\nare not compatible with parallel matrix layout");
     84: end_1:
     85:         jac->n_local = i_end - i_start;
     86:         PetscMalloc(jac->n_local*sizeof(PetscInt),&jac->l_lens);
     87:         PetscMemcpy(jac->l_lens,jac->g_lens+i_start,jac->n_local*sizeof(PetscInt));
     88:       }
     89:     } else { /* no global blocks given, determine then using default layout */
     90:       jac->n_local = jac->n/size + ((jac->n % size) > rank);
     91:       PetscMalloc(jac->n_local*sizeof(PetscInt),&jac->l_lens);
     92:       for (i=0; i<jac->n_local; i++) {
     93:         jac->l_lens[i] = ((M/bs)/jac->n_local + (((M/bs) % jac->n_local) > i))*bs;
     94:         if (!jac->l_lens[i]) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"Too many blocks given");
     95:       }
     96:     }
     97:   } else if (jac->n < 0 && jac->n_local < 0) { /* no blocks given */
     98:     jac->n         = size;
     99:     jac->n_local   = 1;
    100:     PetscMalloc(sizeof(PetscInt),&jac->l_lens);
    101:     jac->l_lens[0] = M;
    102:   } else { /* jac->n > 0 && jac->n_local > 0 */
    103:     if (!jac->l_lens) {
    104:       PetscMalloc(jac->n_local*sizeof(PetscInt),&jac->l_lens);
    105:       for (i=0; i<jac->n_local; i++) jac->l_lens[i] = bs*((M/bs)/jac->n_local + (((M/bs) % jac->n_local) > i));
    106:     }
    107:   }
    108:   if (jac->n_local < 1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Number of blocks is less than number of processors");

    110:   /* -------------------------
    111:       Determines mat and pmat
    112:   ---------------------------*/
    113:   PetscObjectQueryFunction((PetscObject)pc->mat,"MatGetDiagonalBlock_C",&f);
    114:   if (!f && size == 1) {
    115:     mat  = pc->mat;
    116:     pmat = pc->pmat;
    117:   } else {
    118:     if (pc->useAmat) {
    119:       /* use block from Amat matrix, not Pmat for local MatMult() */
    120:       MatGetDiagonalBlock(pc->mat,&mat);
    121:       /* make submatrix have same prefix as entire matrix */
    122:       PetscObjectGetOptionsPrefix((PetscObject)pc->mat,&mprefix);
    123:       PetscObjectSetOptionsPrefix((PetscObject)mat,mprefix);
    124:     }
    125:     if (pc->pmat != pc->mat || !pc->useAmat) {
    126:       MatGetDiagonalBlock(pc->pmat,&pmat);
    127:       /* make submatrix have same prefix as entire matrix */
    128:       PetscObjectGetOptionsPrefix((PetscObject)pc->pmat,&pprefix);
    129:       PetscObjectSetOptionsPrefix((PetscObject)pmat,pprefix);
    130:     } else pmat = mat;
    131:   }

    133:   /* ------
    134:      Setup code depends on the number of blocks
    135:   */
    136:   if (jac->n_local == 1) {
    137:     PCSetUp_BJacobi_Singleblock(pc,mat,pmat);
    138:   } else {
    139:     PCSetUp_BJacobi_Multiblock(pc,mat,pmat);
    140:   }
    141:   return(0);
    142: }

    144: /* Default destroy, if it has never been setup */
    147: static PetscErrorCode PCDestroy_BJacobi(PC pc)
    148: {
    149:   PC_BJacobi     *jac = (PC_BJacobi*)pc->data;

    153:   PetscFree(jac->g_lens);
    154:   PetscFree(jac->l_lens);
    155:   PetscFree(pc->data);
    156:   return(0);
    157: }

    162: static PetscErrorCode PCSetFromOptions_BJacobi(PC pc)
    163: {
    164:   PC_BJacobi     *jac = (PC_BJacobi*)pc->data;
    166:   PetscInt       blocks;
    167:   PetscBool      flg;

    170:   PetscOptionsHead("Block Jacobi options");
    171:   PetscOptionsInt("-pc_bjacobi_blocks","Total number of blocks","PCBJacobiSetTotalBlocks",jac->n,&blocks,&flg);
    172:   if (flg) {
    173:     PCBJacobiSetTotalBlocks(pc,blocks,NULL);
    174:   }
    175:   PetscOptionsTail();
    176:   return(0);
    177: }

    179: #include <petscdraw.h>
    182: static PetscErrorCode PCView_BJacobi(PC pc,PetscViewer viewer)
    183: {
    184:   PC_BJacobi           *jac   = (PC_BJacobi*)pc->data;
    185:   PC_BJacobi_Multiproc *mpjac = (PC_BJacobi_Multiproc*)jac->data;
    186:   PetscErrorCode       ierr;
    187:   PetscMPIInt          rank;
    188:   PetscInt             i;
    189:   PetscBool            iascii,isstring,isdraw;
    190:   PetscViewer          sviewer;

    193:   PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERASCII,&iascii);
    194:   PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERSTRING,&isstring);
    195:   PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERDRAW,&isdraw);
    196:   if (iascii) {
    197:     if (pc->useAmat) {
    198:       PetscViewerASCIIPrintf(viewer,"  block Jacobi: using Amat local matrix, number of blocks = %D\n",jac->n);
    199:     }
    200:     PetscViewerASCIIPrintf(viewer,"  block Jacobi: number of blocks = %D\n",jac->n);
    201:     MPI_Comm_rank(PetscObjectComm((PetscObject)pc),&rank);
    202:     if (jac->same_local_solves) {
    203:       PetscViewerASCIIPrintf(viewer,"  Local solve is same for all blocks, in the following KSP and PC objects:\n");
    204:       if (jac->ksp && !jac->psubcomm) {
    205:         PetscViewerGetSingleton(viewer,&sviewer);
    206:         if (!rank) {
    207:           PetscViewerASCIIPushTab(viewer);
    208:           KSPView(jac->ksp[0],sviewer);
    209:           PetscViewerASCIIPopTab(viewer);
    210:         }
    211:         PetscViewerRestoreSingleton(viewer,&sviewer);
    212:       } else if (jac->psubcomm && !jac->psubcomm->color) {
    213:         PetscViewerASCIIGetStdout(mpjac->psubcomm->comm,&sviewer);
    214:         PetscViewerASCIIPushTab(viewer);
    215:         KSPView(*(jac->ksp),sviewer);
    216:         PetscViewerASCIIPopTab(viewer);
    217:       }
    218:     } else {
    219:       PetscInt n_global;
    220:       MPI_Allreduce(&jac->n_local,&n_global,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)pc));
    221:       PetscViewerASCIISynchronizedAllow(viewer,PETSC_TRUE);
    222:       PetscViewerASCIIPrintf(viewer,"  Local solve info for each block is in the following KSP and PC objects:\n");
    223:       PetscViewerASCIISynchronizedPrintf(viewer,"[%d] number of local blocks = %D, first local block number = %D\n",
    224:                                                 rank,jac->n_local,jac->first_local);
    225:       PetscViewerASCIIPushTab(viewer);
    226:       for (i=0; i<n_global; i++) {
    227:         PetscViewerGetSingleton(viewer,&sviewer);
    228:         if (i < jac->n_local) {
    229:           PetscViewerASCIISynchronizedPrintf(viewer,"[%d] local block number %D\n",rank,i);
    230:           KSPView(jac->ksp[i],sviewer);
    231:           PetscViewerASCIISynchronizedPrintf(viewer,"- - - - - - - - - - - - - - - - - -\n");
    232:         }
    233:         PetscViewerRestoreSingleton(viewer,&sviewer);
    234:       }
    235:       PetscViewerASCIIPopTab(viewer);
    236:       PetscViewerFlush(viewer);
    237:       PetscViewerASCIISynchronizedAllow(viewer,PETSC_FALSE);
    238:     }
    239:   } else if (isstring) {
    240:     PetscViewerStringSPrintf(viewer," blks=%D",jac->n);
    241:     PetscViewerGetSingleton(viewer,&sviewer);
    242:     if (jac->ksp) {KSPView(jac->ksp[0],sviewer);}
    243:     PetscViewerRestoreSingleton(viewer,&sviewer);
    244:   } else if (isdraw) {
    245:     PetscDraw draw;
    246:     char      str[25];
    247:     PetscReal x,y,bottom,h;

    249:     PetscViewerDrawGetDraw(viewer,0,&draw);
    250:     PetscDrawGetCurrentPoint(draw,&x,&y);
    251:     PetscSNPrintf(str,25,"Number blocks %D",jac->n);
    252:     PetscDrawBoxedString(draw,x,y,PETSC_DRAW_RED,PETSC_DRAW_BLACK,str,NULL,&h);
    253:     bottom = y - h;
    254:     PetscDrawPushCurrentPoint(draw,x,bottom);
    255:     /* warning the communicator on viewer is different then on ksp in parallel */
    256:     if (jac->ksp) {KSPView(jac->ksp[0],viewer);}
    257:     PetscDrawPopCurrentPoint(draw);
    258:   }
    259:   return(0);
    260: }

    262: /* -------------------------------------------------------------------------------------*/

    266: static PetscErrorCode  PCBJacobiGetSubKSP_BJacobi(PC pc,PetscInt *n_local,PetscInt *first_local,KSP **ksp)
    267: {
    268:   PC_BJacobi *jac = (PC_BJacobi*)pc->data;;

    271:   if (!pc->setupcalled) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_WRONGSTATE,"Must call KSPSetUp() or PCSetUp() first");

    273:   if (n_local) *n_local = jac->n_local;
    274:   if (first_local) *first_local = jac->first_local;
    275:   *ksp                   = jac->ksp;
    276:   jac->same_local_solves = PETSC_FALSE;        /* Assume that local solves are now different;
    277:                                                   not necessarily true though!  This flag is
    278:                                                   used only for PCView_BJacobi() */
    279:   return(0);
    280: }

    284: static PetscErrorCode  PCBJacobiSetTotalBlocks_BJacobi(PC pc,PetscInt blocks,PetscInt *lens)
    285: {
    286:   PC_BJacobi     *jac = (PC_BJacobi*)pc->data;

    290:   if (pc->setupcalled > 0 && jac->n!=blocks) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_ORDER,"Cannot alter number of blocks after PCSetUp()/KSPSetUp() has been called");
    291:   jac->n = blocks;
    292:   if (!lens) jac->g_lens = 0;
    293:   else {
    294:     PetscMalloc(blocks*sizeof(PetscInt),&jac->g_lens);
    295:     PetscLogObjectMemory(pc,blocks*sizeof(PetscInt));
    296:     PetscMemcpy(jac->g_lens,lens,blocks*sizeof(PetscInt));
    297:   }
    298:   return(0);
    299: }

    303: static PetscErrorCode  PCBJacobiGetTotalBlocks_BJacobi(PC pc, PetscInt *blocks, const PetscInt *lens[])
    304: {
    305:   PC_BJacobi *jac = (PC_BJacobi*) pc->data;

    308:   *blocks = jac->n;
    309:   if (lens) *lens = jac->g_lens;
    310:   return(0);
    311: }

    315: static PetscErrorCode  PCBJacobiSetLocalBlocks_BJacobi(PC pc,PetscInt blocks,const PetscInt lens[])
    316: {
    317:   PC_BJacobi     *jac;

    321:   jac = (PC_BJacobi*)pc->data;

    323:   jac->n_local = blocks;
    324:   if (!lens) jac->l_lens = 0;
    325:   else {
    326:     PetscMalloc(blocks*sizeof(PetscInt),&jac->l_lens);
    327:     PetscLogObjectMemory(pc,blocks*sizeof(PetscInt));
    328:     PetscMemcpy(jac->l_lens,lens,blocks*sizeof(PetscInt));
    329:   }
    330:   return(0);
    331: }

    335: static PetscErrorCode  PCBJacobiGetLocalBlocks_BJacobi(PC pc, PetscInt *blocks, const PetscInt *lens[])
    336: {
    337:   PC_BJacobi *jac = (PC_BJacobi*) pc->data;

    340:   *blocks = jac->n_local;
    341:   if (lens) *lens = jac->l_lens;
    342:   return(0);
    343: }

    345: /* -------------------------------------------------------------------------------------*/

    349: /*@C
    350:    PCBJacobiGetSubKSP - Gets the local KSP contexts for all blocks on
    351:    this processor.

    353:    Note Collective

    355:    Input Parameter:
    356: .  pc - the preconditioner context

    358:    Output Parameters:
    359: +  n_local - the number of blocks on this processor, or NULL
    360: .  first_local - the global number of the first block on this processor, or NULL
    361: -  ksp - the array of KSP contexts

    363:    Notes:
    364:    After PCBJacobiGetSubKSP() the array of KSP contexts is not to be freed.

    366:    Currently for some matrix implementations only 1 block per processor
    367:    is supported.

    369:    You must call KSPSetUp() or PCSetUp() before calling PCBJacobiGetSubKSP().

    371:    Fortran Usage: You must pass in a KSP array that is large enough to contain all the local KSPs.
    372:       You can call PCBJacobiGetSubKSP(pc,nlocal,firstlocal,NULL_OBJECT,ierr) to determine how large the
    373:       KSP array must be.

    375:    Level: advanced

    377: .keywords:  block, Jacobi, get, sub, KSP, context

    379: .seealso: PCBJacobiGetSubKSP()
    380: @*/
    381: PetscErrorCode  PCBJacobiGetSubKSP(PC pc,PetscInt *n_local,PetscInt *first_local,KSP *ksp[])
    382: {

    387:   PetscUseMethod(pc,"PCBJacobiGetSubKSP_C",(PC,PetscInt*,PetscInt*,KSP **),(pc,n_local,first_local,ksp));
    388:   return(0);
    389: }

    393: /*@
    394:    PCBJacobiSetTotalBlocks - Sets the global number of blocks for the block
    395:    Jacobi preconditioner.

    397:    Collective on PC

    399:    Input Parameters:
    400: +  pc - the preconditioner context
    401: .  blocks - the number of blocks
    402: -  lens - [optional] integer array containing the size of each block

    404:    Options Database Key:
    405: .  -pc_bjacobi_blocks <blocks> - Sets the number of global blocks

    407:    Notes:
    408:    Currently only a limited number of blocking configurations are supported.
    409:    All processors sharing the PC must call this routine with the same data.

    411:    Level: intermediate

    413: .keywords:  set, number, Jacobi, global, total, blocks

    415: .seealso: PCSetUseAmat(), PCBJacobiSetLocalBlocks()
    416: @*/
    417: PetscErrorCode  PCBJacobiSetTotalBlocks(PC pc,PetscInt blocks,const PetscInt lens[])
    418: {

    423:   if (blocks <= 0) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_OUTOFRANGE,"Must have positive blocks");
    424:   PetscTryMethod(pc,"PCBJacobiSetTotalBlocks_C",(PC,PetscInt,const PetscInt[]),(pc,blocks,lens));
    425:   return(0);
    426: }

    430: /*@C
    431:    PCBJacobiGetTotalBlocks - Gets the global number of blocks for the block
    432:    Jacobi preconditioner.

    434:    Not Collective

    436:    Input Parameter:
    437: .  pc - the preconditioner context

    439:    Output parameters:
    440: +  blocks - the number of blocks
    441: -  lens - integer array containing the size of each block

    443:    Level: intermediate

    445: .keywords:  get, number, Jacobi, global, total, blocks

    447: .seealso: PCSetUseAmat(), PCBJacobiGetLocalBlocks()
    448: @*/
    449: PetscErrorCode  PCBJacobiGetTotalBlocks(PC pc, PetscInt *blocks, const PetscInt *lens[])
    450: {

    456:   PetscUseMethod(pc,"PCBJacobiGetTotalBlocks_C",(PC,PetscInt*, const PetscInt *[]),(pc,blocks,lens));
    457:   return(0);
    458: }

    462: /*@
    463:    PCBJacobiSetLocalBlocks - Sets the local number of blocks for the block
    464:    Jacobi preconditioner.

    466:    Not Collective

    468:    Input Parameters:
    469: +  pc - the preconditioner context
    470: .  blocks - the number of blocks
    471: -  lens - [optional] integer array containing size of each block

    473:    Note:
    474:    Currently only a limited number of blocking configurations are supported.

    476:    Level: intermediate

    478: .keywords: PC, set, number, Jacobi, local, blocks

    480: .seealso: PCSetUseAmat(), PCBJacobiSetTotalBlocks()
    481: @*/
    482: PetscErrorCode  PCBJacobiSetLocalBlocks(PC pc,PetscInt blocks,const PetscInt lens[])
    483: {

    488:   if (blocks < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Must have nonegative blocks");
    489:   PetscTryMethod(pc,"PCBJacobiSetLocalBlocks_C",(PC,PetscInt,const PetscInt []),(pc,blocks,lens));
    490:   return(0);
    491: }

    495: /*@C
    496:    PCBJacobiGetLocalBlocks - Gets the local number of blocks for the block
    497:    Jacobi preconditioner.

    499:    Not Collective

    501:    Input Parameters:
    502: +  pc - the preconditioner context
    503: .  blocks - the number of blocks
    504: -  lens - [optional] integer array containing size of each block

    506:    Note:
    507:    Currently only a limited number of blocking configurations are supported.

    509:    Level: intermediate

    511: .keywords: PC, get, number, Jacobi, local, blocks

    513: .seealso: PCSetUseAmat(), PCBJacobiGetTotalBlocks()
    514: @*/
    515: PetscErrorCode  PCBJacobiGetLocalBlocks(PC pc, PetscInt *blocks, const PetscInt *lens[])
    516: {

    522:   PetscUseMethod(pc,"PCBJacobiGetLocalBlocks_C",(PC,PetscInt*, const PetscInt *[]),(pc,blocks,lens));
    523:   return(0);
    524: }

    526: /* -----------------------------------------------------------------------------------*/

    528: /*MC
    529:    PCBJACOBI - Use block Jacobi preconditioning, each block is (approximately) solved with
    530:            its own KSP object.

    532:    Options Database Keys:
    533: .  -pc_use_amat - use Amat to apply block of operator in inner Krylov method

    535:    Notes: Each processor can have one or more blocks, but a block cannot be shared by more
    536:      than one processor. Defaults to one block per processor.

    538:      To set options on the solvers for each block append -sub_ to all the KSP, KSP, and PC
    539:         options database keys. For example, -sub_pc_type ilu -sub_pc_factor_levels 1 -sub_ksp_type preonly

    541:      To set the options on the solvers separate for each block call PCBJacobiGetSubKSP()
    542:          and set the options directly on the resulting KSP object (you can access its PC
    543:          KSPGetPC())

    545:    Level: beginner

    547:    Concepts: block Jacobi

    549:    Developer Notes: This preconditioner does not currently work with CUDA/CUSP for a couple of reasons.
    550:        (1) It creates seq vectors as work vectors that should be cusp
    551:        (2) The use of VecPlaceArray() is not handled properly by CUSP (that is it will not know where
    552:            the ownership of the vector is so may use wrong values) even if it did know the ownership
    553:            it may induce extra copy ups and downs. Satish suggests a VecTransplantArray() to handle two
    554:            vectors sharing the same pointer and handling the CUSP side as well instead of VecGetArray()/VecPlaceArray().

    557: .seealso:  PCCreate(), PCSetType(), PCType (for list of available types), PC,
    558:            PCASM, PCSetUseAmat(), PCGetUseAmat(), PCBJacobiGetSubKSP(), PCBJacobiSetTotalBlocks(),
    559:            PCBJacobiSetLocalBlocks(), PCSetModifySubmatrices()
    560: M*/

    564: PETSC_EXTERN PetscErrorCode PCCreate_BJacobi(PC pc)
    565: {
    567:   PetscMPIInt    rank;
    568:   PC_BJacobi     *jac;

    571:   PetscNewLog(pc,PC_BJacobi,&jac);
    572:   MPI_Comm_rank(PetscObjectComm((PetscObject)pc),&rank);

    574:   pc->ops->apply           = 0;
    575:   pc->ops->applytranspose  = 0;
    576:   pc->ops->setup           = PCSetUp_BJacobi;
    577:   pc->ops->destroy         = PCDestroy_BJacobi;
    578:   pc->ops->setfromoptions  = PCSetFromOptions_BJacobi;
    579:   pc->ops->view            = PCView_BJacobi;
    580:   pc->ops->applyrichardson = 0;

    582:   pc->data               = (void*)jac;
    583:   jac->n                 = -1;
    584:   jac->n_local           = -1;
    585:   jac->first_local       = rank;
    586:   jac->ksp               = 0;
    587:   jac->same_local_solves = PETSC_TRUE;
    588:   jac->g_lens            = 0;
    589:   jac->l_lens            = 0;
    590:   jac->psubcomm          = 0;

    592:   PetscObjectComposeFunction((PetscObject)pc,"PCBJacobiGetSubKSP_C",PCBJacobiGetSubKSP_BJacobi);
    593:   PetscObjectComposeFunction((PetscObject)pc,"PCBJacobiSetTotalBlocks_C",PCBJacobiSetTotalBlocks_BJacobi);
    594:   PetscObjectComposeFunction((PetscObject)pc,"PCBJacobiGetTotalBlocks_C",PCBJacobiGetTotalBlocks_BJacobi);
    595:   PetscObjectComposeFunction((PetscObject)pc,"PCBJacobiSetLocalBlocks_C",PCBJacobiSetLocalBlocks_BJacobi);
    596:   PetscObjectComposeFunction((PetscObject)pc,"PCBJacobiGetLocalBlocks_C",PCBJacobiGetLocalBlocks_BJacobi);
    597:   return(0);
    598: }

    600: /* --------------------------------------------------------------------------------------------*/
    601: /*
    602:         These are for a single block per processor; works for AIJ, BAIJ; Seq and MPI
    603: */
    606: PetscErrorCode PCReset_BJacobi_Singleblock(PC pc)
    607: {
    608:   PC_BJacobi             *jac  = (PC_BJacobi*)pc->data;
    609:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
    610:   PetscErrorCode         ierr;

    613:   KSPReset(jac->ksp[0]);
    614:   VecDestroy(&bjac->x);
    615:   VecDestroy(&bjac->y);
    616:   return(0);
    617: }

    621: PetscErrorCode PCDestroy_BJacobi_Singleblock(PC pc)
    622: {
    623:   PC_BJacobi             *jac  = (PC_BJacobi*)pc->data;
    624:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
    625:   PetscErrorCode         ierr;

    628:   PCReset_BJacobi_Singleblock(pc);
    629:   KSPDestroy(&jac->ksp[0]);
    630:   PetscFree(jac->ksp);
    631:   PetscFree(jac->l_lens);
    632:   PetscFree(jac->g_lens);
    633:   PetscFree(bjac);
    634:   PetscFree(pc->data);
    635:   return(0);
    636: }

    640: PetscErrorCode PCSetUpOnBlocks_BJacobi_Singleblock(PC pc)
    641: {
    643:   PC_BJacobi     *jac = (PC_BJacobi*)pc->data;

    646:   KSPSetUp(jac->ksp[0]);
    647:   return(0);
    648: }

    652: PetscErrorCode PCApply_BJacobi_Singleblock(PC pc,Vec x,Vec y)
    653: {
    654:   PetscErrorCode         ierr;
    655:   PC_BJacobi             *jac  = (PC_BJacobi*)pc->data;
    656:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
    657:   PetscScalar            *x_array,*y_array;

    660:   /*
    661:       The VecPlaceArray() is to avoid having to copy the
    662:     y vector into the bjac->x vector. The reason for
    663:     the bjac->x vector is that we need a sequential vector
    664:     for the sequential solve.
    665:   */
    666:   VecGetArray(x,&x_array);
    667:   VecGetArray(y,&y_array);
    668:   VecPlaceArray(bjac->x,x_array);
    669:   VecPlaceArray(bjac->y,y_array);
    670:   KSPSolve(jac->ksp[0],bjac->x,bjac->y);
    671:   VecResetArray(bjac->x);
    672:   VecResetArray(bjac->y);
    673:   VecRestoreArray(x,&x_array);
    674:   VecRestoreArray(y,&y_array);
    675:   return(0);
    676: }

    680: PetscErrorCode PCApplySymmetricLeft_BJacobi_Singleblock(PC pc,Vec x,Vec y)
    681: {
    682:   PetscErrorCode         ierr;
    683:   PC_BJacobi             *jac  = (PC_BJacobi*)pc->data;
    684:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
    685:   PetscScalar            *x_array,*y_array;
    686:   PC                     subpc;

    689:   /*
    690:       The VecPlaceArray() is to avoid having to copy the
    691:     y vector into the bjac->x vector. The reason for
    692:     the bjac->x vector is that we need a sequential vector
    693:     for the sequential solve.
    694:   */
    695:   VecGetArray(x,&x_array);
    696:   VecGetArray(y,&y_array);
    697:   VecPlaceArray(bjac->x,x_array);
    698:   VecPlaceArray(bjac->y,y_array);
    699:   /* apply the symmetric left portion of the inner PC operator */
    700:   /* note this by-passes the inner KSP and its options completely */
    701:   KSPGetPC(jac->ksp[0],&subpc);
    702:   PCApplySymmetricLeft(subpc,bjac->x,bjac->y);
    703:   VecResetArray(bjac->x);
    704:   VecResetArray(bjac->y);
    705:   VecRestoreArray(x,&x_array);
    706:   VecRestoreArray(y,&y_array);
    707:   return(0);
    708: }

    712: PetscErrorCode PCApplySymmetricRight_BJacobi_Singleblock(PC pc,Vec x,Vec y)
    713: {
    714:   PetscErrorCode         ierr;
    715:   PC_BJacobi             *jac  = (PC_BJacobi*)pc->data;
    716:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
    717:   PetscScalar            *x_array,*y_array;
    718:   PC                     subpc;

    721:   /*
    722:       The VecPlaceArray() is to avoid having to copy the
    723:     y vector into the bjac->x vector. The reason for
    724:     the bjac->x vector is that we need a sequential vector
    725:     for the sequential solve.
    726:   */
    727:   VecGetArray(x,&x_array);
    728:   VecGetArray(y,&y_array);
    729:   VecPlaceArray(bjac->x,x_array);
    730:   VecPlaceArray(bjac->y,y_array);

    732:   /* apply the symmetric right portion of the inner PC operator */
    733:   /* note this by-passes the inner KSP and its options completely */

    735:   KSPGetPC(jac->ksp[0],&subpc);
    736:   PCApplySymmetricRight(subpc,bjac->x,bjac->y);

    738:   VecRestoreArray(x,&x_array);
    739:   VecRestoreArray(y,&y_array);
    740:   return(0);
    741: }

    745: PetscErrorCode PCApplyTranspose_BJacobi_Singleblock(PC pc,Vec x,Vec y)
    746: {
    747:   PetscErrorCode         ierr;
    748:   PC_BJacobi             *jac  = (PC_BJacobi*)pc->data;
    749:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
    750:   PetscScalar            *x_array,*y_array;

    753:   /*
    754:       The VecPlaceArray() is to avoid having to copy the
    755:     y vector into the bjac->x vector. The reason for
    756:     the bjac->x vector is that we need a sequential vector
    757:     for the sequential solve.
    758:   */
    759:   VecGetArray(x,&x_array);
    760:   VecGetArray(y,&y_array);
    761:   VecPlaceArray(bjac->x,x_array);
    762:   VecPlaceArray(bjac->y,y_array);
    763:   KSPSolveTranspose(jac->ksp[0],bjac->x,bjac->y);
    764:   VecResetArray(bjac->x);
    765:   VecResetArray(bjac->y);
    766:   VecRestoreArray(x,&x_array);
    767:   VecRestoreArray(y,&y_array);
    768:   return(0);
    769: }

    773: static PetscErrorCode PCSetUp_BJacobi_Singleblock(PC pc,Mat mat,Mat pmat)
    774: {
    775:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
    776:   PetscErrorCode         ierr;
    777:   PetscInt               m;
    778:   KSP                    ksp;
    779:   PC_BJacobi_Singleblock *bjac;
    780:   PetscBool              wasSetup = PETSC_TRUE;

    783:   if (!pc->setupcalled) {
    784:     const char *prefix;

    786:     if (!jac->ksp) {
    787:       wasSetup = PETSC_FALSE;

    789:       KSPCreate(PETSC_COMM_SELF,&ksp);
    790:       PetscObjectIncrementTabLevel((PetscObject)ksp,(PetscObject)pc,1);
    791:       PetscLogObjectParent(pc,ksp);
    792:       KSPSetType(ksp,KSPPREONLY);
    793:       PCGetOptionsPrefix(pc,&prefix);
    794:       KSPSetOptionsPrefix(ksp,prefix);
    795:       KSPAppendOptionsPrefix(ksp,"sub_");

    797:       pc->ops->reset               = PCReset_BJacobi_Singleblock;
    798:       pc->ops->destroy             = PCDestroy_BJacobi_Singleblock;
    799:       pc->ops->apply               = PCApply_BJacobi_Singleblock;
    800:       pc->ops->applysymmetricleft  = PCApplySymmetricLeft_BJacobi_Singleblock;
    801:       pc->ops->applysymmetricright = PCApplySymmetricRight_BJacobi_Singleblock;
    802:       pc->ops->applytranspose      = PCApplyTranspose_BJacobi_Singleblock;
    803:       pc->ops->setuponblocks       = PCSetUpOnBlocks_BJacobi_Singleblock;

    805:       PetscMalloc(sizeof(KSP),&jac->ksp);
    806:       jac->ksp[0] = ksp;

    808:       PetscNewLog(pc,PC_BJacobi_Singleblock,&bjac);
    809:       jac->data = (void*)bjac;
    810:     } else {
    811:       ksp  = jac->ksp[0];
    812:       bjac = (PC_BJacobi_Singleblock*)jac->data;
    813:     }

    815:     /*
    816:       The reason we need to generate these vectors is to serve
    817:       as the right-hand side and solution vector for the solve on the
    818:       block. We do not need to allocate space for the vectors since
    819:       that is provided via VecPlaceArray() just before the call to
    820:       KSPSolve() on the block.
    821:     */
    822:     MatGetSize(pmat,&m,&m);
    823:     VecCreateSeqWithArray(PETSC_COMM_SELF,1,m,NULL,&bjac->x);
    824:     VecCreateSeqWithArray(PETSC_COMM_SELF,1,m,NULL,&bjac->y);
    825:     PetscLogObjectParent(pc,bjac->x);
    826:     PetscLogObjectParent(pc,bjac->y);
    827:   } else {
    828:     ksp  = jac->ksp[0];
    829:     bjac = (PC_BJacobi_Singleblock*)jac->data;
    830:   }
    831:   if (pc->useAmat) {
    832:     KSPSetOperators(ksp,mat,pmat,pc->flag);
    833:   } else {
    834:     KSPSetOperators(ksp,pmat,pmat,pc->flag);
    835:   }
    836:   if (!wasSetup && pc->setfromoptionscalled) {
    837:     KSPSetFromOptions(ksp);
    838:   }
    839:   return(0);
    840: }

    842: /* ---------------------------------------------------------------------------------------------*/
    845: PetscErrorCode PCReset_BJacobi_Multiblock(PC pc)
    846: {
    847:   PC_BJacobi            *jac  = (PC_BJacobi*)pc->data;
    848:   PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
    849:   PetscErrorCode        ierr;
    850:   PetscInt              i;

    853:   if (bjac && bjac->pmat) {
    854:     MatDestroyMatrices(jac->n_local,&bjac->pmat);
    855:     if (pc->useAmat) {
    856:       MatDestroyMatrices(jac->n_local,&bjac->mat);
    857:     }
    858:   }

    860:   for (i=0; i<jac->n_local; i++) {
    861:     KSPReset(jac->ksp[i]);
    862:     if (bjac && bjac->x) {
    863:       VecDestroy(&bjac->x[i]);
    864:       VecDestroy(&bjac->y[i]);
    865:       ISDestroy(&bjac->is[i]);
    866:     }
    867:   }
    868:   PetscFree(jac->l_lens);
    869:   PetscFree(jac->g_lens);
    870:   return(0);
    871: }

    875: PetscErrorCode PCDestroy_BJacobi_Multiblock(PC pc)
    876: {
    877:   PC_BJacobi            *jac  = (PC_BJacobi*)pc->data;
    878:   PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
    879:   PetscErrorCode        ierr;
    880:   PetscInt              i;

    883:   PCReset_BJacobi_Multiblock(pc);
    884:   if (bjac) {
    885:     PetscFree2(bjac->x,bjac->y);
    886:     PetscFree(bjac->starts);
    887:     PetscFree(bjac->is);
    888:   }
    889:   PetscFree(jac->data);
    890:   for (i=0; i<jac->n_local; i++) {
    891:     KSPDestroy(&jac->ksp[i]);
    892:   }
    893:   PetscFree(jac->ksp);
    894:   PetscFree(pc->data);
    895:   return(0);
    896: }

    900: PetscErrorCode PCSetUpOnBlocks_BJacobi_Multiblock(PC pc)
    901: {
    902:   PC_BJacobi     *jac = (PC_BJacobi*)pc->data;
    904:   PetscInt       i,n_local = jac->n_local;

    907:   for (i=0; i<n_local; i++) {
    908:     KSPSetUp(jac->ksp[i]);
    909:   }
    910:   return(0);
    911: }

    913: /*
    914:       Preconditioner for block Jacobi
    915: */
    918: PetscErrorCode PCApply_BJacobi_Multiblock(PC pc,Vec x,Vec y)
    919: {
    920:   PC_BJacobi            *jac = (PC_BJacobi*)pc->data;
    921:   PetscErrorCode        ierr;
    922:   PetscInt              i,n_local = jac->n_local;
    923:   PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
    924:   PetscScalar           *xin,*yin;

    927:   VecGetArray(x,&xin);
    928:   VecGetArray(y,&yin);
    929:   for (i=0; i<n_local; i++) {
    930:     /*
    931:        To avoid copying the subvector from x into a workspace we instead
    932:        make the workspace vector array point to the subpart of the array of
    933:        the global vector.
    934:     */
    935:     VecPlaceArray(bjac->x[i],xin+bjac->starts[i]);
    936:     VecPlaceArray(bjac->y[i],yin+bjac->starts[i]);

    938:     PetscLogEventBegin(PC_ApplyOnBlocks,jac->ksp[i],bjac->x[i],bjac->y[i],0);
    939:     KSPSolve(jac->ksp[i],bjac->x[i],bjac->y[i]);
    940:     PetscLogEventEnd(PC_ApplyOnBlocks,jac->ksp[i],bjac->x[i],bjac->y[i],0);

    942:     VecResetArray(bjac->x[i]);
    943:     VecResetArray(bjac->y[i]);
    944:   }
    945:   VecRestoreArray(x,&xin);
    946:   VecRestoreArray(y,&yin);
    947:   return(0);
    948: }

    950: /*
    951:       Preconditioner for block Jacobi
    952: */
    955: PetscErrorCode PCApplyTranspose_BJacobi_Multiblock(PC pc,Vec x,Vec y)
    956: {
    957:   PC_BJacobi            *jac = (PC_BJacobi*)pc->data;
    958:   PetscErrorCode        ierr;
    959:   PetscInt              i,n_local = jac->n_local;
    960:   PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
    961:   PetscScalar           *xin,*yin;

    964:   VecGetArray(x,&xin);
    965:   VecGetArray(y,&yin);
    966:   for (i=0; i<n_local; i++) {
    967:     /*
    968:        To avoid copying the subvector from x into a workspace we instead
    969:        make the workspace vector array point to the subpart of the array of
    970:        the global vector.
    971:     */
    972:     VecPlaceArray(bjac->x[i],xin+bjac->starts[i]);
    973:     VecPlaceArray(bjac->y[i],yin+bjac->starts[i]);

    975:     PetscLogEventBegin(PC_ApplyTransposeOnBlocks,jac->ksp[i],bjac->x[i],bjac->y[i],0);
    976:     KSPSolveTranspose(jac->ksp[i],bjac->x[i],bjac->y[i]);
    977:     PetscLogEventEnd(PC_ApplyTransposeOnBlocks,jac->ksp[i],bjac->x[i],bjac->y[i],0);

    979:     VecResetArray(bjac->x[i]);
    980:     VecResetArray(bjac->y[i]);
    981:   }
    982:   VecRestoreArray(x,&xin);
    983:   VecRestoreArray(y,&yin);
    984:   return(0);
    985: }

    989: static PetscErrorCode PCSetUp_BJacobi_Multiblock(PC pc,Mat mat,Mat pmat)
    990: {
    991:   PC_BJacobi            *jac = (PC_BJacobi*)pc->data;
    992:   PetscErrorCode        ierr;
    993:   PetscInt              m,n_local,N,M,start,i;
    994:   const char            *prefix,*pprefix,*mprefix;
    995:   KSP                   ksp;
    996:   Vec                   x,y;
    997:   PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
    998:   PC                    subpc;
    999:   IS                    is;
    1000:   MatReuse              scall;

    1003:   MatGetLocalSize(pc->pmat,&M,&N);

    1005:   n_local = jac->n_local;

    1007:   if (pc->useAmat) {
    1008:     PetscBool same;
    1009:     PetscObjectTypeCompare((PetscObject)mat,((PetscObject)pmat)->type_name,&same);
    1010:     if (!same) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_INCOMP,"Matrices not of same type");
    1011:   }

    1013:   if (!pc->setupcalled) {
    1014:     scall = MAT_INITIAL_MATRIX;

    1016:     if (!jac->ksp) {
    1017:       pc->ops->reset         = PCReset_BJacobi_Multiblock;
    1018:       pc->ops->destroy       = PCDestroy_BJacobi_Multiblock;
    1019:       pc->ops->apply         = PCApply_BJacobi_Multiblock;
    1020:       pc->ops->applytranspose= PCApplyTranspose_BJacobi_Multiblock;
    1021:       pc->ops->setuponblocks = PCSetUpOnBlocks_BJacobi_Multiblock;

    1023:       PetscNewLog(pc,PC_BJacobi_Multiblock,&bjac);
    1024:       PetscMalloc(n_local*sizeof(KSP),&jac->ksp);
    1025:       PetscLogObjectMemory(pc,sizeof(n_local*sizeof(KSP)));
    1026:       PetscMalloc2(n_local,Vec,&bjac->x,n_local,Vec,&bjac->y);
    1027:       PetscMalloc(n_local*sizeof(PetscScalar),&bjac->starts);
    1028:       PetscLogObjectMemory(pc,sizeof(n_local*sizeof(PetscScalar)));

    1030:       jac->data = (void*)bjac;
    1031:       PetscMalloc(n_local*sizeof(IS),&bjac->is);
    1032:       PetscLogObjectMemory(pc,sizeof(n_local*sizeof(IS)));

    1034:       for (i=0; i<n_local; i++) {
    1035:         KSPCreate(PETSC_COMM_SELF,&ksp);
    1036:         PetscObjectIncrementTabLevel((PetscObject)ksp,(PetscObject)pc,1);
    1037:         PetscLogObjectParent(pc,ksp);
    1038:         KSPSetType(ksp,KSPPREONLY);
    1039:         KSPGetPC(ksp,&subpc);
    1040:         PCGetOptionsPrefix(pc,&prefix);
    1041:         KSPSetOptionsPrefix(ksp,prefix);
    1042:         KSPAppendOptionsPrefix(ksp,"sub_");

    1044:         jac->ksp[i] = ksp;
    1045:       }
    1046:     } else {
    1047:       bjac = (PC_BJacobi_Multiblock*)jac->data;
    1048:     }

    1050:     start = 0;
    1051:     for (i=0; i<n_local; i++) {
    1052:       m = jac->l_lens[i];
    1053:       /*
    1054:       The reason we need to generate these vectors is to serve
    1055:       as the right-hand side and solution vector for the solve on the
    1056:       block. We do not need to allocate space for the vectors since
    1057:       that is provided via VecPlaceArray() just before the call to
    1058:       KSPSolve() on the block.

    1060:       */
    1061:       VecCreateSeq(PETSC_COMM_SELF,m,&x);
    1062:       VecCreateSeqWithArray(PETSC_COMM_SELF,1,m,NULL,&y);
    1063:       PetscLogObjectParent(pc,x);
    1064:       PetscLogObjectParent(pc,y);

    1066:       bjac->x[i]      = x;
    1067:       bjac->y[i]      = y;
    1068:       bjac->starts[i] = start;

    1070:       ISCreateStride(PETSC_COMM_SELF,m,start,1,&is);
    1071:       bjac->is[i] = is;
    1072:       PetscLogObjectParent(pc,is);

    1074:       start += m;
    1075:     }
    1076:   } else {
    1077:     bjac = (PC_BJacobi_Multiblock*)jac->data;
    1078:     /*
    1079:        Destroy the blocks from the previous iteration
    1080:     */
    1081:     if (pc->flag == DIFFERENT_NONZERO_PATTERN) {
    1082:       MatDestroyMatrices(n_local,&bjac->pmat);
    1083:       if (pc->useAmat) {
    1084:         MatDestroyMatrices(n_local,&bjac->mat);
    1085:       }
    1086:       scall = MAT_INITIAL_MATRIX;
    1087:     } else scall = MAT_REUSE_MATRIX;
    1088:   }

    1090:   MatGetSubMatrices(pmat,n_local,bjac->is,bjac->is,scall,&bjac->pmat);
    1091:   if (pc->useAmat) {
    1092:     PetscObjectGetOptionsPrefix((PetscObject)mat,&mprefix);
    1093:     MatGetSubMatrices(mat,n_local,bjac->is,bjac->is,scall,&bjac->mat);
    1094:   }
    1095:   /* Return control to the user so that the submatrices can be modified (e.g., to apply
    1096:      different boundary conditions for the submatrices than for the global problem) */
    1097:   PCModifySubMatrices(pc,n_local,bjac->is,bjac->is,bjac->pmat,pc->modifysubmatricesP);

    1099:   PetscObjectGetOptionsPrefix((PetscObject)pmat,&pprefix);
    1100:   for (i=0; i<n_local; i++) {
    1101:     PetscLogObjectParent(pc,bjac->pmat[i]);
    1102:     PetscObjectSetOptionsPrefix((PetscObject)bjac->pmat[i],pprefix);
    1103:     if (pc->useAmat) {
    1104:       PetscLogObjectParent(pc,bjac->mat[i]);
    1105:       PetscObjectSetOptionsPrefix((PetscObject)bjac->mat[i],mprefix);
    1106:       KSPSetOperators(jac->ksp[i],bjac->mat[i],bjac->pmat[i],pc->flag);
    1107:     } else {
    1108:       KSPSetOperators(jac->ksp[i],bjac->pmat[i],bjac->pmat[i],pc->flag);
    1109:     }
    1110:     if (pc->setfromoptionscalled) {
    1111:       KSPSetFromOptions(jac->ksp[i]);
    1112:     }
    1113:   }
    1114:   return(0);
    1115: }

    1117: /* ---------------------------------------------------------------------------------------------*/
    1118: /*
    1119:       These are for a single block with multiple processes;
    1120: */
    1123: static PetscErrorCode PCReset_BJacobi_Multiproc(PC pc)
    1124: {
    1125:   PC_BJacobi           *jac   = (PC_BJacobi*)pc->data;
    1126:   PC_BJacobi_Multiproc *mpjac = (PC_BJacobi_Multiproc*)jac->data;
    1127:   PetscErrorCode       ierr;

    1130:   VecDestroy(&mpjac->ysub);
    1131:   VecDestroy(&mpjac->xsub);
    1132:   MatDestroy(&mpjac->submats);
    1133:   if (jac->ksp) {KSPReset(jac->ksp[0]);}
    1134:   return(0);
    1135: }

    1139: static PetscErrorCode PCDestroy_BJacobi_Multiproc(PC pc)
    1140: {
    1141:   PC_BJacobi           *jac   = (PC_BJacobi*)pc->data;
    1142:   PC_BJacobi_Multiproc *mpjac = (PC_BJacobi_Multiproc*)jac->data;
    1143:   PetscErrorCode       ierr;

    1146:   PCReset_BJacobi_Multiproc(pc);
    1147:   KSPDestroy(&jac->ksp[0]);
    1148:   PetscFree(jac->ksp);
    1149:   PetscSubcommDestroy(&mpjac->psubcomm);

    1151:   PetscFree(mpjac);
    1152:   PetscFree(pc->data);
    1153:   return(0);
    1154: }

    1158: static PetscErrorCode PCApply_BJacobi_Multiproc(PC pc,Vec x,Vec y)
    1159: {
    1160:   PC_BJacobi           *jac   = (PC_BJacobi*)pc->data;
    1161:   PC_BJacobi_Multiproc *mpjac = (PC_BJacobi_Multiproc*)jac->data;
    1162:   PetscErrorCode       ierr;
    1163:   PetscScalar          *xarray,*yarray;

    1166:   /* place x's and y's local arrays into xsub and ysub */
    1167:   VecGetArray(x,&xarray);
    1168:   VecGetArray(y,&yarray);
    1169:   VecPlaceArray(mpjac->xsub,xarray);
    1170:   VecPlaceArray(mpjac->ysub,yarray);

    1172:   /* apply preconditioner on each matrix block */
    1173:   PetscLogEventBegin(PC_ApplyOnMproc,jac->ksp[0],mpjac->xsub,mpjac->ysub,0);
    1174:   KSPSolve(jac->ksp[0],mpjac->xsub,mpjac->ysub);
    1175:   PetscLogEventEnd(PC_ApplyOnMproc,jac->ksp[0],mpjac->xsub,mpjac->ysub,0);

    1177:   VecResetArray(mpjac->xsub);
    1178:   VecResetArray(mpjac->ysub);
    1179:   VecRestoreArray(x,&xarray);
    1180:   VecRestoreArray(y,&yarray);
    1181:   return(0);
    1182: }

    1184: #include <petsc-private/matimpl.h>
    1187: static PetscErrorCode PCSetUp_BJacobi_Multiproc(PC pc)
    1188: {
    1189:   PC_BJacobi           *jac   = (PC_BJacobi*)pc->data;
    1190:   PC_BJacobi_Multiproc *mpjac = (PC_BJacobi_Multiproc*)jac->data;
    1191:   PetscErrorCode       ierr;
    1192:   PetscInt             m,n;
    1193:   MPI_Comm             comm,subcomm=0;
    1194:   const char           *prefix;
    1195:   PetscBool            wasSetup = PETSC_TRUE;

    1198:   PetscObjectGetComm((PetscObject)pc,&comm);
    1199:   if (jac->n_local > 1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Only a single block in a subcommunicator is supported");
    1200:   jac->n_local = 1; /* currently only a single block is supported for a subcommunicator */
    1201:   if (!pc->setupcalled) {
    1202:     wasSetup  = PETSC_FALSE;
    1203:     PetscNewLog(pc,PC_BJacobi_Multiproc,&mpjac);
    1204:     jac->data = (void*)mpjac;

    1206:     /* initialize datastructure mpjac */
    1207:     if (!jac->psubcomm) {
    1208:       /* Create default contiguous subcommunicatiors if user does not provide them */
    1209:       PetscSubcommCreate(comm,&jac->psubcomm);
    1210:       PetscSubcommSetNumber(jac->psubcomm,jac->n);
    1211:       PetscSubcommSetType(jac->psubcomm,PETSC_SUBCOMM_CONTIGUOUS);
    1212:       PetscLogObjectMemory(pc,sizeof(PetscSubcomm));
    1213:     }
    1214:     mpjac->psubcomm = jac->psubcomm;
    1215:     subcomm         = mpjac->psubcomm->comm;

    1217:     /* Get matrix blocks of pmat */
    1218:     if (!pc->pmat->ops->getmultiprocblock) SETERRQ(PetscObjectComm((PetscObject)pc->pmat),PETSC_ERR_SUP,"No support for the requested operation");
    1219:     (*pc->pmat->ops->getmultiprocblock)(pc->pmat,subcomm,MAT_INITIAL_MATRIX,&mpjac->submats);

    1221:     /* create a new PC that processors in each subcomm have copy of */
    1222:     PetscMalloc(sizeof(KSP),&jac->ksp);
    1223:     KSPCreate(subcomm,&jac->ksp[0]);
    1224:     PetscObjectIncrementTabLevel((PetscObject)jac->ksp[0],(PetscObject)pc,1);
    1225:     PetscLogObjectParent(pc,jac->ksp[0]);
    1226:     KSPSetOperators(jac->ksp[0],mpjac->submats,mpjac->submats,pc->flag);
    1227:     KSPGetPC(jac->ksp[0],&mpjac->pc);

    1229:     PCGetOptionsPrefix(pc,&prefix);
    1230:     KSPSetOptionsPrefix(jac->ksp[0],prefix);
    1231:     KSPAppendOptionsPrefix(jac->ksp[0],"sub_");
    1232:     /*
    1233:       PetscMPIInt rank,subsize,subrank;
    1234:       MPI_Comm_rank(comm,&rank);
    1235:       MPI_Comm_size(subcomm,&subsize);
    1236:       MPI_Comm_rank(subcomm,&subrank);

    1238:       MatGetLocalSize(mpjac->submats,&m,NULL);
    1239:       MatGetSize(mpjac->submats,&n,NULL);
    1240:       PetscSynchronizedPrintf(comm,"[%d], sub-size %d,sub-rank %d\n",rank,subsize,subrank);
    1241:       PetscSynchronizedFlush(comm);
    1242:     */

    1244:     /* create dummy vectors xsub and ysub */
    1245:     MatGetLocalSize(mpjac->submats,&m,&n);
    1246:     VecCreateMPIWithArray(subcomm,1,n,PETSC_DECIDE,NULL,&mpjac->xsub);
    1247:     VecCreateMPIWithArray(subcomm,1,m,PETSC_DECIDE,NULL,&mpjac->ysub);
    1248:     PetscLogObjectParent(pc,mpjac->xsub);
    1249:     PetscLogObjectParent(pc,mpjac->ysub);

    1251:     pc->ops->reset   = PCReset_BJacobi_Multiproc;
    1252:     pc->ops->destroy = PCDestroy_BJacobi_Multiproc;
    1253:     pc->ops->apply   = PCApply_BJacobi_Multiproc;
    1254:   } else { /* pc->setupcalled */
    1255:     subcomm = mpjac->psubcomm->comm;
    1256:     if (pc->flag == DIFFERENT_NONZERO_PATTERN) {
    1257:       /* destroy old matrix blocks, then get new matrix blocks */
    1258:       if (mpjac->submats) {MatDestroy(&mpjac->submats);}
    1259:       (*pc->pmat->ops->getmultiprocblock)(pc->pmat,subcomm,MAT_INITIAL_MATRIX,&mpjac->submats);
    1260:     } else {
    1261:       (*pc->pmat->ops->getmultiprocblock)(pc->pmat,subcomm,MAT_REUSE_MATRIX,&mpjac->submats);
    1262:     }
    1263:     KSPSetOperators(jac->ksp[0],mpjac->submats,mpjac->submats,pc->flag);
    1264:   }

    1266:   if (!wasSetup && pc->setfromoptionscalled) {
    1267:     KSPSetFromOptions(jac->ksp[0]);
    1268:   }
    1269:   return(0);
    1270: }