/*===========================================================================
  Copyright (C) 2001 European Southern Observatory (ESO)
 
  This program is free software; you can redistribute it and/or 
  modify it under the terms of the GNU General Public License as 
  published by the Free Software Foundation; either version 2 of 
  the License, or (at your option) any later version.
 
  This program is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.
 
  You should have received a copy of the GNU General Public 
  License along with this program; if not, write to the Free 
  Software Foundation, Inc., 675 Massachusetss Ave, Cambridge, 
  MA 02139, USA.
 
  Corresponding concerning ESO-MIDAS should be addressed as follows:
    Internet e-mail: midas@eso.org
    Postal address: European Southern Observatory
            Data Management Division 
            Karl-Schwarzschild-Strasse 2
            D 85748 Garching bei Muenchen 
            GERMANY
===========================================================================*/
/* Program  : scatter.c                                                    */
/* Author   : G. Mulas  -  ITAL_FLAMES Consortium                          */
/* Date     :                                                              */
/*                                                                         */
/* Purpose  : Missing                                                      */
/*                                                                         */
/*                                                                         */
/* Input:  see interface                                                   */ 
/*                                                                      */
/* Output:                                                              */
/*                                                                         */
/* DRS Functions called:                                                   */
/* none                                                                    */ 
/*                                                                         */ 
/* Pseudocode:                                                             */
/* Missing                                                                 */ 
/*                                                                         */ 
/* Version  :                                                              */
/* Last modification date: 2002/08/05                                      */
/* Who     When        Why                Where                            */
/* AMo     02-08-05   Add header         header                            */
/*-------------------------------------------------------------------------*/

#ifdef HAVE_CONFIG_H
#  include <config.h>
#endif


#include <stdlib.h>
#include <stdio.h>
#include <math.h>
#include <string.h>
#include <flames.h>
#include <flames_midas_def.h>
#include <flames_def_drs_par.h>
#include <flames_scatter.h>
#include <flames_dfs.h>
#include <flames_uves.h>
#include <flames_dgaussj.h>
#include <flames_newmatrix.h>
#include <flames_trimback.h>
#include <uves_msg.h>


typedef struct _deviation {
  double zdev;
  int32_t zind;
} deviation;

typedef struct _pixel {
  frame_data value;
  int32_t ix;
  int32_t iy;
} pixel;


static int 
devcomp(const void *zdev1, const void *zdev2);

static int 
pixelcomp(const void *pixel1, const void *pixel2);


flames_err 
scatter(flames_frame *Frame, 
        orderpos *Order, 
        scatterswitch bkgswitch, 
    scatterswitch2 bkgswitch2, 
        int badwinxsize, 
    int badwinysize, 
        double badfracthres, 
        int badtotthres, 
    double kappa2, 
        int32_t maxbackiters, 
    double maxdiscardfract,
        int *OUTPUTI)
{
 flames_err status;

 char output[70];
 int k=0;
 int l=0;
 int i=0;
 int j=0;
 int npar=0;
 int freed_sw=0;
 int qq;
 int* mid_unit=0;

 double di;
 double dj;
 double xscale;
 double yscale;
 double zdiff;
 double **d;
 double **x;

 int drs_bkg_pix;
 int32_t goodpixels;
 int32_t goodpixelsorig;
 int32_t newgoodpixels;
 int32_t niters;
 int32_t i2;
 int32_t j2;

 int32_t goodwindows=0; 
 frame_mask windowisgood=0;

 double **xmatrix;
 double **ymatrix;
 double **xmatrixnew;
 double **ymatrixnew;

 double *zvector;
 double *zsigma;
 double *zvectornew;
 double *zsigmanew;
 double *zfitted;

 char *zmask;
 deviation *zdevs; 


 int32_t maxrejects=0;

 double *xbuf;
 double *xbuf1;
 double *xbuf2;
 double *ybuf;
 double *ybuf1;
 double *ybuf2;

 int32_t *Window_x_low=0;
 int32_t *Window_x_up=0;
 int32_t *Window_y_low=0;
 int32_t *Window_y_up=0;

 pixel *pixellist;

 int32_t medianindex=0;
 int32_t i1=0;
 int32_t j1=0;
 int32_t minix=0;
 int32_t miniy=0;

 frame_data minvalue=0;
 frame_data medianvalue=0;

 double normfactor=0;
 double normweight=0;
 double averix=0;
 double averiy=0;
 double avervalue=0;
 double aversigma=0;

 frame_data *fdvecbuf1=0;
 frame_data *fdvecbuf2=0;
 frame_mask *fmvecbuf1=0;

 int32_t ijoffset=0;
 int32_t ijindex=0;
 int32_t i1j1index=0;
 int32_t i2j2index=0;
 int actvals=0;
 char drs_verbosity[10];
 int mid_stat=0;

 
 memset(output, '\0', 70);

 uves_msg_debug("bkgswitch=%d",bkgswitch); 
 uves_msg_debug("bkgswitch2=%d",bkgswitch2); 
 uves_msg_debug("badwinxsize=%d",badwinxsize); 
 uves_msg_debug("badwinysize=%d",badwinysize); 
 uves_msg_debug("badfracthres=%f",badfracthres); 
 uves_msg_debug("badtotthres=%d",badtotthres); 
 uves_msg_debug("kappa2=%f",kappa2); 
 uves_msg_debug("maxbackiters=%d",maxbackiters); 
 uves_msg_debug("maxdiscardfract=%f",maxdiscardfract);


 fdvecbuf1 = Frame->frame_array[0];
 fdvecbuf2 = Frame->frame_sigma[0];
 fmvecbuf1 = Frame->badpixel[0];


#if FLAMES_DEBUG
 {
     int i;
     for (i = 0; i < 100; i++){
         fprintf(stderr, "%d %f %f %d\n", 
                 i, 
                 fdvecbuf1[200*50 + i*100],
                 fdvecbuf2[200*50 + i*100],
                 fmvecbuf1[200*50 + i*100]);
     }
 }
#endif



 if (Frame->substepx == 0) {
   SCTPUT("Error: the x step must be non zero");
   SCSEPI();
   return flames_midas_fail();
 }
 if (Frame->substepy == 0) {
   SCTPUT("Error: the y step must be non zero");
   SCSEPI();
   return flames_midas_fail();
 }


 memset(drs_verbosity, 0, 10);
 if ((mid_stat=SCKGETC(DRS_VERBOSITY, 1, 3, &actvals, drs_verbosity))
      != 0) {
    /* the keyword seems undefined, protest... */
    return(MAREMMA);
 }



 /* trim the background table to avoid pixels containing light from the 
    (possibly shifted) fibres */
 sprintf(output, 
     "trimming background table from %d windows...", 
     Frame->back.Window_Number);
 SCTPUT(output);




 uves_msg_debug("processing frame=%s\n",Frame->framename);
 uves_msg_debug("bkgswitch=%d\n",bkgswitch);
 uves_msg_debug("bkgswitch2=%d\n",bkgswitch2);
 uves_msg_debug("badwinxsize=%d\n", badwinxsize);
 uves_msg_debug("badwinysize=%d\n", badwinysize);
 uves_msg_debug("badfracthres=%f\n", badfracthres);
 uves_msg_debug("badtotthres=%d\n",badtotthres);
 uves_msg_debug("kappa2=%f\n",kappa2);
 uves_msg_debug("maxbackiters=%d\n",maxbackiters);
 uves_msg_debug("maxdiscardfract=%f\n",maxdiscardfract);

 if ((status=trimback(Frame, Order, &Window_x_low, &Window_x_up, 
              &Window_y_low, &Window_y_up, bkgswitch2, badwinxsize, 
              badwinysize, badfracthres, badtotthres))!=NOERR) {
   SCTPUT("Error trimming background table");
   return flames_midas_fail();
 }
 /* remember that trimback allocates the Window_* vectors, we will have 
    to free them before returning */
 sprintf(output, "...%d windows left\n", Frame->back.Window_Number);
 SCTPUT(output); 

 /* put together the initial data for the fit, now */
 /* we want to have several possible options for this, depending
    on the trade off between speed and quality which we want to 
    achieve */

 /* was neighborhood bad pixel scanning required? */


 goodpixels = 0;

 for (qq=1; qq<=Frame->back.Window_Number; qq++) {
   for (i=Window_y_low[qq]; i<=Window_y_up[qq]; i++) {
     ijoffset = i*Frame->subcols;
     
     for (j=Window_x_low[qq]; j<=Window_x_up[qq]; j++) {
       ijindex = ijoffset+j;
       //uves_msg("fmvecbuf1[ijindex]=%d",fmvecbuf1[ijindex]);
       if (fmvecbuf1[ijindex] == 0) {
     /* good pixel, use it in the fit */
	 //uves_msg("good pix");
     goodpixels++;
       }
     }
   }
 }

 /* did we find any good pixels? */
 if (goodpixels==0) {
   SCTPUT("Background estimation impossible: no usable pixels!\n");
   return flames_midas_fail();
 }
 npar = (int32_t)((Frame->back.xdegree+1)*(Frame->back.ydegree+1));
 xscale = Frame->subcols>=2 ? (double)Frame->subcols-1 : 1;
 yscale = Frame->subrows>=2 ? (double)Frame->subrows-1 : 1;

 /* we want to have several possible options for this, depending
    on the trade off between speed and quality that we want to 
    achieve */
 switch(bkgswitch) {

 case USEALL:
 default:
   /* use all the pixels in all the windows */
   /* did we find enough pixels? */
   if (goodpixels<npar) {
     SCTPUT("Background estimation impossible1:\n less usable pixels than \
parameters\n");
     return flames_midas_fail();
   }
   sprintf(output, "Using %d pixels for background fitting...\n",goodpixels);
   SCTPUT(output);
  
   drs_bkg_pix = goodpixels;
   SCKWRI(OUTPUTI,&drs_bkg_pix,4,1,mid_unit);

   xmatrix = dmatrix(1,2*Frame->back.xdegree,1,goodpixels);
   ymatrix = dmatrix(1,2*Frame->back.ydegree,1,goodpixels);
   zvector = dvector(1,goodpixels);
   zsigma = dvector(1,goodpixels);
   goodpixels = 0;

   for (qq=1; qq<=Frame->back.Window_Number; qq++) {
     for (i=Window_y_low[qq]; i<=Window_y_up[qq]; i++) {
       ijoffset = i*Frame->subcols;
       di = ((double) i)/yscale;
       for (j=Window_x_low[qq]; j<=Window_x_up[qq]; j++) {
     ijindex = ijoffset+j;
     if (fmvecbuf1[ijindex] == 0) {
       /* good pixel, use it in the fit */
       goodpixels++;
       xmatrix[1][goodpixels] = ((double) j)/xscale;
       ymatrix[1][goodpixels] = di;
       zvector[goodpixels] = fdvecbuf1[ijindex];
       zsigma[goodpixels] = fdvecbuf2[ijindex];
     }
       }
     }
   }
   break;

 case USEMEDIAN:

   /* find the median in each window and use only that */
   /* count the windows containing at least one good pixel */
   goodwindows = 0;
   goodpixels = 0;
   for (qq=1; qq<=Frame->back.Window_Number; qq++) {
     windowisgood = FALSE;
     for (i=Window_y_low[qq]; windowisgood==FALSE && i<=Window_y_up[qq]; i++){
       ijoffset = i*Frame->subcols;
       for (j=Window_x_low[qq]; windowisgood==FALSE && j<=Window_x_up[qq]; 
        j++){
     ijindex = ijoffset+j;
     if (fmvecbuf1[ijindex]==0) windowisgood = TRUE;
       }
     }
     if (windowisgood == TRUE) {
       goodwindows++;
       newgoodpixels = (Window_y_up[qq]-Window_y_low[qq]+1)*
     (Window_x_up[qq]-Window_x_low[qq]+1);
       if (newgoodpixels>goodpixels) goodpixels=newgoodpixels;
     }
   }
   /* ok, we can now define the arrays with the correct size */
   /* did we find enough pixels? */

   if (goodwindows<npar) {
     SCTPUT("Background estimation impossible2:\n less usable windows than \
parameters\n");
     return flames_midas_fail();
   }
   pixellist = ((pixel *) calloc((size_t) goodpixels, sizeof(pixel)));
   sprintf(output, "Using %d pixels for background fitting...\n",goodwindows);
   SCTPUT(output);
   drs_bkg_pix = goodwindows;
   SCKWRI(OUTPUTI,&drs_bkg_pix,3,1,mid_unit);


   xmatrix = dmatrix(1,2*Frame->back.xdegree,1,goodwindows);
   ymatrix = dmatrix(1,2*Frame->back.ydegree,1,goodwindows);
   zvector = dvector(1,goodwindows);
   zsigma = dvector(1,goodwindows);
   goodwindows = 0;
   for (qq=1; qq<=Frame->back.Window_Number; qq++) {
     windowisgood = FALSE;
     goodpixels = 0;
     for (i=Window_y_low[qq]; i<=Window_y_up[qq]; i++) {
       ijoffset = i*Frame->subcols;
       for (j=Window_x_low[qq]; j<=Window_x_up[qq]; j++) {
     ijindex = ijoffset+j;
     if (fmvecbuf1[ijindex] == 0) {
       /* good pixel, use it in the fit */
       pixellist[goodpixels].iy = i;
       pixellist[goodpixels].ix = j;
       pixellist[goodpixels].value = fdvecbuf1[ijindex];
       windowisgood = TRUE;
       goodpixels++;
     }
       }
     }

     /* is this a good window? */
     if (windowisgood==TRUE) {
       /* to find the median, sort the pixels */
       qsort(pixellist, (size_t) goodpixels, sizeof(pixel), pixelcomp);

       /* now that the pixels are sorted, pick the median */
       if (2*(goodpixels/2) != goodpixels) {
     /* goodpixels is odd, just pick the right index */
     medianindex = (goodpixels-1)/2;
     i = pixellist[medianindex].iy;
     j = pixellist[medianindex].ix;
     ijindex = (i*Frame->subcols)+j;
     goodwindows++;
     xmatrix[1][goodwindows] = ((double) j)/xscale;
     ymatrix[1][goodwindows] = ((double) i)/yscale;
     zvector[goodwindows] = fdvecbuf1[ijindex];
     zsigma[goodwindows] = fdvecbuf2[ijindex];
       }
       else {
     /* goodpixels is even, compute the average of the two
        median values */
     medianindex = goodpixels/2;
     i1 = pixellist[medianindex-1].iy;
     j1 = pixellist[medianindex-1].ix;
     i1j1index = (i1*Frame->subcols)+j1;
     i2 = pixellist[medianindex].iy;
     j2 = pixellist[medianindex].ix;
     i2j2index = (i2*Frame->subcols)+j2;
     goodwindows++;
     xmatrix[1][goodwindows] = .5*((double)j1+(double)j2)/xscale;
     ymatrix[1][goodwindows] = .5*((double)i1+(double)i2)/yscale;
     zvector[goodwindows] = 
       .5*(fdvecbuf1[i1j1index]+fdvecbuf1[i2j2index]);
     zsigma[goodwindows] = 
       .5*(fdvecbuf2[i1j1index]+fdvecbuf2[i2j2index]);
       }
     }
   }
   free(pixellist);
   goodpixels = goodwindows;
   break;

 case USEMINIMUM:

   /* use the pixel with the minimum value in each window */
   /* count the windows containing at least one good pixel */
   goodwindows = 0;
   goodpixels = 0;
   for (qq=1; qq<=Frame->back.Window_Number; qq++) {
     windowisgood = FALSE;
     for (i=Window_y_low[qq]; windowisgood==FALSE && i<=Window_y_up[qq]; i++) {
       ijoffset = i*Frame->subcols;
       for (j=Window_x_low[qq]; windowisgood==FALSE && j<=Window_x_up[qq]; 
        j++) {
     ijindex = ijoffset+j;
     if (fmvecbuf1[ijindex]==0) windowisgood = TRUE;
       }
     }
     if (windowisgood == TRUE) goodwindows++;
   }
   /* ok, we can now define the arrays with the correct size */
   /* did we find enough pixels? */
   if (goodwindows<npar) {
     SCTPUT("Background estimation impossible3:\n less usable windows than \
parameters\n");
     return flames_midas_fail();
   }
   sprintf(output, "Using %d pixels for background fitting...\n",goodwindows);
   SCTPUT(output);
  
  drs_bkg_pix = goodwindows;

   SCKWRI(OUTPUTI,&drs_bkg_pix,3,1,mid_unit);

   xmatrix = dmatrix(1,2*Frame->back.xdegree,1,goodwindows);
   ymatrix = dmatrix(1,2*Frame->back.ydegree,1,goodwindows);
   zvector = dvector(1,goodwindows);
   zsigma = dvector(1,goodwindows);
   goodwindows = 0;
   for (qq=1; qq<=Frame->back.Window_Number; qq++) {
     windowisgood = FALSE;
     for (i=Window_y_low[qq]; i<=Window_y_up[qq]; i++) {
       ijoffset = i*Frame->subcols;
       for (j=Window_x_low[qq]; j<=Window_x_up[qq]; j++) {
     ijindex = ijoffset+j;
     if (fmvecbuf1[ijindex] == 0) {
       /* good pixel, use it in the fit */
       if (windowisgood==FALSE) {
         /* this is the first good pixel we found in this window */
         minix = j;
         miniy = i;
         minvalue = fdvecbuf1[ijindex];
         windowisgood = TRUE;
       }
       else {
         /* this is not the first good pixel in this window */
         if (fdvecbuf1[ijindex]<minvalue) {
         minix = j;
         miniy = i;
         minvalue = fdvecbuf1[ijindex];
         }
       }
     }
       }
     }
     /* is this a good window? */
     if (windowisgood==TRUE) {
       goodwindows++;
       xmatrix[1][goodwindows] = ((double) minix)/xscale;
       ymatrix[1][goodwindows] = ((double) miniy)/yscale;
       ijindex = (miniy*Frame->subcols)+minix;
       zvector[goodwindows] = fdvecbuf1[ijindex];
       zsigma[goodwindows] = fdvecbuf2[ijindex];
     }
   }
   goodpixels = goodwindows;
   break;

 case USEAVERAGE:

   /* find the median in each window and that to sigma-clip other pixels 
      in the same window*/
   /* count the windows containing at least one good pixel */
   goodwindows = 0;
   goodpixels = 0;
   for (qq=1; qq<=Frame->back.Window_Number; qq++) {
     windowisgood = FALSE;
     for (i=Window_y_low[qq]; windowisgood==FALSE && i<=Window_y_up[qq]; i++){
       ijoffset = i*Frame->subcols;
       for (j=Window_x_low[qq]; windowisgood==FALSE && j<=Window_x_up[qq]; 
        j++) {
     ijindex = ijoffset+j;
     if (fmvecbuf1[ijindex]==0) windowisgood = TRUE;
       }
     }
     if (windowisgood == TRUE) {
       goodwindows++;
       newgoodpixels = (Window_y_up[qq]-Window_y_low[qq]+1)*
     (Window_x_up[qq]-Window_x_low[qq]+1);
       if (newgoodpixels>goodpixels) goodpixels=newgoodpixels;
     }
   }
   /* ok, we can now define the arrays with the correct size */
   /* did we find enough pixels? */

   if (goodwindows<npar) {
     SCTPUT("Background estimation impossible4:\n less usable windows than \
parameters\n");
     return flames_midas_fail();
   }
   pixellist = ((pixel *) calloc((size_t) goodpixels, sizeof(pixel)));
   sprintf(output, "Using %d pixels for background fitting...\n",goodwindows);
   SCTPUT(output);

   drs_bkg_pix = goodwindows;

   SCKWRI(OUTPUTI,&drs_bkg_pix,3,1,mid_unit);

   xmatrix = dmatrix(1,2*Frame->back.xdegree,1,goodwindows);
   ymatrix = dmatrix(1,2*Frame->back.ydegree,1,goodwindows);
   zvector = dvector(1,goodwindows);
   zsigma = dvector(1,goodwindows);
   goodwindows = 0;
   for (qq=1; qq<=Frame->back.Window_Number; qq++) {
     windowisgood = FALSE;
     goodpixels = 0;
     for (i=Window_y_low[qq]; i<=Window_y_up[qq]; i++) {
       ijoffset = i*Frame->subcols;
       for (j=Window_x_low[qq]; j<=Window_x_up[qq]; j++) {
     ijindex = ijoffset+j;
     if (fmvecbuf1[ijindex] == 0) {
       /* good pixel, use it in the fit */
       pixellist[goodpixels].iy = i;
       pixellist[goodpixels].ix = j;
       pixellist[goodpixels].value = fdvecbuf1[ijindex];
       windowisgood = TRUE;
       goodpixels++;
     }
       }
     }
     /* is this a good window? */
     if (windowisgood==TRUE) {
       /* to find the median, sort the pixels */
       qsort(pixellist, (size_t) goodpixels, sizeof(pixel), pixelcomp);
       /* now that the pixels are sorted, pick the median */
       if (2*(goodpixels/2) != goodpixels) {
     /* goodpixels is odd, just pick the right index */
     medianindex = (goodpixels-1)/2;
     i = pixellist[medianindex].iy;
     j = pixellist[medianindex].ix;
     ijindex = (i*Frame->subcols)+j;
     medianvalue = fdvecbuf1[ijindex];
       }
       else {
     /* goodpixels is even, compute the average of the two
        median values */
     medianindex = goodpixels/2;
     i1 = pixellist[medianindex-1].iy;
     j1 = pixellist[medianindex-1].ix;
     i1j1index = (i1*Frame->subcols)+j1;
     i2 = pixellist[medianindex].iy;
     j2 = pixellist[medianindex].ix;
     i2j2index = (i2*Frame->subcols)+j2;
     medianvalue = .5*(fdvecbuf1[i1j1index]+fdvecbuf1[i2j2index]);
       }
       /* ok, now build the average with the pixels which do not exceed the 
      cutting threshold */       
       normfactor=0;
       averix=0;
       averiy=0;
       avervalue=0;
       aversigma=0;
       for (i=Window_y_low[qq]; i<=Window_y_up[qq]; i++) {
     ijoffset = i*Frame->subcols;
     for (j=Window_x_low[qq]; j<=Window_x_up[qq]; j++) {
       ijindex=ijoffset+j;
       if (fmvecbuf1[ijindex]==0 && 
           fabs((double)((fdvecbuf1[ijindex]-medianvalue)/
                 fdvecbuf2[ijindex]))<kappa2) {
         /* good pixel, use it in the fit */
         normweight = (double) 1/fdvecbuf2[ijindex];
         averix += normweight*(double)j;
         averiy += normweight*(double)i;
         avervalue += normweight*(double)fdvecbuf1[ijindex];
         normfactor += normweight;
       }
     }
       }
       goodwindows++;
       xmatrix[1][goodwindows] = averix/(normfactor*xscale);
       ymatrix[1][goodwindows] = averiy/(normfactor*yscale);
       zvector[goodwindows] = avervalue/normfactor;
       zsigma[goodwindows] = 1/normfactor;
     }
   }
   free(pixellist);
   goodpixels = goodwindows;
   break;
 }

 /* free the Window_* boundary vectors here, we do not need them further */
 free_lvector(Window_x_low, 1, Frame->back.Window_Number);
 free_lvector(Window_x_up, 1, Frame->back.Window_Number);
 free_lvector(Window_y_low, 1, Frame->back.Window_Number);
 free_lvector(Window_y_up, 1, Frame->back.Window_Number);

 xbuf1 = xmatrix[1];
 for (j=2; j<=2*Frame->back.xdegree; j++) {
   xbuf = xmatrix[j];
   xbuf2 = xmatrix[j-1];
   dj = (double) j;
   for (qq=1; qq<=goodpixels; qq++) 
     xbuf[qq] = xbuf1[qq]*xbuf2[qq];
 }
 ybuf1 = ymatrix[1];
 for (i=2; i<=2*Frame->back.ydegree; i++) {
   ybuf = ymatrix[i];
   ybuf2 = ymatrix[i-1];
   di = (double) i;
   for (qq=1; qq<=goodpixels; qq++) 
     ybuf[qq] = ybuf1[qq]*ybuf2[qq];
 }
 goodpixelsorig = newgoodpixels = goodpixels;
 niters = 0;
 x=dmatrix(1,npar,1,1);
 d=dmatrix(1,npar,1,npar);

 /* start the fitting/kappa-sigma-clipping loop */
 do {
   niters++;

   if ( strcmp(drs_verbosity,"LOW") == 0 ){
   } else {

       sprintf(output, "background fitting iteration %d running...", niters);
       SCTPUT(output);

       }
   goodpixels = newgoodpixels;
   maxrejects = (int32_t) ceil(maxdiscardfract*goodpixels);
   if (goodpixels<maxrejects) maxrejects=goodpixels;
   else if (maxrejects<1) maxrejects=1;

   zfitted = dvector(1,goodpixels);
   zdevs = ((deviation *) calloc((size_t) (goodpixels),  sizeof(deviation)))-1;
   zmask = cvector(1,goodpixels);
   for (k=1; k<=npar; k++) {
     x[k][1] = 0;
     for (l=1; l<=npar; l++) {
       d[k][l] = 0;
     }
   }
   /* the first running index is k, the second is l: here k = l = 1 */
   k = l = 1;
   for (qq=1; qq<=goodpixels; qq++) {
     /* i = i2 = j = j2 = 0 */
     x[1][1] += zvector[qq]/zsigma[qq];
     d[1][1] += 1/zsigma[qq];
   }
   /* span all the l>1 values for k = 1; */
   for (j2=1; j2<=Frame->back.xdegree; j2++){
     /* i = i2 = j = 0, j2 > 0 */
     l++;
     xbuf = xmatrix[j2];
     for (qq=1; qq<=goodpixels; qq++)
       d[1][l] += xbuf[qq]/zsigma[qq];
   }
   for (i2=1; i2<=Frame->back.ydegree; i2++){
     /* i = j = 0, i2>0, j2 = 0 */
     l++;
     ybuf = ymatrix[i2];
     for (qq=1; qq<=goodpixels; qq++) 
       d[1][l] += ybuf[qq]/zsigma[qq];
     for (j2=1; j2<=Frame->back.xdegree; j2++){
       /* i = j = 0, i2>0, j2>0 */
       l++;
       xbuf = xmatrix[j2];
       for (qq=1; qq<=goodpixels; qq++)
     d[1][l] += xbuf[qq]*ybuf[qq]/zsigma[qq];
     }
   }
   /* run over the k values for which the polynomial has 0 degree in y */
   for (j=1; j<=Frame->back.xdegree; j++){
     /* i = 0, j>0, i2 = 0, j2 = j */
     k++;
     xbuf2 = xmatrix[2*j];
     xbuf = xmatrix[j];
     for (qq=1; qq<=goodpixels; qq++) {
       x[k][1] += xbuf[qq]*zvector[qq]/zsigma[qq];
       d[k][k] += xbuf2[qq]/zsigma[qq];
     }
     /* run over non diagonal terms of d */
     l=k;
     /* first the l values corresponding to 0 degree in y */
     for (j2=j+1; j2<=Frame->back.xdegree; j2++){
       /* i = 0, j>0, i2 = 0, j2>j */
       l++;
       xbuf = xmatrix[j+j2];
       for (qq=1; qq<=goodpixels; qq++)
     d[k][l] += xbuf[qq]/zsigma[qq];
     }
     /* run over l values corresponding to y degree > 0 */
     for (i2=1; i2<=Frame->back.ydegree; i2++){
       /* i = 0, j > 0, i2>i, j2 = 0 */
       l++;
       xbuf = xmatrix[j];
       ybuf = ymatrix[i2];
       for (qq=1; qq<=goodpixels; qq++) 
     d[k][l] += xbuf[qq]*ybuf[qq]/zsigma[qq];
       for (j2=1; j2<=Frame->back.xdegree; j2++){
     /* i = 0, j>0, i2>i, j2>0 */
     l++;
     xbuf = xmatrix[j+j2];
     for (qq=1; qq<=goodpixels; qq++)
       d[k][l] += xbuf[qq]*ybuf[qq]/zsigma[qq];
       }
     }
   }
 
   /* run over k values corresponding to y degree > 0 */
   for (i=1; i<=Frame->back.ydegree; i++){
     /* i>0, j = 0, i2=i, j2=0 */
     k++;
     ybuf = ymatrix[i];
     ybuf2 = ymatrix[2*i];
     for (qq=1; qq<=goodpixels; qq++) {
       x[k][1] += ybuf[qq]*zvector[qq]/zsigma[qq];
       d[k][k] += ybuf2[qq]/zsigma[qq];
     }
     l=k;
     for (j2=1; j2<=Frame->back.xdegree; j2++) {
       /* i>0, j=0, i2=i, j2>0 */
       l++;
       xbuf = xmatrix[j2];
       for (qq=1; qq<=goodpixels; qq++)
     d[k][l] += ybuf2[qq]*xbuf[qq]/zsigma[qq];
     }
     /* run over terms with i2>i */
     for (i2=i+1; i2<=Frame->back.ydegree; i2++){
       /* i>0, j=0, i2>i, j2=0 */
       l++;
       ybuf = ymatrix[i+i2];
       for (qq=1; qq<=goodpixels; qq++) 
     d[k][l] += ybuf[qq]/zsigma[qq];
       for (j2=1; j2<=Frame->back.xdegree; j2++){
     /* i>0, j=0, i2>i, j2>0 */
     l++;
     xbuf = xmatrix[j2];
       for (qq=1; qq<=goodpixels; qq++)
         d[k][l] += ybuf[qq]*xbuf[qq]/zsigma[qq];
       }
     }
     for (j=1; j<=Frame->back.xdegree; j++){
       /* i>0, j>0, i2=i, j2=j */
       k++;
       xbuf = xmatrix[j];
       xbuf2 = xmatrix[2*j];
       ybuf = ymatrix[i];
       ybuf2 = ymatrix[2*i];
       for (qq=1; qq<=goodpixels; qq++) {
     x[k][1] += ybuf[qq]*xbuf[qq]*zvector[qq]/zsigma[qq];
     d[k][k] += xbuf2[qq]*ybuf2[qq]/zsigma[qq];
       }
       l=k;
       for (j2=j+1; j2<=Frame->back.xdegree; j2++){
     /* i>0, j>0, i2=i, j2>j */
     l++;
     xbuf = xmatrix[j+j2];
     for (qq=1; qq<=goodpixels; qq++)
       d[k][l] += xbuf[qq]*ybuf2[qq]/zsigma[qq];
       }
       for (i2=i+1; i2<=Frame->back.ydegree; i2++){
     /* i>0, j>0, i2>i, j2=0 */
     l++;
     xbuf = xmatrix[j];
     ybuf = ymatrix[i+i2];
     for (qq=1; qq<=goodpixels; qq++)
       d[k][l] += xbuf[qq]*ybuf[qq]/zsigma[qq];
     for (j2=1; j2<=Frame->back.xdegree; j2++){
       /* i>0, j>0, i2>i, j2>0 */
       l++;
       xbuf = xmatrix[j+j2];
       for (qq=1; qq<=goodpixels; qq++)
         d[k][l] += xbuf[qq]*ybuf[qq]/zsigma[qq];
     }
       }
     }
   }

   /* now take advantage of the simmetry */
   for (k=1; k<=npar; k++) {
     for (l=k+1; l<=npar; l++) {
       d[l][k] = d[k][l];
     }
   }

   /* perform the actual fit */
   dgaussj(d,npar,x,1);

   /* compute the fitted points on the whole points set */
   /* zero degree terms first */
   /* i = j = 0 */
   for (qq=1; qq<=goodpixels; qq++) zfitted[qq] = x[1][1];
   k=1;
   for (j=1; j<=Frame->back.xdegree; j++) {
     /* i = 0, j>0 */
     k++;
     xbuf = xmatrix[j];
     for(qq=1; qq<=goodpixels; qq++) zfitted[qq] += x[k][1]*xbuf[qq];
   }
   for (i=1; i<=Frame->back.ydegree; i++) {
     /* i>0, j=0 */
     k++;
     ybuf = ymatrix[i];
     for (qq=1; qq<=goodpixels; qq++) zfitted[qq] += x[k][1]*ybuf[qq];
     for (j=1; j<=Frame->back.xdegree; j++) {
       /* i>0, j>0 */
       k++;
       xbuf = xmatrix[j];
       for (qq=1; qq<=goodpixels; qq++) 
     zfitted[qq] += x[k][1]*xbuf[qq]*ybuf[qq];
     }
   }
   /* select and count the pixels which pass the sigma clipping */
   /* compute normalised deviations */
   for (qq=1; qq<=goodpixels; qq++) {
     zdiff = zfitted[qq]-zvector[qq];
     zdevs[qq].zdev = (zdiff*zdiff)/(kappa2*zsigma[qq]);
     zdevs[qq].zind = qq;
   }
   /* sort deviations */
   qsort(zdevs+1, (size_t) goodpixels, sizeof(deviation), devcomp);
   /* initialise the whole mask first */
   for (qq=1; qq<=goodpixels; qq++) zmask[qq]=TRUE;
   /* now run over the worst pixels and mark the bad ones */
   newgoodpixels = goodpixels;
   for (qq=1; (qq<=maxrejects)&&(zdevs[qq].zdev>1); qq++) {
     zmask[zdevs[qq].zind]=FALSE;
     newgoodpixels--;
   }
   /* build the new arrays with the remaining good pixels */
   if (newgoodpixels != goodpixels) {

       if ( strcmp(drs_verbosity,"LOW") == 0 ){
       } else {

           sprintf(output, "sigma-clipped %d pixels on %d",
         goodpixels-newgoodpixels, goodpixels);
           SCTPUT(output);
      }
     /* allocate the new arrays */
     xmatrixnew = dmatrix(1,2*Frame->back.xdegree,1,newgoodpixels);
     ymatrixnew = dmatrix(1,2*Frame->back.ydegree,1,newgoodpixels);
     zvectornew = dvector(1,newgoodpixels);
     zsigmanew = dvector(1,newgoodpixels);
     newgoodpixels = 0;
     for (qq=1; qq<=goodpixels; qq++) {
       if (zmask[qq] == TRUE) {
     newgoodpixels++;
     for (i=1; i<=2*Frame->back.ydegree; i++)
       ymatrixnew[i][newgoodpixels] = ymatrix[i][qq];
     for (j=1; j<=2*Frame->back.xdegree; j++)
       xmatrixnew[j][newgoodpixels] = xmatrix[j][qq];
     zvectornew[newgoodpixels] = zvector[qq];
     zsigmanew[newgoodpixels] = zsigma[qq];
       }
     }

     /* free the old arrays */
     free_dmatrix(xmatrix,1,2*Frame->back.xdegree,1,goodpixels);
     xmatrix = xmatrixnew;
     free_dmatrix(ymatrix,1,2*Frame->back.ydegree,1,goodpixels);
     ymatrix = ymatrixnew;
     free_dvector(zvector,1,goodpixels);
     zvector = zvectornew;
     free_dvector(zsigma,1,goodpixels);
     zsigma = zsigmanew;

     free_dvector(zfitted,1,goodpixels);
     free(zdevs+1);
     free_cvector(zmask,1,goodpixels);
     freed_sw=1;
   }
   if ( strcmp(drs_verbosity,"LOW") == 0 ){
   } else {
      sprintf(output, "background fitting iteration %d finished\n", niters);
      SCTPUT(output);
   }
 } while ((newgoodpixels!=goodpixels) && (niters<maxbackiters));


 /* free the arrays holding the points to be used for the fit */
 free_dmatrix(xmatrix, 1,2*Frame->back.xdegree,1,goodpixels);
 free_dmatrix(ymatrix, 1,2*Frame->back.xdegree,1,goodpixels);
 free_dvector(zvector, 1,goodpixels);
 free_dvector(zsigma, 1,goodpixels);

 if(freed_sw != 1) free_dvector(zfitted, 1,goodpixels);
 if(freed_sw != 1) free_cvector(zmask, 1,goodpixels);
 if(freed_sw != 1) free(zdevs+1);

 if(!(Frame->back.coeff)) Frame->back.coeff=dvector(1,npar);

 for (k=1; k<=npar; k++) Frame->back.coeff[k]=x[k][1];

 /* free locally allocated arrays */
 free_dmatrix(x,1,npar,1,1);
 free_dmatrix(d,1,npar,1,npar);

 return 0;
}


static int 
devcomp(const void *d1, const void *d2)
{
   const deviation *zdev1= (const deviation*) d1;
   const deviation *zdev2= (const deviation*) d2;
  if ((*zdev1).zdev > (*zdev2).zdev) return -1;
  else if ((*zdev1).zdev < (*zdev2).zdev) return 1;
  else return 0;
}

static int 
pixelcomp(const void *p1, const void *p2)
{
   const pixel* pixel1 = (const pixel*) p1;
   const pixel* pixel2 = (const pixel*) p2;

  if (pixel1->value > pixel2->value) return -1;
  else if (pixel1->value < pixel2->value) return 1;
  else return 0;
}
