/* $Id: kmo_priv_sky_tweak.c,v 1.12 2013-09-13 09:10:28 erw Exp $
 *
 * This file is part of the KMOS Pipeline
 * Copyright (C) 2002,2003 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

/*
 * $Author: erw $
 * $Date: 2013-09-13 09:10:28 $
 * $Revision: 1.12 $
 * $Name: not supported by cvs2svn $
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

/*-----------------------------------------------------------------------------
 *                              Includes
 *----------------------------------------------------------------------------*/

#include <math.h>
#include <string.h>
#include <stdio.h>

#include <cpl.h>

#include "kmclipm_constants.h"
#include "kmclipm_priv_splines.h"
#include "kmclipm_math.h"

#include "kmo_error.h"
#include "kmo_priv_sky_tweak.h"
#include "kmo_priv_lcorr.h"
#include "kmo_utils.h"

#define nrerror printf
#define TINY 1.0e-10
#define NMAX 5000
#define GET_PSUM \
                  for (j=1;j<=ndim;j++) {\
                       for (sum=0.0,i=1;i<=mpts;i++) sum += p[i][j];\
                       psum[j]=sum;}
#define SWAP(a,b) {swap=(a);(a)=(b);(b)=swap;}

void amoeba(double **p, double y[], int ndim, double ftol,
        double (*funk)(double []), int *nfunk)
{
        double amotry(double **p, double y[], double psum[], int ndim,
                double (*funk)(double []), int ihi, double fac);
        int i,ihi,ilo,inhi,j,mpts=ndim+1;
        double rtol, sum, swap, ysave, ytry, *psum = NULL;

        psum=vector(ndim+1);
        *nfunk=0;
        GET_PSUM
        for (;;) {
                ilo=1;
                ihi = y[1]>y[2] ? (inhi=2,1) : (inhi=1,2);
                for (i=1;i<=mpts;i++) {
                        if (y[i] <= y[ilo]) ilo=i;
                        if (y[i] > y[ihi]) {
                                inhi=ihi;
                                ihi=i;
                        } else if (y[i] > y[inhi] && i != ihi) inhi=i;
                }
                rtol=2.0*fabs(y[ihi]-y[ilo])/(fabs(y[ihi])+fabs(y[ilo])+TINY);
                if (rtol < ftol) {
                        SWAP(y[1],y[ilo])
                        for (i=1;i<=ndim;i++) SWAP(p[1][i],p[ilo][i])
                        break;
                }
                if (*nfunk >= NMAX) nrerror("NMAX exceeded\n");
                *nfunk += 2;
                ytry=amotry(p,y,psum,ndim,funk,ihi,-1.0);
                if (ytry <= y[ilo])
                        ytry=amotry(p,y,psum,ndim,funk,ihi,2.0);
                else if (ytry >= y[inhi]) {
                        ysave=y[ihi];
                        ytry=amotry(p,y,psum,ndim,funk,ihi,0.5);
                        if (ytry >= ysave) {
                                for (i=1;i<=mpts;i++) {
                                        if (i != ilo) {
                                                for (j=1;j<=ndim;j++)
                                                        p[i][j]=psum[j]=0.5*(p[i][j]+p[ilo][j]);
                                                y[i]=(*funk)(psum);
                                        }
                                }
                                *nfunk += ndim;
                                GET_PSUM
                        }
                } else --(*nfunk);
        }
        free_vector(psum);
}

double amotry(double **p, double y[], double psum[], int ndim,
    double (*funk)(double []), int ihi, double fac)
{
    int j;
    double fac1,fac2,ytry,*ptry=NULL;

    ptry=vector(ndim+1);
    fac1=(1.0-fac)/ndim;
    fac2=fac1-fac;
    for (j=1;j<=ndim;j++) ptry[j]=psum[j]*fac1-p[ihi][j]*fac2;
    ytry=(*funk)(ptry);
    if (ytry < y[ihi]) {
        y[ihi]=ytry;
        for (j=1;j<=ndim;j++) {
            psum[j] += ptry[j]-p[ihi][j];
            p[ihi][j]=ptry[j];
        }
    }
    free_vector(ptry);
    return ytry;
}

#undef SWAP
#undef GET_PSUM
#undef NMAX

int    cont_region_size,
       line_region_size;
double *cont_sky_intensities=NULL,
       *line_sky_intensities=NULL,
       *cont_object_intensities=NULL,
       *line_object_intensities=NULL,
       *cont_lambda=NULL,
       *line_lambda=NULL;

double fitsky(double *p) {
    double result = 0.;
    double *cont_sky = NULL,
           *cont_object = NULL;
    double avg = 0.;
    int ix;

    cont_sky = polynomial_irreg_irreg_nonans(cont_region_size, cont_lambda,
            cont_sky_intensities,
            line_region_size, line_lambda, 1);
    cont_object = polynomial_irreg_irreg_nonans(cont_region_size, cont_lambda,
            cont_object_intensities,
            line_region_size, line_lambda, 1);
    for (ix=0; ix<line_region_size; ix++) {
        double diff = (line_object_intensities[ix] - cont_object[ix]) -
                      (line_sky_intensities[ix] - cont_sky[ix]) * p[1];
        avg += diff * diff;
    }
    avg /= line_region_size;
    result = sqrt(avg);

    if (cont_sky != NULL)    { free_vector(cont_sky); }
    if (cont_object != NULL) { free_vector(cont_object); }

    return result;
}

int spectrum_size;
double *spectrum_lambda=NULL;
double *spectrum_value=NULL;
double *thermal_background=NULL;

#define HC_K 14387.7512979   // h*c/k in [micrometer*K]
#define PLANCK(lambda,t) (pow((lambda),-5.0) / (exp(HC_K/((lambda) * fabs(t))) - 1.))

double fitbkd(double *p) {
    double result = 0.;
    double *tmp = NULL;
    double max    = 0.;
    int i = 0;

    tmp = cpl_malloc(spectrum_size * sizeof(double));

    max = -1.;
    for (i=0; i<spectrum_size; i++) {
        tmp[i] = PLANCK(spectrum_lambda[i], p[3]);
        if (tmp[i] > max) {
            max = tmp[i];
        }
    }

    if (max > 0.) {
        for (i=0; i<spectrum_size; i++) {
            thermal_background[i] = p[1] + tmp[i] / max * fabs(p[2]);
        }
    } else {
        for (i=0; i<spectrum_size; i++) {
            thermal_background[i] = tmp[i];
        }
    }

    result = 0.;
    for (i=0; i<spectrum_size; i++) {
        result += (spectrum_value[i] - thermal_background[i]) *
                  (spectrum_value[i] - thermal_background[i]);
    }

    if (tmp != NULL) cpl_free(tmp);
    return result;
}

cpl_bivector* kmo_get_thermal_background(cpl_bivector* spectrum, int remove_it) {

    cpl_bivector *result = NULL;
    cpl_vector *spectrum_v = NULL,
            *thermal_v  = NULL,
            *tmp_v      = NULL;
    const int ndim = 3;
    const int nsimplex = ndim + 1;
    double **p = NULL;
    double p_init[ndim+1];
    double  min = 0.,
            max = 0.,
            tmp = 0.,
            limit = 0.,
            diff = 0.;
    double *lspectrum = NULL,
           *vspectrum = NULL;
    double       *y_amoeba  = NULL,
            *x_amoeba  = NULL;
    int new_size = 0,
            niter    = 0;
    int i = 0,
        j = 0,
        ix = 0,
        loop = 0;

    KMO_TRY {
        KMO_TRY_EXIT_IF_NULL(
                lspectrum = cpl_bivector_get_x_data(spectrum));
        KMO_TRY_EXIT_IF_NULL(
                vspectrum = cpl_bivector_get_y_data(spectrum));

        KMO_TRY_EXIT_IF_NULL(
                spectrum_lambda = cpl_malloc(cpl_bivector_get_size(spectrum) * sizeof(double)));
        KMO_TRY_EXIT_IF_NULL(
                spectrum_value = cpl_malloc(cpl_bivector_get_size(spectrum) * sizeof(double)));
        KMO_TRY_EXIT_IF_NULL(
                thermal_background = cpl_malloc(cpl_bivector_get_size(spectrum) * sizeof(double)));

        min = +1.e30;
        new_size = 0;
        int skip = 11;
        for (ix=0; ix < cpl_bivector_get_size(spectrum); ix++) {
            if (vspectrum[ix] != 0.0) {
                break;
            }
        }
        for (i=ix+skip; i<cpl_bivector_get_size(spectrum); i++) {
            if ( (vspectrum[i] != 0.0) && (kmclipm_is_nan_or_inf(vspectrum[i]) ==0) ) {
                spectrum_lambda[new_size] = lspectrum[i];
                spectrum_value[new_size] = vspectrum[i];
                if (vspectrum[i] < min) { min = vspectrum[i]; }
                new_size++;
            }
        }
        for (ix=new_size; ix >= 0; ix--) {
            if (vspectrum[ix] != 0.0) {
                break;
            }
        }
        spectrum_size = ix-skip;

        KMO_TRY_EXIT_IF_NULL(
                y_amoeba = cpl_malloc((nsimplex+1) * sizeof(double)));
        KMO_TRY_EXIT_IF_NULL(
                x_amoeba = cpl_malloc((ndim+1) * sizeof(double)));
        p = matrix(nsimplex+1,ndim+1);

        p_init[1] = min;
        p_init[2] = spectrum_value[spectrum_size-1];
        p_init[3] = 280.;
        p[1][1] = p_init[1];
        p[1][2] = p_init[2];
        p[1][3] = p_init[3];

        for (loop=0; loop<5; loop++) {
            for (i=2; i<nsimplex+1; i++) {
                for (j=1; j<ndim+1; j++) {
                    p[i][j] = p[1][j];
                }
            }
            for (i=2; i<nsimplex+1; i++) {
                p[i][i-1] = p[i][i-1] * 1.2;
            }
            for (i=1; i<nsimplex+1; i++) {
                for (j=1; j<ndim+1; j++) {
                    x_amoeba[j] = p[i][j];
                }
                y_amoeba[i] = fitbkd(x_amoeba);
            }

            amoeba(p, y_amoeba, 3, 1.e-5, fitbkd, &niter  );

            KMO_TRY_EXIT_IF_NULL(
                    spectrum_v = cpl_vector_wrap(spectrum_size, spectrum_value));
            KMO_TRY_EXIT_IF_NULL(
                    thermal_v = cpl_vector_wrap(spectrum_size, thermal_background));
            KMO_TRY_EXIT_IF_NULL(
                    tmp_v =  cpl_vector_duplicate(spectrum_v));
            KMO_TRY_EXIT_IF_ERROR(
                    cpl_vector_subtract(tmp_v, thermal_v));
            limit = cpl_vector_get_median(tmp_v) + (2. - loop * .25) *  cpl_vector_get_stdev(tmp_v);
            cpl_vector_delete(tmp_v);
            cpl_vector_unwrap(spectrum_v);
            cpl_vector_unwrap(thermal_v);

            min = +1.e30;
            new_size = 0;
            for (i=0; i<spectrum_size; i++) {
                if (loop < 2) {
                    diff = fabs(spectrum_value[i] - thermal_background[i]);
                } else{
                    diff = spectrum_value[i] - thermal_background[i];
                }
                if (diff < limit) {
                    spectrum_lambda[new_size] = spectrum_lambda[i];
                    spectrum_value[new_size] = spectrum_value[i];
                    if (spectrum_value[i] < min) {
                        min = spectrum_value[i];
                    }
                    new_size++;
                }
            }
            spectrum_size = new_size;
        }

        max = -1.e30;
        for (i=0; i<spectrum_size; i++) {
            tmp = PLANCK(spectrum_lambda[i], p[1][3]);
            if (tmp > max) {
                max = tmp;
            }
        }

        spectrum_size = cpl_bivector_get_size(spectrum);
        for (i=0; i<spectrum_size; i++) {
            spectrum_lambda[i] = lspectrum[i];
            spectrum_value[i] = vspectrum[i];
            tmp = PLANCK(spectrum_lambda[i], p[1][3]);
            thermal_background[i] = p[1][1] + tmp / max * p[1][2];
            if (remove_it) {
                vspectrum[i] -= thermal_background[i];
            }
        }
        KMO_TRY_EXIT_IF_NULL(
                thermal_v = cpl_vector_wrap(spectrum_size, thermal_background));
        KMO_TRY_EXIT_IF_NULL(
                tmp_v =  cpl_vector_duplicate(thermal_v));
        cpl_vector_unwrap(thermal_v);

        KMO_TRY_EXIT_IF_NULL(
                result = cpl_bivector_wrap_vectors(
                        cpl_vector_duplicate(cpl_bivector_get_x_const(spectrum)),
                        tmp_v));
    }
    KMO_CATCH {
        KMO_CATCH_MSG();
        if (result != NULL) {cpl_bivector_delete(result);}
        result = NULL;
    }
    if (p != NULL) { free_matrix(p, 5); }
    if (x_amoeba != NULL) { cpl_free(x_amoeba); }
    if (y_amoeba != NULL) { cpl_free(y_amoeba); }
    if (spectrum_lambda != NULL) { cpl_free(spectrum_lambda); }
    if (spectrum_value != NULL) { cpl_free(spectrum_value); }
    if (thermal_background != NULL) { cpl_free(thermal_background); }

    return result;
}

double get_average_disregarding_outliers (const cpl_vector *vdata) {

    double avg = 0./0.;

    int nr_data,
        i,
        nr_i;
    double median,
           stdev,
           clip;
    cpl_vector *tmpv1 = NULL,
               *tmpv2 = NULL,
               *tmpv3 = NULL,
               *tmpv4 = NULL;
    const double  *data=NULL;
    double        *tmpv1_data=NULL;

    KMO_TRY {

        nr_data = cpl_vector_get_size(vdata);
        KMO_TRY_EXIT_IF_NULL(
                data = cpl_vector_get_data_const(vdata));

        KMO_TRY_EXIT_IF_NULL(
                tmpv1 = cpl_vector_new(nr_data));
        KMO_TRY_EXIT_IF_NULL(
                tmpv1_data = cpl_vector_get_data(tmpv1));

        median = cpl_vector_get_median_const(vdata);
        for (i=0; i<nr_data; i++) {
            tmpv1_data[i] = fabs(data[i] - median);
        }
        KMO_TRY_EXIT_IF_ERROR(
                cpl_vector_sort(tmpv1, CPL_SORT_ASCENDING));
        clip = cpl_vector_get(tmpv1, (int) .8*nr_data);
        KMO_TRY_EXIT_IF_NULL(
                tmpv2 = kmo_idl_where(tmpv1, clip*5., le));
        KMO_TRY_EXIT_IF_NULL(
                tmpv3 = kmo_idl_values_at_indices(vdata, tmpv2));
        median = cpl_vector_get_median_const(tmpv3);
        stdev = cpl_vector_get_stdev(tmpv3);
        nr_i = 0;
        for (i=0; i<nr_data; i++) {
            if ((data[i] < median + 3. * stdev) && (data[i] > median - 3. * stdev)) {
                tmpv1_data[nr_i] = data[i];
                nr_i++;
            }
        }
        KMO_TRY_EXIT_IF_NULL(
                tmpv4 = cpl_vector_wrap(nr_i, tmpv1_data));
        avg = cpl_vector_get_mean(tmpv4);
    }
    KMO_CATCH {
        KMO_CATCH_MSG();
    }
    if (tmpv4 != NULL) { cpl_vector_unwrap(tmpv4); }
    if (tmpv3 != NULL) { cpl_vector_delete(tmpv3); }
    if (tmpv2 != NULL) { cpl_vector_delete(tmpv2); }
    if (tmpv1 != NULL) { cpl_vector_delete(tmpv1); }

    return avg;
}

cpl_error_code kmo_priv_sky_tweak_get_spectra(const cpl_imagelist *object,
                                         const cpl_imagelist *sky,
                                         const cpl_vector *lambda,
                                         const cpl_image *mask,
                                         const int no_nans,
                                         cpl_bivector **obj_spectrum_ptr,
                                         cpl_bivector **sky_spectrum_ptr) {
    cpl_error_code err = CPL_ERROR_NONE;
    cpl_bivector *obj_spectrum = NULL,
                 *sky_spectrum = NULL;
    cpl_vector *intobj = NULL,
               *intsky = NULL;
    const cpl_image    *oimg = NULL,
                       *simg = NULL;
    double *intobj_data=NULL,
           *intsky_data=NULL;
    int nx = 0,
        ny = 0,
        nz = 0;
    double *vo = NULL,
           *vs = NULL;
    int ix, iy, iz, ni;
    int found = 0,
        nr_valid_int = 0;
    int m_is_rejected, o_is_rejected, s_is_rejected;
    double mpix, opix, spix;
    cpl_vector *ovec=NULL,
               *svec=NULL;


    KMO_TRY {
        nx = cpl_image_get_size_x(cpl_imagelist_get_const(object, 0));
        ny = cpl_image_get_size_y(cpl_imagelist_get_const(object, 0));
        nz = cpl_imagelist_get_size(object);
        KMO_TRY_CHECK_ERROR_STATE();

        {
            int snx = 0,
                    sny = 0,
                    snz = 0;


            snx = cpl_image_get_size_x(cpl_imagelist_get_const(sky, 0));
            sny = cpl_image_get_size_y(cpl_imagelist_get_const(sky, 0));
            snz = cpl_imagelist_get_size(sky);
            KMO_TRY_CHECK_ERROR_STATE();

            KMO_TRY_ASSURE(nx == snx && ny == sny && nz ==snz,
                    CPL_ERROR_ILLEGAL_INPUT,
                    "Dimensions of object and sky cubes must fit");
        }

        KMO_TRY_ASSURE(nz == cpl_vector_get_size(lambda),
                CPL_ERROR_ILLEGAL_INPUT,
                "lambda dimensions of input cubes doesn't match with header");

        KMO_TRY_EXIT_IF_NULL(
                intobj = cpl_vector_new(cpl_vector_get_size(lambda)));
        KMO_TRY_EXIT_IF_NULL(
                intsky = cpl_vector_new(cpl_vector_get_size(lambda)));
        KMO_TRY_EXIT_IF_NULL(
                intobj_data = cpl_vector_get_data(intobj));
        KMO_TRY_EXIT_IF_NULL(
                intsky_data = cpl_vector_get_data(intsky));

        KMO_TRY_EXIT_IF_NULL(
                vo = cpl_malloc(nx * ny * sizeof(double)));
        KMO_TRY_EXIT_IF_NULL(
                vs = cpl_malloc(nx * ny * sizeof(double)));

        nr_valid_int = 0;
        for (iz = 0; iz <nz; iz++) {
            KMO_TRY_EXIT_IF_NULL(
                    oimg = cpl_imagelist_get_const(object, iz));
            KMO_TRY_EXIT_IF_NULL(
                    simg = cpl_imagelist_get_const(sky, iz));
            found = 0;
            for (ix = 1; ix<=nx; ix++) {
                for (iy = 1; iy<=ny; iy++) {
                    mpix = cpl_image_get(mask, ix, iy, &m_is_rejected);
                    opix = cpl_image_get(oimg, ix, iy, &o_is_rejected);
                    spix = cpl_image_get(simg, ix, iy, &s_is_rejected);
                    KMO_TRY_CHECK_ERROR_STATE();
                    if ( mpix > .5 &&
                         m_is_rejected == 0 &&
                         o_is_rejected == 0 &&
                         s_is_rejected == 0 ) {
                        vo[found] = opix;
                        vs[found] = spix;
                        found++;
                    }
                }
            }
            if (found >= nx*ny/4.) {
                nr_valid_int++;
                KMO_TRY_EXIT_IF_NULL(
                        ovec = cpl_vector_wrap(found, vo));
                KMO_TRY_EXIT_IF_NULL(
                        svec = cpl_vector_wrap(found, vs));

                if (found < nx*ny/2. ) {
                    intobj_data[iz] = cpl_vector_get_median(ovec);
                    intsky_data[iz] = cpl_vector_get_median(svec);
                } else {
                    intobj_data[iz] = get_average_disregarding_outliers(ovec);
                    intsky_data[iz] = get_average_disregarding_outliers(svec);
                    KMO_TRY_CHECK_ERROR_STATE();
                }

                cpl_vector_unwrap(ovec);
                cpl_vector_unwrap(svec);

            } else {
                intobj_data[iz] = 0./0.;
                intsky_data[iz] = 0./0.;
            }
        }

        if (no_nans) {
            cpl_vector *new_lambda=NULL,
                       *new_intobj=NULL,
                       *new_intsky=NULL;
            double *new_lambda_d=NULL,
                   *new_intobj_d=NULL,
                   *new_intsky_d=NULL;
            const double *lambda_d=NULL;

            KMO_TRY_EXIT_IF_NULL(
                    new_lambda = cpl_vector_new(nr_valid_int));
            KMO_TRY_EXIT_IF_NULL(
                    new_intobj = cpl_vector_new(nr_valid_int));
            KMO_TRY_EXIT_IF_NULL(
                    new_intsky = cpl_vector_new(nr_valid_int));
            KMO_TRY_EXIT_IF_NULL(
                    new_lambda_d = cpl_vector_get_data(new_lambda));
            KMO_TRY_EXIT_IF_NULL(
                    new_intobj_d = cpl_vector_get_data(new_intobj));
            KMO_TRY_EXIT_IF_NULL(
                    new_intsky_d = cpl_vector_get_data(new_intsky));
            KMO_TRY_EXIT_IF_NULL(
                    lambda_d = cpl_vector_get_data_const(lambda));

            ni = 0;
            for (iz=0 ; iz<nz; iz++) {
                if ((! isnan(intobj_data[iz])) && ni < nr_valid_int) {
                    new_lambda_d[ni] = lambda_d[iz];
                    new_intobj_d[ni] = intobj_data[iz];
                    new_intsky_d[ni] = intsky_data[iz];
                    ni++;
                }
            }
            KMO_TRY_EXIT_IF_NULL(
                    obj_spectrum = cpl_bivector_wrap_vectors(
                            new_lambda, new_intobj));
            KMO_TRY_EXIT_IF_NULL(
                    sky_spectrum = cpl_bivector_wrap_vectors(
                            cpl_vector_duplicate(new_lambda), new_intsky));

            if (intobj != NULL) {cpl_vector_delete(intobj); intobj = NULL;}
            if (intsky != NULL) {cpl_vector_delete(intsky); intsky = NULL;}

        } else {
            KMO_TRY_EXIT_IF_NULL(
                    obj_spectrum = cpl_bivector_wrap_vectors(
                            cpl_vector_duplicate(lambda), intobj));
            KMO_TRY_EXIT_IF_NULL(
                    sky_spectrum = cpl_bivector_wrap_vectors(
                            cpl_vector_duplicate(lambda), intsky));
        }
    }
    KMO_CATCH {
        KMO_CATCH_MSG();
    }

    if (vo != NULL) { cpl_free(vo); }
    if (vs != NULL) { cpl_free(vs); }

    *obj_spectrum_ptr = obj_spectrum;
    *sky_spectrum_ptr = sky_spectrum;
    return err;
}

cpl_bivector* kmo_priv_sky_tweak_correct_vibrational_trans(
                                        cpl_bivector *obj_spectrum,
                                        cpl_bivector *sky_spectrum
                                        /*, int line_full_width*/)
{

    cpl_bivector *result = NULL;

    const double lambda_boundaries[] = {1.025, 1.067,  1.125,  1.196,  1.252, 1.289,  1.400,
                                  1.472, 1.5543, 1.6356, 1.7253, 1.840, 1.9570, 2.095,
                                  2.30, 2.460};
    const int nr_boundaries = sizeof(lambda_boundaries) / sizeof(double);
    const char *labels[] = {"4-1 transitions", "5-2 transitions", "6-3 transitions",
                            "7-4 transitions", " 02 transitions", "8-5 transitions",
                            "2-0 transitions", "3-1 transitions", "4-2 transitions",
                            "5-3 transitions", "6-4 transitions", "7-5 transitions",
                            "8-6 transitions", "9-7 transitions", "final bit"};
    double scalings[nr_boundaries];

    int ix,
        il,
        nr_lambda,
        nr_transitions,
        nr_samples;
    double median = 0.,
           stdev  = 0.;
    const double *lambda = NULL,
                 *intobj = NULL,
                 *intsky = NULL;
    double *tmp1_l = NULL,
           *tmp1_o = NULL,
           *tmp1_s = NULL,
           *tmp3_s = NULL,
           *tmp4_d = NULL,
           *line_interpolate_object=NULL,
           *line_interpolate_sky=NULL,
           *flineres=NULL;
    cpl_vector *tmp1_l_v = NULL,
               *tmp1_o_v = NULL,
               *tmp1_s_v = NULL,
//               *tmp2_l_v = NULL,
//               *tmp2_o_v = NULL,
               *tmp2_s_v = NULL,
               *tmp3_s_v = NULL,
               *tmp4_v = NULL;/*,
               *tmp_i_v = NULL*/;

    KMO_TRY {
        KMO_TRY_ASSURE(obj_spectrum != NULL && sky_spectrum != NULL,
                CPL_ERROR_NULL_INPUT,
                "Not all input data is provided!");

        nr_lambda = cpl_bivector_get_size(obj_spectrum);
        KMO_TRY_ASSURE(nr_lambda == cpl_bivector_get_size(sky_spectrum),
                CPL_ERROR_ILLEGAL_INPUT,
                "Size of object and sky spectrum doesn't match");

        KMO_TRY_EXIT_IF_NULL(
                lambda = cpl_vector_get_data_const(cpl_bivector_get_x_const(obj_spectrum)));
        KMO_TRY_EXIT_IF_NULL(
                intobj = cpl_vector_get_data_const(cpl_bivector_get_y_const(obj_spectrum)));
        KMO_TRY_EXIT_IF_NULL(
                intsky = cpl_vector_get_data_const(cpl_bivector_get_y_const(sky_spectrum)));

        for (ix=0; ix<nr_boundaries; ix++) {
            scalings[ix] = 0.;
        }

        for (ix=0; ix<nr_lambda; ix++) {
            double tol = 1.e-7;
            KMO_TRY_ASSURE(
                    fabs(lambda[ix] - cpl_vector_get(cpl_bivector_get_x_const(sky_spectrum), ix))
                    < tol,
                    CPL_ERROR_ILLEGAL_INPUT,
                    "lambda axis of object and sky spectrum doesn't match");
        }

        KMO_TRY_EXIT_IF_NULL(
                tmp1_l_v = cpl_vector_new(nr_lambda));
        KMO_TRY_EXIT_IF_NULL(
                tmp1_o_v = cpl_vector_new(nr_lambda));
        KMO_TRY_EXIT_IF_NULL(
                tmp1_s_v = cpl_vector_new(nr_lambda));
        KMO_TRY_EXIT_IF_NULL(
                tmp1_l = cpl_vector_get_data(tmp1_l_v));
        KMO_TRY_EXIT_IF_NULL(
                tmp1_o = cpl_vector_get_data(tmp1_o_v));
        KMO_TRY_EXIT_IF_NULL(
                tmp1_s = cpl_vector_get_data(tmp1_s_v));

        char dbg_message[4096];
        dbg_message[0] = '\0';
        char *tmp_dbg_message = NULL;
        sprintf(dbg_message, "> ");
        nr_transitions = 0;
        for (ix = 0; ix<nr_boundaries-1; ix++) {
            tmp_dbg_message = cpl_sprintf("Checking %2d: %s", ix, labels[ix]);
            strcat(dbg_message, tmp_dbg_message);
            cpl_free(tmp_dbg_message); tmp_dbg_message = NULL;

            nr_samples = 0;
            for (il=0; il<nr_lambda; il++) {
                if ((lambda[il] >= lambda_boundaries[ix] ) &&
                    (lambda[il] < lambda_boundaries[ix+1]) ) {
                    tmp1_l[nr_samples] = lambda[il];
                    tmp1_o[nr_samples] = intobj[il];
                    tmp1_s[nr_samples] = intsky[il];
                    nr_samples++;
                }
            }
            tmp_dbg_message = cpl_sprintf("   Found %d samples", nr_samples);
            strcat(dbg_message, tmp_dbg_message);
            cpl_free(tmp_dbg_message); tmp_dbg_message = NULL;

            if (nr_samples != 0) {
//                KMO_TRY_EXIT_IF_NULL(
//                        tmp2_l_v = cpl_vector_extract(tmp1_l_v,0, nr_samples-1,1));
//                KMO_TRY_EXIT_IF_NULL(
//                        tmp2_o_v = cpl_vector_extract(tmp1_o_v,0, nr_samples-1,1));
//                KMO_TRY_EXIT_IF_NULL(
//                        tmp2_s_v = cpl_vector_extract(tmp1_s_v,0, nr_samples-1,1));
               KMO_TRY_EXIT_IF_NULL(
                        tmp2_s_v = cpl_vector_wrap(nr_samples, tmp1_s));
                median = cpl_vector_get_median_const(tmp2_s_v);
                stdev = cpl_vector_get_stdev(tmp2_s_v);
                KMO_TRY_EXIT_IF_NULL(
                        tmp3_s_v = cpl_vector_duplicate(tmp2_s_v));
                cpl_vector_unwrap(tmp2_s_v);
                KMO_TRY_EXIT_IF_NULL(
                        tmp3_s = cpl_vector_get_data(tmp3_s_v));
                int i;
                for (i=0; i<cpl_vector_get_size(tmp3_s_v); i++) {
                    if (tmp1_s[i] > median+stdev) {
                        tmp3_s[i] =10.;
                    } else {
                        tmp3_s[i] = 0;
                    }
                }

//                cpl_vector *tmp3_v;
//                KMO_TRY_EXIT_IF_NULL(
//                        tmp4_v = cpl_vector_filter_median_create(tmp3_s_v,line_full_width/2));
                KMO_TRY_EXIT_IF_NULL(
                        tmp4_v = cpl_vector_filter_lowpass_create(tmp3_s_v,CPL_LOWPASS_LINEAR,2));
                if (tmp3_s_v != NULL) { cpl_vector_delete(tmp3_s_v); tmp3_s_v = NULL;}
//                KMO_TRY_EXIT_IF_NULL(
//                        tmp3_v = cpl_vector_new(line_full_width/2));
//                KMO_TRY_EXIT_IF_ERROR(
//                        cpl_vector_fill(tmp3_v, 1.));
//                KMO_TRY_EXIT_IF_ERROR(
//                        cpl_vector_convolve_symmetric(tmp3_s_v, tmp3_v));
//                cpl_vector_delete(tmp3_v);
                KMO_TRY_EXIT_IF_NULL(
                        tmp4_d = cpl_vector_get_data(tmp4_v));

// make sure cont/line regions start and end with a continuum region

                int first_cont_region = -1;
                int last_cont_region = -1;
                int previous_line_region = 0;
                cont_region_size = 0;
                line_region_size = 0;
                for (i=0; i<cpl_vector_get_size(tmp4_v); i++) {
                    if (tmp4_d[i] > 0.) {
                        if (first_cont_region == -1) {
                            continue;   //regions have to start with continuum
                        }
                        previous_line_region++;
                    } else {
                        if (first_cont_region == -1) {
                            first_cont_region = i;
                        }
                        last_cont_region = i;
                        cont_region_size++;
                        line_region_size += previous_line_region;
                        previous_line_region = 0;
                    }
                }
                tmp_dbg_message = cpl_sprintf(" cont region size: %d,  first line region size: %d",
                                              cont_region_size, line_region_size);
                strcat(dbg_message, tmp_dbg_message);
                cpl_free(tmp_dbg_message); tmp_dbg_message = NULL;

                if (line_region_size == 0) {
                    tmp_dbg_message = cpl_sprintf(
                  ", either no line region found or a single which is not surrounded by continuum");
                    strcat(dbg_message, tmp_dbg_message);
                    cpl_free(tmp_dbg_message); tmp_dbg_message = NULL;
                    cpl_msg_debug(__func__, "%s", dbg_message);
                    dbg_message[0] = '\0';
                    continue;
                }

// allocate arrays to hold concatenated line and continuum regions

                KMO_TRY_EXIT_IF_NULL(
                        cont_sky_intensities = cpl_malloc(cont_region_size * sizeof(double)));
                KMO_TRY_EXIT_IF_NULL(
                        cont_object_intensities = cpl_malloc(cont_region_size * sizeof(double)));
                KMO_TRY_EXIT_IF_NULL(
                        cont_lambda = cpl_malloc(cont_region_size * sizeof(double)));
                KMO_TRY_EXIT_IF_NULL(
                        line_sky_intensities = cpl_malloc(line_region_size * sizeof(double)));
                KMO_TRY_EXIT_IF_NULL(
                        line_object_intensities = cpl_malloc(line_region_size * sizeof(double)));
                KMO_TRY_EXIT_IF_NULL(
                        line_lambda = cpl_malloc(line_region_size * sizeof(double)));

// sort input spectra to line and continuum arrays (for this spectrum range)

                int cx = 0;
                int lx = 0;
                for(i=first_cont_region; i<=last_cont_region; i++) {
                    if (tmp4_d[i] > 0.) {
                        line_sky_intensities[lx] = tmp1_s[i];
                        line_object_intensities[lx] = tmp1_o[i];
                        line_lambda[lx] = tmp1_l[i];
                        lx++;
                    } else {
                        cont_sky_intensities[cx] = tmp1_s[i];
                        cont_object_intensities[cx] = tmp1_o[i];
                        cont_lambda[cx] = tmp1_l[i];
                        cx++;
                    }
                }

// minimize object - sky difference (line regions - interpolated background) using a single factor

                const int ndim = 1;
                const int nsimplex = ndim + 1;
                double **p = NULL;
                double *x_amoeba  = NULL,
                       *y_amoeba  = NULL;
                int niter;
                int ip,jp;
                double scale;
                KMO_TRY_EXIT_IF_NULL(
                        y_amoeba = cpl_malloc((nsimplex+1) * sizeof(double)));
                KMO_TRY_EXIT_IF_NULL(
                        x_amoeba = cpl_malloc((ndim+1) * sizeof(double)));
                p = matrix(nsimplex+1,ndim+1);

                p[1][1] = 1.;
                p[2][1] = 1.25;
                for (ip=1; ip<nsimplex+1; ip++) {
                    for (jp=1; jp<ndim+1; jp++) {
                        x_amoeba[jp] = p[ip][jp];
                    }
                    y_amoeba[ip] = fitsky(x_amoeba);
                }


                amoeba(p, y_amoeba, ndim, 1.e-5, fitsky, &niter  );
                scale = p[1][1];

                KMO_TRY_EXIT_IF_NULL(
                        flineres = cpl_malloc(line_region_size * sizeof(double)));

                line_interpolate_object = polynomial_irreg_irreg_nonans(cont_region_size, cont_lambda,
                        cont_object_intensities,
                        line_region_size, line_lambda, 2);

                line_interpolate_sky = polynomial_irreg_irreg_nonans(cont_region_size, cont_lambda,
                        cont_sky_intensities,
                        line_region_size, line_lambda, 2);

                for (i=0; i<line_region_size; i++) {
                    flineres[i] = (line_object_intensities[i] - line_interpolate_object[i]) -
                            (line_sky_intensities[i] - line_interpolate_sky[i]) * scale;
                }

                cpl_vector *tmpx_v = NULL;
                KMO_TRY_EXIT_IF_NULL(
                         tmpx_v = cpl_vector_wrap(line_region_size, flineres));
                median = cpl_vector_get_median_const(tmpx_v);
                stdev = cpl_vector_get_stdev(tmpx_v);
                if (tmpx_v != NULL) { cpl_vector_unwrap(tmpx_v); tmpx_v = NULL;}

                int clip_cnt=0;
                for (i=0; i<line_region_size; i++) {
                    if ( fabs(flineres[i] - median) <= (3 * stdev) ) {
                        clip_cnt++;
                    }
                }
                tmp_dbg_message = cpl_sprintf("  outliers: %d", line_region_size-clip_cnt);
                strcat(dbg_message, tmp_dbg_message);
                cpl_free(tmp_dbg_message); tmp_dbg_message = NULL;

                if ((clip_cnt != line_region_size) && (clip_cnt >= 3)) {
                    lx = 0;
                    for (i=0; i<line_region_size; i++) {
                        if ( fabs(flineres[i] - median) <= (3 * stdev) ) {
                            line_sky_intensities[lx]    = line_sky_intensities[i];
                            line_object_intensities[lx] = line_object_intensities[i];
                            line_lambda[lx]             = line_lambda[i];
                            lx++;
                        }
                    }
                    line_region_size = lx;
                    tmp_dbg_message = cpl_sprintf("   second line region size: %d", line_region_size);
                    strcat(dbg_message, tmp_dbg_message);
                    cpl_free(tmp_dbg_message); tmp_dbg_message = NULL;

                    p[1][1] = 1.;
                    p[2][1] = 1.5;
                    for (ip=1; ip<nsimplex+1; ip++) {
                        for (jp=1; jp<ndim+1; jp++) {
                            x_amoeba[jp] = p[ip][jp];
                        }
                        y_amoeba[ip] = fitsky(x_amoeba);
                    }
                    amoeba(p, y_amoeba, ndim, 1.e-5, fitsky, &niter  );
                    scale = p[1][1];

                }
                tmp_dbg_message = cpl_sprintf("   scale: %f",scale);
                strcat(dbg_message, tmp_dbg_message);
                cpl_free(tmp_dbg_message); tmp_dbg_message = NULL;
                scalings[ix] = scale;

                if (tmp4_v != NULL) { cpl_vector_delete(tmp4_v);  tmp4_v = NULL; }
                if (p != NULL) { free_matrix(p, nsimplex+1); p = NULL; }
                if (x_amoeba != NULL) { cpl_free(x_amoeba); x_amoeba = NULL; }
                if (y_amoeba != NULL) { cpl_free(y_amoeba); y_amoeba = NULL; }
                if (cont_sky_intensities != NULL) {
                    cpl_free(cont_sky_intensities); cont_sky_intensities = NULL; }
                if (line_sky_intensities != NULL) {
                    cpl_free(line_sky_intensities); line_sky_intensities = NULL; }
                if (cont_object_intensities != NULL) {
                    cpl_free(cont_object_intensities); cont_object_intensities = NULL; }
                if (line_object_intensities != NULL) {
                    cpl_free(line_object_intensities); line_object_intensities = NULL; }
                if (cont_lambda != NULL) {
                    cpl_free(cont_lambda); cont_lambda = NULL; }
                if (line_lambda != NULL) {
                    cpl_free(line_lambda); line_lambda = NULL; }
                if (flineres != NULL) {
                    cpl_free(flineres); flineres = NULL; }
                if (line_interpolate_object != NULL) {
                    cpl_free(line_interpolate_object); line_interpolate_object = NULL; }
                if (line_interpolate_sky != NULL) {
                    cpl_free(line_interpolate_sky); line_interpolate_sky = NULL; }

                nr_transitions++;
            } // if (nr_samples != 0)
            cpl_msg_debug(__func__, "%s", dbg_message);
            dbg_message[0] = '\0';
        } // end for (ix = 0; ix<nr_boundaries-1; ix++)

// check for outliers in the scaling factors
        int nr_valid = 0,
            v_ix = 0;
        double valid_scales[nr_boundaries];
        cpl_vector *valid_scales_v = NULL;
        for (ix=0; ix<nr_boundaries; ix++) {
            if (scalings[ix] != 0.0 ) {
                valid_scales[nr_valid] = scalings[ix];
                nr_valid++;
            }
        }
        KMO_TRY_EXIT_IF_NULL(
                valid_scales_v = cpl_vector_wrap(nr_valid, valid_scales));
        median = cpl_vector_get_median_const(valid_scales_v);
        stdev = cpl_vector_get_stdev(valid_scales_v);
        v_ix = 0;
        for (ix=0; ix<nr_boundaries; ix++) {
            if (scalings[ix] != 0.0 ) {
                if (fabs(valid_scales[v_ix] - median) > 2 * stdev) {
                    scalings[ix] = 0.;
                }
                v_ix++;
            }
        }
        if (valid_scales_v != NULL) {cpl_vector_unwrap(valid_scales_v); valid_scales_v = NULL; }

        int scale0_size;
        double *scale0_lambda = NULL,
               *scale0_value = NULL,
               *scale = NULL;
        cpl_vector *scale_v = NULL;

        KMO_TRY_EXIT_IF_NULL(
                scale0_lambda = cpl_malloc(nr_lambda * sizeof(double)));
        KMO_TRY_EXIT_IF_NULL(
                scale0_value = cpl_malloc(nr_lambda * sizeof(double)));

        scale0_size = 0;
        for (il=0; il<nr_lambda; il++) {
            for (ix = 0; ix<nr_boundaries-1; ix++) {
                if (scalings[ix] != 0.) {
                    if ((lambda[il] >= lambda_boundaries[ix] ) &&
                            (lambda[il] < lambda_boundaries[ix+1]) ) {
                        scale0_lambda[scale0_size] = lambda[il];
                        scale0_value[scale0_size] = scalings[ix];
                        scale0_size++;
                    }
                }
            }
        }

        scale = polynomial_irreg_irreg_nonans(scale0_size, scale0_lambda, scale0_value,
                    nr_lambda, lambda, 1);

        KMO_TRY_EXIT_IF_NULL(
                scale_v = cpl_vector_wrap(nr_lambda, scale));

        if (scale0_lambda != NULL) {cpl_free(scale0_lambda); scale0_lambda = NULL; }
        if (scale0_value != NULL) {cpl_free(scale0_value); scale0_value = NULL; }

        KMO_TRY_EXIT_IF_NULL(
                result = cpl_bivector_wrap_vectors(
                        cpl_vector_duplicate(cpl_bivector_get_x_const(obj_spectrum)),
                        scale_v));


    }
    KMO_CATCH {
        KMO_CATCH_MSG();
    }
    if (tmp1_l_v != NULL) { cpl_vector_delete(tmp1_l_v); }
    if (tmp1_o_v != NULL) { cpl_vector_delete(tmp1_o_v); }
    if (tmp1_s_v != NULL) { cpl_vector_delete(tmp1_s_v); }
    if (tmp4_v != NULL) { cpl_vector_delete(tmp4_v);  tmp4_v = NULL; }

    return result;
}

cpl_imagelist* kmo_priv_sky_tweak (cpl_imagelist *object, cpl_imagelist *sky,
        const cpl_propertylist *header, float min_frac, int tbsub) {

    cpl_imagelist *result = NULL;

//    int line_full_width = 8;
    cpl_image *mask = NULL;
    cpl_bivector *obj1_spectrum = NULL,
                 *obj2_spectrum = NULL,
                 *sky_spectrum = NULL,
                 *thermal_background = NULL,
                 *vscales = NULL;
     cpl_vector *lambda = NULL;
     cpl_imagelist *modified_sky = NULL;
     int nx = 0,
        ny = 0,
        nz = 0,
        ix = 0,
        kx = 0,
        kmax = 0;

    KMO_TRY {
        KMO_TRY_ASSURE(object != NULL && sky != NULL && header != NULL,
                CPL_ERROR_NULL_INPUT,
                "Not all input data is provided!");

        nx = cpl_image_get_size_x(cpl_imagelist_get_const(object, 0));
        ny = cpl_image_get_size_y(cpl_imagelist_get_const(object, 0));
        nz = cpl_imagelist_get_size(object);
        KMO_TRY_CHECK_ERROR_STATE();

        {
            int snx = 0,
                sny = 0,
                snz = 0;

            snx = cpl_image_get_size_x(cpl_imagelist_get_const(sky, 0));
            sny = cpl_image_get_size_y(cpl_imagelist_get_const(sky, 0));
            snz = cpl_imagelist_get_size(sky);
            KMO_TRY_CHECK_ERROR_STATE();

            KMO_TRY_ASSURE(nx == snx && ny == sny && nz ==snz,
                    CPL_ERROR_ILLEGAL_INPUT,
                    "Dimensions of object and sky cubes must fit");
        }

        KMO_TRY_EXIT_IF_NULL(
                lambda = kmo_lcorr_create_lambda_vector(header));
        KMO_TRY_ASSURE(nz == cpl_vector_get_size(lambda),
                CPL_ERROR_ILLEGAL_INPUT,
                "lambda dimensions of input cubes doesn't match with header");

        KMO_TRY_EXIT_IF_NULL(
                mask = kmo_lcorr_create_object_mask(object, min_frac, NULL, NULL));

        KMO_TRY_EXIT_IF_ERROR(
                kmo_priv_sky_tweak_get_spectra(object, sky, lambda, mask, 1,
                                                &obj1_spectrum, &sky_spectrum));

        int remove_it = 1;
        KMO_TRY_EXIT_IF_NULL(
                thermal_background = kmo_get_thermal_background(obj1_spectrum, remove_it));

        KMO_TRY_EXIT_IF_NULL(
                vscales = kmo_priv_sky_tweak_correct_vibrational_trans(obj1_spectrum,
                                                                       sky_spectrum/*,
                                                                       8*/));

        KMO_TRY_EXIT_IF_NULL(
                result = cpl_imagelist_duplicate(object));
        KMO_TRY_EXIT_IF_NULL(
                modified_sky = cpl_imagelist_duplicate(sky));

        kmax = cpl_vector_get_size(cpl_bivector_get_x_const(obj1_spectrum));
        KMO_TRY_CHECK_ERROR_STATE();
        kx = 0;
        for (ix=0; ix<nz; ix++) {
            if ( ( kx < kmax ) &&
                 ( cpl_vector_get(lambda,ix) ==
                   cpl_vector_get(cpl_bivector_get_x_const(obj1_spectrum),kx) ) ) {
        // subtract thermal background from new sky cube
                KMO_TRY_EXIT_IF_ERROR(
                        cpl_image_subtract_scalar(
                                cpl_imagelist_get(modified_sky,ix),
                                cpl_vector_get(cpl_bivector_get_y_const(thermal_background), kx)
                        ));
                KMO_TRY_CHECK_ERROR_STATE();
        // multiply new sky cube with scaling factors from OH line fitting
                KMO_TRY_EXIT_IF_ERROR(
                        cpl_image_multiply_scalar(
                                cpl_imagelist_get(modified_sky,ix),
                                cpl_vector_get(cpl_bivector_get_y_const(vscales), kx)
                        ));
                KMO_TRY_CHECK_ERROR_STATE();
        // subtract new sky cube from object cube copy to get new object cube
                KMO_TRY_EXIT_IF_ERROR(
                        cpl_image_subtract(
                                cpl_imagelist_get(result, ix),
                                cpl_imagelist_get(modified_sky,ix)
                        ));
                KMO_TRY_CHECK_ERROR_STATE();

  		// subtract thermal background from new object cube as well
                if (tbsub) {
                    KMO_TRY_EXIT_IF_ERROR(
                            cpl_image_subtract_scalar(
                                    cpl_imagelist_get(result, ix),
                                    cpl_vector_get(cpl_bivector_get_y_const(thermal_background), kx)
                            ));
                    KMO_TRY_CHECK_ERROR_STATE();
                }

                kx++;
            }
        }

    }
    KMO_CATCH {
        KMO_CATCH_MSG();
        if (result != NULL) {cpl_imagelist_delete(result);}
        result = NULL;
    }

    if (modified_sky != NULL) { cpl_imagelist_delete(modified_sky); }
    if (vscales != NULL) { cpl_bivector_delete(vscales); }
    if (thermal_background != NULL) { cpl_bivector_delete(thermal_background); }
    if (mask != NULL) { cpl_image_delete(mask); }
    if (lambda != NULL) { cpl_vector_delete(lambda); }
    if (obj1_spectrum != NULL) { cpl_bivector_delete(obj1_spectrum); }
    if (obj2_spectrum != NULL) { cpl_bivector_delete(obj2_spectrum); }
    if (sky_spectrum != NULL) { cpl_bivector_delete(sky_spectrum); }
    return result;

}
