/*                                                                            *
 *   This file is part of the ESO X-shooter Pipeline                          *
 *   Copyright (C) 2006 European Southern Observatory                         *
 *                                                                            *
 *   This library is free software; you can redistribute it and/or modify     *
 *   it under the terms of the GNU General Public License as published by     *
 *   the Free Software Foundation; either version 2 of the License, or        *
 *   (at your option) any later version.                                      *
 *                                                                            *
 *   This program is distributed in the hope that it will be useful,          *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of           *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the            *
 *   GNU General Public License for more details.                             *
 *                                                                            *
 *   You should have received a copy of the GNU General Public License        *
 *   along with this program; if not, write to the Free Software              *
 *   Foundation, 51 Franklin St, Fifth Floor, Boston, MA  02111-1307  USA     *
 *                                                                            */

/* 
 * $Author: amodigli $
 * $Date: 2013-10-13 07:37:56 $
 * $Revision: 1.122 $
*/

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif


/*----------------------------------------------------------------------------*/
/**
 * @defgroup xsh_subtract_sky_single  Subtract Sky in single frame
 * @ingroup drl_functions
 *
 * Function ...
 */
/*----------------------------------------------------------------------------*/
/**@{*/

/*-----------------------------------------------------------------------------
  Includes
  ----------------------------------------------------------------------------*/

#include <math.h>
#include <hdrl.h>

#include <cpl.h>

#include <gsl/gsl_bspline.h>
#include <gsl/gsl_linalg.h>
#include <gsl/gsl_rng.h>
#include <gsl/gsl_statistics_double.h>
#include <gsl/gsl_multifit.h>

#include <xsh_drl.h>
#include <xsh_drl_check.h>
#include <xsh_badpixelmap.h>
#include <xsh_data_pre.h>
#include <xsh_data_order.h>
#include <xsh_data_wavemap.h>
#include <xsh_data_localization.h>
#include <xsh_data_rec.h>
#include <xsh_dfs.h>
#include <xsh_pfits.h>
#include <xsh_error.h>
#include <xsh_utils.h>
#include <xsh_utils_image.h>
#include <xsh_utils_table.h>
#include <xsh_msg.h>
#include <xsh_fit.h>
#include <xsh_model_r250.h>
#include <xsh_efficiency_response.h>

#define HAVE_INLINE 1
#define REGDEBUG_MEDIAN_SPLINE 0
#define REGDEBUG_BSPLINE 0

/*-----------------------------------------------------------------------------
  Functions prototypes
  -----------------------------------------------------------------------------*/
static xsh_wavemap_list* xsh_wavemap_list_new( cpl_frame *wavemap_frame,
  cpl_frame *slitmap_frame, xsh_localization *list,
  cpl_frame *order_table_frame, xsh_pre *pre_sci, int nbkpts,
  cpl_frame* usr_def_sampl_points,cpl_frame* ref_sky_list,
  cpl_frame* sky_orders_chunks,
  xsh_subtract_sky_single_param* sky_par, xsh_instrument *instrument);
static xsh_rec_list* xsh_wavemap_list_build_sky( xsh_wavemap_list* list,
						 xsh_instrument *instrument);

static int xsh_wave_compare( const void * un, const void * deux ) ;

static cpl_frame* xsh_wavelist_subtract_sky( xsh_pre * pre_sci,
					     xsh_rec_list* sky_list, 
					     xsh_wavemap_list * wave_list,
					     xsh_instrument* instrument,
					     const char* rec_prefix,
					     xsh_subtract_sky_single_param* sky_par);


/* remove duplicate wavelength entries */
static cpl_table*
xsh_table_unique_wave(cpl_table* tab,const char* wname, const char* fname) {

    cpl_table* xtab=NULL;
    xtab=cpl_table_duplicate(tab);
    int norg=cpl_table_get_nrow(xtab);
    //xsh_msg("N orig %d",norg);
    double* pwave=cpl_table_get_data_double(xtab,wname);
    double* pflux=cpl_table_get_data_double(xtab,fname);
    int ninv=1;
    int nrow=0;
    int nbad=0;
    int j=0;

    for(j=0;ninv>0 && j<10;j++) {

        nrow=cpl_table_get_nrow(xtab);
        /*
        if (!(pwave[0] < pwave[1])) {
            cpl_table_set_invalid(xtab,fname,1);
            nbad++;
        }
        */
        for(int i=1;i < nrow; i++) {
            //xsh_msg("j=%d i=%d norg=%d nbad=%d nrow=%d",j,i,norg,nbad,nrow);
            if ((pwave[i-1] == pwave[i])) {
                /*
                if(pwave[i-1]>1665.0 && pwave[i-1]<1665.8) {
                                       xsh_msg("pwave=%g",pwave[i-1]);
                                   }
                                   */
                if( ( pflux[i-1] <= pflux[i] ) && i >1 ) {

                    cpl_table_set_invalid(xtab,fname,i-1);
                } else {
                    cpl_table_set_invalid(xtab,fname,i);
                }
                nbad++;
            }
        }
        /*
        if (!(pwave[nrow-2] < pwave[nrow-1])) {
            cpl_table_set_invalid(xtab,fname,nrow-2);
            nbad++;
        }
        */

        ninv=cpl_table_count_invalid(xtab,fname);
        xsh_msg("iter=%d nrow=%d nbad=%d ninv=%d",j,nrow,nbad,ninv);
        if(ninv>0) {
            cpl_table_erase_invalid(xtab);
        } else {
            break;
        }
    }


    int nnew=cpl_table_get_nrow(xtab);
    xsh_msg("niter %d N orig %d flagged %d expected %d new %d",
            j,norg,nbad,norg-nbad,nnew);
    return xtab;
}
/* not uniform data distribution */
static double *
xsh_bspline_fit_smooth(double* wave, double* flux,const int size_data,
                            double* swave, const int sno,
                            double ron2, double gain, xsh_instrument* inst)
{
    double * sflux=NULL;
    size_t n = 0;
    size_t order = 4;
    size_t ncoeffs = 0;
    size_t nbreak = 0;

    size_t i, j;
    gsl_bspline_workspace *bw;
    gsl_vector *B;
    gsl_vector *Bkpts;
    //double dy;

    gsl_vector *c, *w;
    gsl_vector * x, *y;
    gsl_matrix *X, *cov;
    gsl_multifit_linear_workspace *mw;
    double chisq=0, Rsq=0, dof=0;
    double* pwav = NULL;
    double* pfit = NULL;
    cpl_table* tab_resp_fit = NULL;

    double dump_factor=1.e10;
    double *Bkpts_ptr;

    /* set optimal number of coeffs for each arm */
    if (xsh_instrument_get_arm(inst) == XSH_ARM_UVB) {
        ncoeffs = 21;
    } else if (xsh_instrument_get_arm(inst) == XSH_ARM_VIS) {
        ncoeffs = 16;
    } else if (xsh_instrument_get_arm(inst) == XSH_ARM_NIR) {
        ncoeffs = 12;
    } else {
        xsh_msg("instrument arm not set");
        abort();
    }
    ncoeffs=(wave[0]-wave[n-1])/0.5;
    //xsh_msg("ncoeffs=%d",ncoeffs);
    nbreak = ncoeffs + 2 - order;

    n = size_data ;

    gsl_rng_env_setup();

    /* allocate a cubic bspline workspace (ORDER = order) */
    bw = gsl_bspline_alloc(order, nbreak);

    B = gsl_vector_alloc(ncoeffs);

    Bkpts = gsl_vector_alloc(nbreak);

    x = gsl_vector_alloc(n);
    y = gsl_vector_alloc(n);
    X = gsl_matrix_alloc(n, ncoeffs);
    c = gsl_vector_alloc(ncoeffs);
    w = gsl_vector_alloc(n);

    cov = gsl_matrix_alloc(ncoeffs, ncoeffs);
    mw = gsl_multifit_linear_alloc(n, ncoeffs);

    /* this is the data to be fitted */
    for (i = 0; i < n; ++i) {
        double variance;
        double xi = wave[i];
        double yi = flux[i];

        variance = ron2 * fabs(yi)/gain ;

        gsl_vector_set(x, i, xi);
        if(isnan(yi) || isinf(yi)) {
            gsl_vector_set(y, i, 0);
            gsl_vector_set(w, i, 1.0 / dump_factor);
        } else {
            gsl_vector_set(y, i, yi);
            gsl_vector_set(w, i, 1.0 / variance );
        }
    }
    //xsh_msg("ok5");
    /* use uniform breakpoints on [wave[0], wave[n]] */
    Bkpts_ptr=gsl_vector_ptr(Bkpts,0);
    int nfit_div_nbreak=n/nbreak;

    for(i=0;i<nbreak;i++) {
        Bkpts_ptr[i]=wave[i*nfit_div_nbreak];
    }
    Bkpts_ptr[0]=wave[0];
    Bkpts_ptr[nbreak-1]=wave[n-1];

    gsl_bspline_knots(Bkpts, bw);
    /*
    for(i=0;i<10;i++){
    xsh_msg("x[%d]=%g",i,gsl_vector_get(x,i));
    }
    for(i=n-10;i<n;i++){
        xsh_msg("x[%d]=%g",i,gsl_vector_get(x,i));
    }
    */
    //xsh_bspline_knots_not_uniform(x,bw);
    //xsh_msg("ok1");
    /* construct the fit matrix X */
    for (i = 0; i < n; ++i) {
        double xi = gsl_vector_get(x, i);
        //xsh_msg("xi[%d]=%g",i,xi);
        /* compute B_j(xi) for all j */
        //xsh_msg("ok2");
        gsl_bspline_eval(xi, B, bw);
        //xsh_msg("ok3");
        /* fill in row i of X */
        for (j = 0; j < ncoeffs; ++j) {
            double Bj = gsl_vector_get(B, j);
            //xsh_msg("ok4 Bj=%g",Bj);
            gsl_matrix_set(X, i, j, Bj);
            //xsh_msg("ok5");
        }
        //xsh_msg("ok6");
    }

    //xsh_msg("ok7");
    /* do the fit */
    gsl_multifit_wlinear(X, w, y, c, cov, &chisq, mw);
    //xsh_msg("ok8");
    dof = n - ncoeffs;
    /*
  tss = gsl_stats_wtss(w->data, 1, y->data, 1, y->size);
  Rsq = 1.0 - chisq / tss;
     */
    printf("chisq/dof = %e, Rsq = %f\n", chisq / dof, Rsq);

    tab_resp_fit = cpl_table_new(sno);
    cpl_table_new_column(tab_resp_fit, "wave", CPL_TYPE_DOUBLE);
    cpl_table_new_column(tab_resp_fit, "fit", CPL_TYPE_DOUBLE);
    cpl_table_fill_column_window_double(tab_resp_fit, "wave", 0, sno, 0);
    cpl_table_fill_column_window_double(tab_resp_fit, "fit", 0, sno, 0);
    pwav = cpl_table_get_data_double(tab_resp_fit, "wave");
    pfit = cpl_table_get_data_double(tab_resp_fit, "fit");
    sflux=cpl_calloc(sno, sizeof(double));
    /* output the smoothed curve */
    {
        double xi, yi, yerr;
        //printf("#m=1,S=0\n");
        for (i = 0; i < sno; i++) {
            xi = (double) swave[i];
            gsl_bspline_eval(xi, B, bw);
            gsl_multifit_linear_est(B, c, cov, &yi, &yerr);
            //printf("%f %g\n", xi, yi);
            pwav[i] = xi;
            pfit[i] = yi;
            swave[i]=xi;
            sflux[i]=yi;
        }
    }
    cpl_table_save(tab_resp_fit, NULL, NULL, "resp_fit.fits", CPL_IO_DEFAULT);
    //xsh_msg("ok9");
    //cleanup:
    xsh_free_table(&tab_resp_fit);

    gsl_bspline_free(bw);
    gsl_vector_free(B);
    gsl_vector_free(Bkpts);
    gsl_vector_free(x);
    gsl_vector_free(y);
    gsl_matrix_free(X);
    gsl_vector_free(c);
    gsl_vector_free(w);
    gsl_matrix_free(cov);
    gsl_multifit_linear_free(mw);

    return sflux;
}


static cpl_table*
xsh_detect_outliers_tab(const int order,const int idx, const int nitem,
                 const xsh_wavemap_list* wlist, const float* yf,
                 const int iter)
{
    double* pw=NULL;
    double* pslit=NULL;

    double* pf=NULL;
    double* ps=NULL;
    double* px=NULL;
    double* py=NULL;
    double* pfit=NULL;
    double* prat=NULL;
    double* perr=NULL;
    int * pflag=NULL;
    wavemap_item* pentry=NULL;
    char fname[128];

    cpl_table* otab=cpl_table_new(nitem);
    cpl_table_new_column(otab,"WAVE",CPL_TYPE_DOUBLE);
    cpl_table_new_column(otab,"SLIT",CPL_TYPE_DOUBLE);
    cpl_table_new_column(otab,"FLUX",CPL_TYPE_DOUBLE);
    cpl_table_new_column(otab,"SIGMA",CPL_TYPE_DOUBLE);
    cpl_table_new_column(otab,"FIT",CPL_TYPE_DOUBLE);

    cpl_table_new_column(otab,"ERR",CPL_TYPE_DOUBLE);
    cpl_table_new_column(otab,"RAT",CPL_TYPE_DOUBLE);
    cpl_table_new_column(otab,"X",CPL_TYPE_DOUBLE);
    cpl_table_new_column(otab,"Y",CPL_TYPE_DOUBLE);
    cpl_table_new_column(otab,"FLAG",CPL_TYPE_INT);

    cpl_table_fill_column_window_double(otab,"WAVE",0,nitem,0.);
    cpl_table_fill_column_window_double(otab,"SLIT",0,nitem,0.);
    cpl_table_fill_column_window_double(otab,"FLUX",0,nitem,0.);
    cpl_table_fill_column_window_double(otab,"SIGMA",0,nitem,0.);
    cpl_table_fill_column_window_double(otab,"FIT",0,nitem,0.);
    cpl_table_fill_column_window_double(otab,"ERR",0,nitem,0.);
    cpl_table_fill_column_window_double(otab,"RAT",0,nitem,0.);
    cpl_table_fill_column_window_int(otab,"FLAG",0,nitem,0.);
    cpl_table_fill_column_window_double(otab,"X",0,nitem,0.);
    cpl_table_fill_column_window_double(otab,"Y",0,nitem,0.);

    pw=cpl_table_get_data_double(otab,"WAVE");
    pf=cpl_table_get_data_double(otab,"FLUX");
    pslit=cpl_table_get_data_double(otab,"SLIT");
    ps=cpl_table_get_data_double(otab,"SIGMA");
    pfit=cpl_table_get_data_double(otab,"FIT");
    prat=cpl_table_get_data_double(otab,"RAT");
    perr=cpl_table_get_data_double(otab,"ERR");
    px=cpl_table_get_data_double(otab,"X");
    py=cpl_table_get_data_double(otab,"Y");



    pentry = wlist->list[idx].sky;
    int i;
    /* fill table with values */
    for (i = 0; i < nitem; i++, pentry++) {
        double yerr = 0;
        double yi = yf[i];
        pentry->fitted = yi;
        pentry->fit_err = yerr;

        pw[i] = pentry->lambda;
        pslit[i] = pentry->slit;
        pf[i] = pentry->flux;
        ps[i] = pentry->sigma;
        pfit[i] = pentry->fitted;
        perr[i] = pentry->fit_err;
        prat[i] = pf[i] / pfit[i];
        px[i] = pentry->ix;
        py[i] = pentry->iy;
    }

    cpl_table_duplicate_column(otab,"FLUX_SMOOTH",otab,"FLUX");
    cpl_vector* vec_flux=NULL;
    cpl_vector* vec_smooth=NULL;
    vec_flux = cpl_vector_wrap(nitem,cpl_table_get_data_double(otab,"FLUX"));
    vec_smooth=cpl_vector_filter_lowpass_create(vec_flux,CPL_LOWPASS_LINEAR,1);
    cpl_vector_unwrap(vec_flux);
    double* pts=cpl_table_get_data_double(otab,"FLUX_SMOOTH");
    double* pvs=cpl_vector_get_data(vec_smooth);
    pentry = wlist->list[idx].sky;
    for (i = 0; i < nitem; i++, pentry++) {
        pts[i]=pvs[i];
    }
    cpl_vector_delete(vec_smooth);
    cpl_table_duplicate_column(otab,"FLUX_SMO",otab,"FLUX");
    cpl_table_divide_columns(otab,"FLUX_SMO","FLUX_SMOOTH");



    double mean=0;
    double median=0;
    double min=0;
    double max=0;
    double rms=0;
    double kappa=10;
    int niter=1;
    int n=0;
    int j=0;
    prat=cpl_table_get_data_double(otab,"RAT");
    pflag=cpl_table_get_data_int(otab,"FLAG");

    /* check for bad pix */
    prat=cpl_table_get_data_double(otab,"FLUX_SMO");
    pflag=cpl_table_get_data_int(otab,"FLAG");

    for(i=0 ;i<niter;i++) {
        mean=cpl_table_get_column_mean(otab,"FLUX_SMO");
        median=cpl_table_get_column_median(otab,"FLUX_SMO");
        rms=cpl_table_get_column_stdev(otab,"FLUX_SMO");
        min=cpl_table_get_column_min(otab,"FLUX_SMO");
        max=cpl_table_get_column_max(otab,"FLUX_SMO");
        for(j=0;j<nitem;j++) {
            if( prat[j] < mean-kappa*rms ) {
                cpl_table_set_invalid(otab,"FLUX_SMO",j);
                pflag[j]=-2;
            } else if( prat[j] > mean+kappa*rms ) {
                cpl_table_set_invalid(otab,"FLUX_SMO",j);
                pflag[j]=2;
            }
        }

        n=cpl_table_count_invalid(otab,"FLUX_SMO");
        xsh_msg("mean=%g median=%g rms=%g min=%g max=%g invalid=%d",
                mean,median,rms,min,max,n);

    }


    prat=cpl_table_get_data_double(otab,"RAT");
    /* check for bad fit */
    for(i=0 ;i<niter;i++) {
        mean=cpl_table_get_column_mean(otab,"RAT");
        median=cpl_table_get_column_median(otab,"RAT");
        rms=cpl_table_get_column_stdev(otab,"RAT");
        min=cpl_table_get_column_min(otab,"RAT");
        max=cpl_table_get_column_max(otab,"RAT");
        for(j=0;j<nitem;j++) {
            if( prat[j] < mean-kappa*rms ) {
                cpl_table_set_invalid(otab,"RAT",j);
                pflag[j]=-1;
            } else if( prat[j] > mean+kappa*rms ) {
                cpl_table_set_invalid(otab,"RAT",j);
                pflag[j]=1;
            }
        }

        n=cpl_table_count_invalid(otab,"RAT");
        xsh_msg("mean=%g median=%g rms=%g min=%g max=%g invalid=%d",
                mean,median,rms,min,max,n);

    }


    FILE* fout = NULL;
    xsh_msg_dbg_medium("Output fitted data");
#if REGDEBUG_BSPLINE
    sprintf(fname, "outlier1d_ord_%02d_iter_%02d.reg", order,iter);
    fout = fopen(fname, "w");
    fprintf(fout, "# x y\n");
#endif
    pentry = wlist->list[idx].sky;
    for (i = 0; i < nitem; i++, pentry++) {

        if(pflag[i] == -1 || pflag[i] == 1 )
        {
#if REGDEBUG_BSPLINE
            fprintf(fout,"x point (%f %f) #color=red \n", px[i]+1,py[i]+1);
#endif
            pentry->qual=QFLAG_SKY_MODEL_BAD_FIT;
        }
        if(pflag[i] == -2 || pflag[i] == 2 ) {
#if REGDEBUG_BSPLINE
            fprintf(fout,"x point (%f %f) #color=blue \n",px[i]+1,py[i]+1);
#endif
            pentry->qual=QFLAG_SKY_MODEL_BAD_PIX;
        }

    }


    sprintf(fname,"outlier1d_order_%02d_iter_%02d.fits",order,iter);
    //cpl_table_save(otab,NULL,NULL,fname,CPL_IO_DEFAULT);
    cpl_table_and_selected_int(otab,"FLAG", CPL_EQUAL_TO, 2);
    cpl_table_or_selected_int(otab,"FLAG", CPL_EQUAL_TO, -2);
    cpl_table* xtab=cpl_table_extract_selected(otab);



    cpl_table_erase_column(xtab,"RAT");
    cpl_table_erase_column(xtab,"FLUX_SMOOTH");
    cpl_table_erase_column(xtab,"FLUX_SMO");
    cpl_table_erase_column(xtab,"SIGMA");
    cpl_table_erase_column(xtab,"FIT");
    cpl_table_erase_column(xtab,"FLAG");
    cpl_table_erase_column(xtab,"SLIT");
    cpl_table_erase_column(xtab,"FLUX");
    cpl_table_erase_column(xtab,"ERR");

    /* adjust column names */
    cpl_table_name_column(otab,"WAVE","WAVELENGTH");
    /* make sure there are no 0 */
    cpl_table_and_selected_double(otab,"WAVELENGTH",CPL_EQUAL_TO,0);
    cpl_table_erase_selected(otab);


    //xsh_msg("iter=%d idx=%d",iter,idx);


    xsh_free_table(&otab);
    if ( fout ) fclose( fout ) ;

    return xtab;
}

/**
 * @short determines sky outliers on a 1D spectrum via kappa-sigma clipping
 * @param[in] wlist i/o structure with sky data and its B-spline model info
 * @param[in] order extraction order (relative no)
 *
 * @return[out] table with relevant info on sky, model, flagged pixels.
 *
 */
static cpl_table*
xsh_model_to_table(const xsh_wavemap_list* wlist, const int order)
{
    double* pwave=NULL;
    double* pflux=NULL;
    double* px=NULL;
    double* py=NULL;
    double* perr=NULL;
    double* pslit=NULL;
    int* pqual=NULL;
    int sky_ndata = wlist->list[order].sky_size;

    /* put data in a table */
    cpl_table* otab=cpl_table_new(sky_ndata);
    cpl_table_new_column(otab,"WAVE",CPL_TYPE_DOUBLE);
    cpl_table_new_column(otab,"FLUX",CPL_TYPE_DOUBLE);
    cpl_table_new_column(otab,"ERR",CPL_TYPE_DOUBLE);
    cpl_table_new_column(otab,"QUAL",CPL_TYPE_INT);
    cpl_table_new_column(otab,"SLIT",CPL_TYPE_DOUBLE);
    cpl_table_new_column(otab,"X",CPL_TYPE_DOUBLE);
    cpl_table_new_column(otab,"Y",CPL_TYPE_DOUBLE);

    cpl_table_fill_column_window_double(otab,"WAVE",0,sky_ndata,0.);
    cpl_table_fill_column_window_double(otab,"FLUX",0,sky_ndata,0.);
    cpl_table_fill_column_window_double(otab,"ERR",0,sky_ndata,0.);
    cpl_table_fill_column_window_int(otab,"QUAL",0,sky_ndata,0);
    cpl_table_fill_column_window_double(otab,"SLIT",0,sky_ndata,0.);
    cpl_table_fill_column_window_double(otab,"X",0,sky_ndata,0.);
    cpl_table_fill_column_window_double(otab,"Y",0,sky_ndata,0.);

    pwave=cpl_table_get_data_double(otab,"WAVE");
    pflux=cpl_table_get_data_double(otab,"FLUX");
    perr=cpl_table_get_data_double(otab,"ERR");
    pqual=cpl_table_get_data_int(otab,"QUAL");
    pslit=cpl_table_get_data_double(otab,"SLIT");
    px=cpl_table_get_data_double(otab,"X");
    py=cpl_table_get_data_double(otab,"Y");
    //int decode_bp=wlist->instrument->decode_bp;
    wavemap_item* psky=NULL;
    psky = wlist->list[order].sky;

    /* fill table with values */
    for (int i = 0; i < sky_ndata; i++, psky++) {
        pwave[i] = psky->lambda;
        pflux[i] = psky->flux;
        perr[i] = psky->sigma;
        pqual[i] = psky->qual;
        pslit[i] = psky->slit;
        px[i] = psky->ix;
        py[i] = psky->iy;
    }

    return otab;
}
/**
 * @short determines sky outliers on a 1D spectrum via kappa-sigma clipping
 * @param[in] tab i/o table
 * @param[in] name_i input column name to be smoothed
 * @param[in] name_o output column name with smoothed values
 * @param[in] hsize median filter half window size
 *
 * @return[out] table with relevant info on sky, model, flagged pixels.
 *
 */
static cpl_error_code
xsh_table_column_smooth(cpl_table** tab,
                        const char* name_i, const char* name_o, const int hsize)
{
    /* smooth data to later find bad pixels */
    cpl_table_duplicate_column(*tab,name_o,*tab,name_i);

    cpl_vector* v_flux=NULL;
    cpl_vector* v_smooth=NULL;
    int nrow=cpl_table_get_nrow(*tab);
    v_flux = cpl_vector_wrap(nrow,cpl_table_get_data_double(*tab,name_i));
    v_smooth=cpl_vector_filter_median_create(v_flux,hsize);
    cpl_vector_unwrap(v_flux);
    double* pts=cpl_table_get_data_double(*tab,"FLUX_SMOOTH");

    double* pvs=cpl_vector_get_data(v_smooth);

    for (int i = 0; i < nrow; i++) {
        pts[i]=pvs[i];
    }
    cpl_vector_delete(v_smooth);

    return cpl_error_get_code();
}


static cpl_error_code
xsh_table_column_clip(cpl_table** tab, const char* name_d, const char* name_s,
                      const double kappa, const int niter,
                      const double gain, const double ron2, const int decode_bp)
{
    double median=0;
    int n=0;

    int nrow=cpl_table_get_nrow(*tab);
    cpl_table_new_column(*tab,"FLAG",CPL_TYPE_INT);
    cpl_table_fill_column_window_int(*tab,"FLAG",0,nrow,0);
    int* pflag=cpl_table_get_data_int(*tab,"FLAG");

    //cpl_table_duplicate_column(*tab,"RAT",*tab,"FLUX");
    cpl_table_duplicate_column(*tab,"DIF",*tab,name_d);
    cpl_table_subtract_columns(*tab,"DIF",name_s);
    //CONTROLLA
    cpl_table_duplicate_column(*tab,"SIG",*tab,name_s);
    //cpl_table_duplicate_column(*tab,"SIG",*tab,"ERR");

    cpl_table_abs_column(*tab,"SIG");
    cpl_table_divide_scalar(*tab,"SIG",gain);
    cpl_table_add_scalar(*tab,"SIG",ron2);
    cpl_table_power_column(*tab,"SIG",0.5);
    //cpl_table_divide_columns(*tab,"RAT","FLUX_SMOOTH");
    //prat=cpl_table_get_data_double(*tab,"RAT");
    double* pdif=cpl_table_get_data_double(*tab,"DIF");
    double* perr=cpl_table_get_data_double(*tab,"SIG");
    int* pqual=cpl_table_get_data_int(*tab,"QUAL");
    int ninv=0;
    int n_tot=0;
    int k=0;
    //cpl_table_save(*tab,NULL,NULL,"pippo.fits",CPL_IO_DEFAULT);

    for(int i=0 ;i<niter;i++) {
        median=cpl_table_get_column_median(*tab,"DIF");

        double mean=cpl_table_get_column_mean(*tab,"DIF");
        //xsh_msg("median=%g mean=%g",median,mean);
        //we keep the 1st and last points even if outliers to keep the full
        // wave range
        for (k = 1; k < nrow-1; k++) {
            int check_data_quality = (( ( pqual[k] & decode_bp ) > 0  ) &&
                            ( pqual[k] != QFLAG_SATURATED_DATA )) ? 1:0;
            check_data_quality=0;

            if( (pdif[k] < median-kappa*perr[k] ||
                            pdif[k] > median+kappa*perr[k] )  ||
                            (check_data_quality == 1) ){
                /*
                xsh_msg("median=%g kappa=%g err=%g thresh min=%g max=%g",
                                median,kappa,perr[k],-kappa*perr[k],kappa*perr[k]);
                xsh_msg("xsh dif=%g thresh min=%g max=%g",pdif[k],median-kappa*perr[k],median+kappa*perr[k]);
                */
                cpl_table_set_invalid(*tab,"DIF",k);
                pflag[k]=1;
                ninv++;
            }

            n=cpl_table_count_invalid(*tab,"DIF");
        }
        n_tot+=n;

        //xsh_msg("mean=%g median=%g rms=%g min=%g max=%g invalid=%d",
        //        mean,median,rms,min,max,n);

    }
    xsh_msg("Points flagged as invalid %d vs n_tot %d", ninv,n_tot);

    //exit(0);
    return cpl_error_get_code();
}



/**
 * @short determines sky outliers on a 1D spectrum via kappa-sigma clipping
 * @param[in] wlist i/o structure with sky data and its B-spline model info
 * @param[in] hsize median filter half window size
 * @param[in] ron2  square of RON
 * @param[in] gain gain value
 * @param[in] order extraction order (relative no)
 * @param[in] iter model iteration ID
 *
 * @return[out] table with relevant info on sky, model, flagged pixels.
 *
 */
static cpl_table*
xsh_detect_raw_data_outliers_1d(const xsh_wavemap_list* wlist, const int hsize,
                                const double ron2, const double gain,
                                const int order, const int iter,
                                const int decode_bp)
{

    cpl_table* otab=NULL;
    /* put data in a table */
    otab=xsh_model_to_table(wlist,order);

    // smooth data to later find bad pixels
    xsh_table_column_smooth(&otab,"FLUX", "FLUX_SMOOTH", hsize);
    double kappa=5;
    int niter=10;
    xsh_table_column_clip(&otab, "FLUX", "FLUX_SMOOTH", kappa, niter, gain,
                          ron2, decode_bp);

    /* save outliers location in region files */
    FILE* fout = NULL;

    xsh_msg_dbg_medium("Output fitted data");
    char fname[128];
#if REGDEBUG_BSPLINE
    sprintf(fname, "outlier1d_ord_%02d_iter_%02d.reg", order,iter);
    fout = fopen(fname, "w");
    fprintf(fout, "# x y\n");
#endif
    /* reset the sky to the starting point */
    wavemap_item* psky = wlist->list[order].sky;
    psky = wlist->list[order].sky;

    /*we keep the 1st and last points even if outliers to keep the full
             * wave range */
    double* px=NULL;
    double* py=NULL;
    int * pflag=NULL;
    px=cpl_table_get_data_double(otab,"X");
    py=cpl_table_get_data_double(otab,"Y");
    pflag=cpl_table_get_data_int(otab,"FLAG");
    int* pqual=cpl_table_get_data_int(otab,"QUAL");
    int sky_ndata = wlist->list[order].sky_size;

    for (int i = 1; i < sky_ndata-1; i++,psky++) {
        if( pflag[i] == 1 ) {
#if REGDEBUG_BSPLINE
            fprintf(fout,"x point (%f %f) #color=blue \n",px[i]+1,py[i]+1);
#endif
            psky->qual=QFLAG_SKY_MODEL_BAD_PIX;
            pqual[i]=QFLAG_SKY_MODEL_BAD_PIX;
        }
    }

#if REGDEBUG_BSPLINE
    sprintf(fname,"outlier1d_order_%02d_iter_%02d.fits",order,iter);
    cpl_table_save(otab,NULL,NULL,fname,CPL_IO_DEFAULT);
#endif

    /* keep only good points */
    cpl_table_and_selected_int(otab,"FLAG", CPL_EQUAL_TO, 0);
    cpl_table* xtab=cpl_table_extract_selected(otab);
    int n=cpl_table_get_nrow(xtab);
    xsh_msg("Points remained after flagging outliers %d", n);
    /* as the initial table was oversampled make sure there are no 0 */
    cpl_table_and_selected_double(xtab,"WAVE",CPL_EQUAL_TO,0);
    cpl_table_erase_selected(xtab);
    n=cpl_table_get_nrow(xtab);
#if REGDEBUG_BSPLINE
    sprintf(fname,"tab_clean_order_%02d_iter_%02d.fits",order,iter);
    cpl_table_save(xtab,NULL,NULL,fname,CPL_IO_DEFAULT);
#endif
    xsh_msg("Points remained after removal 0 waves %d", n);

    cpl_table_erase_column(xtab,"SIG");
    xsh_free_table(&otab);
    if ( fout ) fclose( fout ) ;
cleanup:
    return xtab;
}



/**
 * @short determines sky outliers on a 1D spectrum via kappa-sigma clipping
 * @param[in] wlist i/o structure with sky data and its B-spline model info
 * @param[in] hsize median filter half window size
 * @param[in] ron2  square of RON
 * @param[in] gain gain value
 * @param[in] order extraction order (relative no)
 * @param[in] iter model iteration ID
 *
 * @return[out] table with relevant info on sky, model, flagged pixels.
 *
 */
static cpl_table*
xsh_detect_raw_data_outliers_1d_slice(const xsh_wavemap_list* wlist, const int hsize,
                                const double ron2, const double gain,
                                const int order, const int iter,
                                const int decode_bp,
                                const double s1,
                                const double s2, const int slice_id)
{

    cpl_table* otab=NULL;
    /* put data in a table */
    check(otab=xsh_model_to_table(wlist,order));
    xsh_msg("s1=%g s2=%g",s1,s2);
    cpl_table_and_selected_double(otab,"SLIT",CPL_NOT_LESS_THAN,s1);
    cpl_table_and_selected_double(otab,"SLIT",CPL_LESS_THAN,s2);
    //cpl_table_and_selected_double(otab,"SLIT",CPL_NOT_GREATER_THAN,s2);
    cpl_table* stab=NULL;
    stab=cpl_table_extract_selected(otab);
    xsh_free_table(&otab);
    //xsh_msg("ck1 hsize=%d",hsize);
    //cpl_table_dump_structure(stab,stdout);
    // smooth data to later find bad pixels
    check(xsh_table_column_smooth(&stab,"FLUX", "FLUX_SMOOTH", hsize));
    double kappa=5;
    int niter=10;
    check(xsh_table_column_clip(&stab, "FLUX", "FLUX_SMOOTH", kappa, niter, gain,
                          ron2, decode_bp));
    /* save outliers location in region files */
    FILE* fout = NULL;

    xsh_msg_dbg_medium("Output fitted data");
    char fname[128];
#if REGDEBUG_BSPLINE
    sprintf(fname, "outlier1d_ord_%02d_slice_%02d_order_%02d.reg",
            order,slice_id,iter);
    fout = fopen(fname, "w");
    fprintf(fout, "# x y\n");
#endif
    /* reset the sky to the starting point */
    wavemap_item* psky = wlist->list[order].sky;
    psky = wlist->list[order].sky;

    /*we keep the 1st and last points even if outliers to keep the full
             * wave range */
    double* px=NULL;
    double* py=NULL;
    double* pw=NULL;
    int * pflag=NULL;
    int nrow=0;
    nrow=cpl_table_get_nrow(stab);
    px=cpl_table_get_data_double(stab,"X");
    py=cpl_table_get_data_double(stab,"Y");
    pw=cpl_table_get_data_double(stab,"WAVE");
    pflag=cpl_table_get_data_int(stab,"FLAG");
    int* pqual=cpl_table_get_data_int(stab,"QUAL");
    int sky_ndata = wlist->list[order].sky_size;

    for(int k=0; k<nrow; k++) {
        if( pflag[k] == 1 ) {
            psky = wlist->list[order].sky;
            for (int i = 1; i < sky_ndata-1; i++,psky++) {
                if( psky->slit >= s1 && psky->slit <= s2 ) {
                    if( pw[k] == psky->lambda &&
                                    px[k] == psky->ix && py[k] == psky->iy ) {
#if REGDEBUG_BSPLINE
                        fprintf(fout,"x point (%f %f) #color=blue \n",px[k]+1,py[k]+1);
#endif
                        psky->qual=QFLAG_SKY_MODEL_BAD_PIX;
                        pqual[k]=QFLAG_SKY_MODEL_BAD_PIX;
                    }
                }
            }
        }
    }

#if REGDEBUG_BSPLINE
    sprintf(fname,"outlier1d_order_%02d_slice_%02d_iter_%02d.fits",
            order,slice_id,iter);
    cpl_table_save(stab,NULL,NULL,fname,CPL_IO_DEFAULT);
#endif

    /* keep only good points */
    check(cpl_table_and_selected_int(stab,"FLAG", CPL_EQUAL_TO, 0));
    cpl_table* xtab=cpl_table_extract_selected(stab);

    xsh_free_table(&stab);
    int n=cpl_table_get_nrow(xtab);
    xsh_msg("Points remained after flagging outliers %d", n);
    /* as the initial table was oversampled make sure there are no 0 */
    cpl_table_and_selected_double(xtab,"WAVE",CPL_EQUAL_TO,0);
    cpl_table_erase_selected(xtab);
    n=cpl_table_get_nrow(xtab);
#if REGDEBUG_BSPLINE
    sprintf(fname,"tab_clean_order_%02d_slice_%02d_iter_%02d.fits",
            order,slice_id,iter);
    cpl_table_save(xtab,NULL,NULL,fname,CPL_IO_DEFAULT);
#endif
    xsh_msg("Points remained after removal 0 waves %d", n);
    cpl_table_erase_column(xtab,"SIG");

    if ( fout ) fclose( fout ) ;
cleanup:
    return xtab;
}

static cpl_error_code
xsh_dump_sky_fit(const int order,const int idx, const int nitem,
                 const xsh_wavemap_list* wlist, const float* yf,
                 const int iter)
{
    double* pw=NULL;
    double* pslit=NULL;

    double* pf=NULL;
    double* ps=NULL;
    double* px=NULL;
    double* py=NULL;
    double* pfit=NULL;
    double* prat=NULL;
    double* perr=NULL;
    double* pdif=NULL;
    wavemap_item* pentry=NULL;
    char fname[128];


    cpl_table* debug_fit=cpl_table_new(nitem);
    cpl_table_new_column(debug_fit,"WAVE",CPL_TYPE_DOUBLE);
    cpl_table_new_column(debug_fit,"SLIT",CPL_TYPE_DOUBLE);
    cpl_table_new_column(debug_fit,"FLUX",CPL_TYPE_DOUBLE);
    cpl_table_new_column(debug_fit,"SIGMA",CPL_TYPE_DOUBLE);
    cpl_table_new_column(debug_fit,"DIF",CPL_TYPE_DOUBLE);
    cpl_table_new_column(debug_fit,"FIT",CPL_TYPE_DOUBLE);
    cpl_table_new_column(debug_fit,"ERR",CPL_TYPE_DOUBLE);
    cpl_table_new_column(debug_fit,"RAT",CPL_TYPE_DOUBLE);
    cpl_table_new_column(debug_fit,"X",CPL_TYPE_DOUBLE);
    cpl_table_new_column(debug_fit,"Y",CPL_TYPE_DOUBLE);

    cpl_table_fill_column_window_double(debug_fit,"WAVE",0,nitem,0.);
    cpl_table_fill_column_window_double(debug_fit,"SLIT",0,nitem,0.);
    cpl_table_fill_column_window_double(debug_fit,"FLUX",0,nitem,0.);
    cpl_table_fill_column_window_double(debug_fit,"SIGMA",0,nitem,0.);
    cpl_table_fill_column_window_double(debug_fit,"DIF",0,nitem,0.);
    cpl_table_fill_column_window_double(debug_fit,"FIT",0,nitem,0.);
    cpl_table_fill_column_window_double(debug_fit,"ERR",0,nitem,0.);
    cpl_table_fill_column_window_double(debug_fit,"RAT",0,nitem,0.);
    cpl_table_fill_column_window_double(debug_fit,"X",0,nitem,0.);
    cpl_table_fill_column_window_double(debug_fit,"Y",0,nitem,0.);

    pw=cpl_table_get_data_double(debug_fit,"WAVE");
    pf=cpl_table_get_data_double(debug_fit,"FLUX");
    pslit=cpl_table_get_data_double(debug_fit,"SLIT");
    ps=cpl_table_get_data_double(debug_fit,"SIGMA");
    pfit=cpl_table_get_data_double(debug_fit,"FIT");
    pdif=cpl_table_get_data_double(debug_fit,"DIF");
    prat=cpl_table_get_data_double(debug_fit,"RAT");
    perr=cpl_table_get_data_double(debug_fit,"ERR");
    px=cpl_table_get_data_double(debug_fit,"X");
    py=cpl_table_get_data_double(debug_fit,"Y");
    xsh_msg_dbg_medium("Output fitted data");

    FILE* fout = NULL;
#if REGDEBUG_BSPLINE
    sprintf(fname, "sky_fit_%02d_%02d.dat", order,iter);
    fout = fopen(fname, "w");
#endif
    pentry = wlist->list[idx].sky;
    int i;
    for (i = 0; i < nitem; i++, pentry++) {
        double yerr = 0;
        double yi = yf[i];
        pentry->fitted = yi;
        pentry->fit_err = yerr;
#if REGDEBUG_BSPLINE
        fprintf(fout, "%f %f %f %d %d ", pentry->lambda, pentry->flux,
                pentry->sigma, pentry->ix, pentry->iy);
        fprintf(fout, "%lf %lf\n", yi, yerr);
#endif

        pw[i] = pentry->lambda;
        pslit[i] = pentry->slit;
        pf[i] = pentry->flux;
        ps[i] = pentry->sigma;
        pfit[i] = pentry->fitted;
        perr[i] = pentry->fit_err;
        prat[i] = pf[i] / pfit[i];
        pdif[i] = pf[i] - pfit[i];
        px[i] = pentry->ix;
        py[i] = pentry->iy;
    }


#if REGDEBUG_BSPLINE
    sprintf(fname,"sky_fit_%02d_%02d.fits",order,iter);
    cpl_table_save(debug_fit,NULL,NULL,fname,CPL_IO_DEFAULT);
#endif
    double rms =cpl_table_get_column_stdev(debug_fit,"DIF");
    xsh_msg("iter=%d idx=%d rms=%g",iter,idx,rms);

    xsh_free_table(&debug_fit);
    if ( fout ) fclose( fout ) ;
    return cpl_error_get_code();
}

static int
xsh_get_edge_x_min(xsh_order_list* otab, const int iorder, const int iy)
{
    int x_min_offset = -2; /* was +1 */

    int minx = xsh_order_list_eval_int(otab, otab->list[iorder].edglopoly, iy)
                    + x_min_offset;

    return minx;
}

static int
xsh_get_edge_x_max(xsh_order_list* otab, const int iorder, const int iy)
{
    int x_max_offset = -1; /* was -1 */

    int maxx = xsh_order_list_eval_int(otab, otab->list[iorder].edguppoly, iy)
                    + x_max_offset;

    return maxx;
}
static cpl_error_code
xsh_dump_sky_sampl(const int order, const int idx, const int nitem,
                   const xsh_wavemap_list* wlist, gsl_vector *Bkpts,
                   const int nbkpts)
{

    double* pw=NULL;
    double* px=NULL;
    double* py=NULL;
    wavemap_item* pentry=NULL;
    wavemap_item* pentry_start=NULL;
    double *Bkpts_ptr=gsl_vector_ptr(Bkpts,0);
    char fname[128];

    pentry_start = wlist->list[idx].sky;
    int i,j;

#if REGDEBUG_BSPLINE
    FILE* fout = NULL;
    sprintf(fname, "sky_line_sampl_pts_%02d.reg", order);
    fout = fopen(fname, "w");
#endif

    cpl_table* sampl_fit=cpl_table_new(nbkpts);
    cpl_table_new_column(sampl_fit,"WAVE",CPL_TYPE_DOUBLE);
    cpl_table_new_column(sampl_fit,"X",CPL_TYPE_DOUBLE);
    cpl_table_new_column(sampl_fit,"Y",CPL_TYPE_DOUBLE);
    cpl_table_fill_column_window_double(sampl_fit,"WAVE",0,nbkpts,0.);
    cpl_table_fill_column_window_double(sampl_fit,"X",0,nbkpts,0.);
    cpl_table_fill_column_window_double(sampl_fit,"Y",0,nbkpts,0.);
    pw=cpl_table_get_data_double(sampl_fit,"WAVE");
    px=cpl_table_get_data_double(sampl_fit,"X");
    py=cpl_table_get_data_double(sampl_fit,"Y");

    for(i=0;i<nbkpts-1;i++) {
        pw[i]=Bkpts_ptr[i];
        for (j = 0,pentry=pentry_start; j < nitem; j++, pentry++) {
            //xsh_msg("sk4 i=%d nbkpts=%d j=%d nitem=%d",i,nbkpts,j,nitem);
            /*
            xsh_msg("pw=%g  lambda=%g diff=%g",
                            pw[i],pentry->lambda,pw[i]-pentry->lambda);
                            */
            if(pw[i]==pentry->lambda) {
                px[i] = pentry->ix;
                py[i] = pentry->iy;
#if REGDEBUG_BSPLINE
                fprintf(fout, "x point (%g %g) #color=blue \n", px[i]+1, py[i]+1);
#endif
            }
        }
    }
    //exit(0);
#if REGDEBUG_BSPLINE
    sprintf(fname,"sky_lines_sampl_pts.fits");
    if(idx == 0) {
        cpl_table_save(sampl_fit,NULL,NULL,fname,CPL_IO_DEFAULT);
    } else {
        cpl_table_save(sampl_fit,NULL,NULL,fname,CPL_IO_EXTEND);
    }
#endif

    xsh_free_table(&sampl_fit);
#if REGDEBUG_BSPLINE
    if ( fout ) fclose( fout ) ;
#endif
    return cpl_error_get_code();
}

static void
xsh_dump_table_on_region_file(cpl_table* tab, const char* fname, const int symbol){
    //double * w = NULL;
    double * x = NULL;
    double * y = NULL;
    FILE* fout = NULL;
    fout = fopen(fname, "w");
    int nrow = cpl_table_get_nrow(tab);
    //cpl_table_dump_structure(tab,stdout);
    //w=cpl_table_get_data_double(tab,"WAVELENGTH");
    x=cpl_table_get_data_double(tab,"X");
    y=cpl_table_get_data_double(tab,"Y");
    fprintf(fout, "# x y\n");
    switch(symbol){
    case 0:
        for(int i=0;i<nrow;i++) {
            fprintf(fout,"x point (%g %g) #color=green \n",x[i]+1,y[i]+1);
            /*
            fprintf( fout, "point(%g,%g) #point=cross color=green "\
                     "font=\"helvetica 10 normal\" text={%5.1f %5.1f}\n",
                     x[i]+1, y[i]+1, x[i]+1,y[i]+1);
                     */

        }
        break;
    case 1:
        for(int i=0;i<nrow;i++) {
            fprintf(fout,"circle point (%g %g) #color=red \n",x[i]+1,y[i]+1);
            /*
            fprintf( fout, "point(%5g,%5g) #point=cross color=red "\
                     "font=\"helvetica 10 normal\" text={%5.1f %5.1f}\n",
                     x[i]+1, y[i]+1, x[i]+1,y[i]+1);
                     */
        }
        break;
    case 2:
        for(int i=0;i<nrow;i++) {
            fprintf(fout,"diamond point (%g %g) #color=cyan \n",x[i]+1,y[i]+1);
            /*
            fprintf( fout, "point(%g,%g) #point=cross color=cyan "\
                     "font=\"helvetica 10 normal\" text={%5.1f %5.1f}\n",
                     x[i]+1, y[i]+1, x[i]+1,y[i]+1);
                     */
        }
        break;
    case 3:
        for(int i=0;i<nrow;i++) {
            fprintf(fout,"cross point (%g %g) #color=black \n",x[i]+1,y[i]+1);
            /*
            fprintf( fout, "point(%g,%g) #point=cross color=black "\
                     "font=\"helvetica 10 normal\" text={%5.1f %5.1f}\n",
                     x[i]+1, y[i]+1, x[i]+1,y[i]+1);
                     */
        }
        break;
    case 4:
        for(int i=0;i<nrow;i++) {
            fprintf(fout,"point (%g %g) #color=red \n",x[i]+1,y[i]+1);
            /*
            fprintf( fout, "point(%g,%g) #point=cross color=orange "\
                     "font=\"helvetica 10 normal\" text={%5.1f %5.1f}\n",
                     x[i]+1, y[i]+1, x[i]+1,y[i]+1);
                     */
        }
        break;
    case 5:
        for(int i=0;i<nrow;i++) {
            fprintf(fout,"box point (%g %g) #color=blue \n",x[i]+1,y[i]+1);
            /*
            fprintf( fout, "point(%g,%g) #point=cross color=blue "\
                     "font=\"helvetica 10 normal\" text={%5.1f %5.1f}\n",
                     x[i]+1, y[i]+1, x[i]+1,y[i]+1);
            */
        }
        break;
    default:
        for(int i=0;i<nrow;i++) {
            fprintf(fout,"point (%g %g) #color=yellow \n",x[i]+1,y[i]+1);
            fprintf( fout, "point(%g,%g) #point=cross color=yellow "\
                     "font=\"helvetica 10 normal\" text={%5.1f %5.1f}\n",
                     x[i]+1, y[i]+1, x[i]+1,y[i]+1);
        }
        break;
    }

    if ( fout ) fclose( fout ) ;
    return;
}

static cpl_table*
xsh_create_sampl_table(const int iorder, double* wave_ref, const int nref,
                       xsh_wavemap_list* wlist,const int nsampl_2,
                       const char* prefix)
{

    double * w_mod = NULL;
    double * x_mod = NULL;
    double * y_mod = NULL;
    int    * o_mod =NULL;
    wavemap_item* pentry = NULL;
    wavemap_item* pentry_start = NULL;
    wavemap_item* pentry_ref = NULL;
    pentry_start = wlist->list[iorder].sky;
    int nsampl_line=2*nsampl_2+1;
    int nitem = wlist->list[iorder].sky_size;
    double wtol = 10.e-3;//10.e-4;
    int nsampl=nsampl_line*nref;
    char rname[80];
    char tname[80];
    cpl_table* sampl_tab=cpl_table_new(nsampl);
    cpl_table_new_column(sampl_tab,"WAVELENGTH",CPL_TYPE_DOUBLE);
    cpl_table_new_column(sampl_tab,"X",CPL_TYPE_DOUBLE);
    cpl_table_new_column(sampl_tab,"Y",CPL_TYPE_DOUBLE);
    cpl_table_new_column(sampl_tab,"ORDER",CPL_TYPE_INT);
    cpl_table_fill_column_window_double(sampl_tab,"WAVELENGTH",0,nsampl,0.);
    cpl_table_fill_column_window_double(sampl_tab,"X",0,nsampl,0.);
    cpl_table_fill_column_window_double(sampl_tab,"Y",0,nsampl,0.);
    cpl_table_fill_column_window_int(sampl_tab,"ORDER",0,nsampl,0.);
    w_mod=cpl_table_get_data_double(sampl_tab,"WAVELENGTH");
    x_mod=cpl_table_get_data_double(sampl_tab,"X");
    y_mod=cpl_table_get_data_double(sampl_tab,"Y");
    o_mod=cpl_table_get_data_int(sampl_tab,"ORDER");

    int k=0;
    int j=0;
    int m=0;

    double wdif_min_start=9999;
    double wdif_min=wdif_min_start;
    double wdif_max=-wdif_min_start;
    double wdif_tmp=0;
    int found_match=0;
    int k_step=10;
/*
    xsh_msg("nref=%d nitem=%d",nref,nitem);
    xsh_msg("wave_ref[0]=%g wave_ref[nref-1]=%g",
            wave_ref[0],wave_ref[nref-1]);
*/
    xsh_msg("Ref sky tab sampl points=%d sky data sampl points=%d",nref,nitem);
    for(int i = 0; i < nref ; i++ ) {
        //xsh_msg("ok2 i=%d nsampl=%d j=%d nitem=%d m=%d nsampl=%d",i,nrow,j,nitem,m,nsampl);
        wdif_min=wdif_min_start;
        found_match=0;
        for(j = 0, pentry = pentry_start ; j < nitem ; j++, pentry++) {
            /* we loop over the wave sampling space to find the best match
             * to the reference wavelength
             */

            if( fabs( wave_ref[i] - pentry->lambda ) <= wtol ) {
                /* if there is a sampled wavelength close enough to the
                 * reference one, we search for the closest one.
                 */
                //xsh_msg("differenze %g",fabs( wave_ref[i] - pentry->lambda ));
                wdif_tmp=fabs(wave_ref[i] - pentry->lambda);
                if(wdif_tmp<wdif_min) {
                    found_match=1;
                    wdif_min=wdif_tmp;
                    pentry_ref = pentry;
                    /*
                    xsh_msg("wref=%10.9g wmod=%10.9g wdif=%10.9g wdif_min=%10.9g",
                            wave_ref[i],pentry->lambda,wdif_tmp,wdif_min);
                            */

                }
            }
        }
        /* now we found the model wavelength closest to the reference one
         * (wave_ref[i]) we can start to take sampling points around its value
         */
        if(found_match) {
            for(k = -nsampl_2; k <= nsampl_2; k+=k_step) {
                m=i*nsampl_line+k+nsampl_2;
                //xsh_msg("m=%d nsampl=%d",m,nsampl);
                w_mod[m]=(pentry_ref-k)->lambda;
                x_mod[m]=(pentry_ref-k)->ix;
                y_mod[m]=(pentry_ref-k)->iy;
                o_mod[m]=iorder;
            }
        }


    }
    //xsh_msg("m=%d",m);
    xsh_msg("wdif_min=%18.15g wdif_max=%18.15g",wdif_min,wdif_max);
    /* there are cases where the sampl_tabl was over dimensioned.
     * Thus we need to remove empty wavelengths. In our case as wave
     * cannot be less than ~298 we check that it is not less than 200.
     * We also know that result should be smaller than 2500,
     * thus we remove values above 3000.
     */
    cpl_table_and_selected_double(sampl_tab,"WAVELENGTH",CPL_LESS_THAN,200.);
    cpl_table_erase_selected(sampl_tab);
    cpl_table_and_selected_double(sampl_tab,"WAVELENGTH",CPL_GREATER_THAN,3000.);
    cpl_table_erase_selected(sampl_tab);
    check(xsh_sort_table_3(sampl_tab,"WAVELENGTH","Y","X",CPL_TRUE,CPL_TRUE,CPL_TRUE));
    sprintf(rname,"%s.reg",prefix);
    sprintf(tname,"%s.fits",prefix);
    //xsh_msg("rname=%s",rname);
    int nrow=cpl_table_get_nrow(sampl_tab);
    xsh_msg(">>>>>order %d line sampling points %d",iorder,nrow);
    //xsh_msg("tname=%s",tname);
#if REGDEBUG_BSPLINE
    xsh_dump_table_on_region_file(sampl_tab, rname,0);
    cpl_table_save(sampl_tab,NULL,NULL,tname,CPL_IO_DEFAULT);
#endif



     cleanup:
    if(cpl_error_get_code() != CPL_ERROR_NONE) {
         xsh_print_rec_status(0);
         return NULL;
     } else {
         return sampl_tab;
     }


}





static cpl_table*
xsh_create_sampl_random(const int iorder,
                         xsh_wavemap_list* wlist,const int nsampl,
                         const char* prefix)
{

    double * w_mod = NULL;
    double * x_mod = NULL;
    double * y_mod = NULL;
    int    * o_mod =NULL;
    /* we add two points to cover order wmin,wmax */
    int nrow=nsampl+2;
    wavemap_item* pentry = NULL;
    wavemap_item* pentry_start = NULL;
    pentry_start = wlist->list[iorder].sky;
    int nitem = wlist->list[iorder].sky_size;
    char rname[80];
    char tname[80];


    cpl_table* sampl_tab=cpl_table_new(nrow);
    cpl_table_new_column(sampl_tab,"WAVELENGTH",CPL_TYPE_DOUBLE);
    cpl_table_new_column(sampl_tab,"X",CPL_TYPE_DOUBLE);
    cpl_table_new_column(sampl_tab,"Y",CPL_TYPE_DOUBLE);
    cpl_table_new_column(sampl_tab,"ORDER",CPL_TYPE_INT);
    cpl_table_fill_column_window_double(sampl_tab,"WAVELENGTH",0,nrow,0.);
    cpl_table_fill_column_window_double(sampl_tab,"X",0,nrow,0.);
    cpl_table_fill_column_window_double(sampl_tab,"Y",0,nrow,0.);
    cpl_table_fill_column_window_int(sampl_tab,"ORDER",0,nrow,0.);
    w_mod=cpl_table_get_data_double(sampl_tab,"WAVELENGTH");
    x_mod=cpl_table_get_data_double(sampl_tab,"X");
    y_mod=cpl_table_get_data_double(sampl_tab,"Y");
    o_mod=cpl_table_get_data_int(sampl_tab,"ORDER");

    int j=0;
    sprintf(rname,"%s.reg",prefix);
    sprintf(tname,"%s.fits",prefix);
    //xsh_msg("rname=%s",rname);
    //xsh_msg("tname=%s",tname);


    //xsh_msg("nitem=%d",nitem);
    /* generate random distribution of sampling points in range [wmin,wmax] */
    pentry=pentry_start;

    xsh_random_init();

    /*add start and end wave and corresponding x,y points */
    w_mod[0]=pentry->lambda;
    x_mod[0]=pentry->ix;
    y_mod[0]=pentry->iy;
    o_mod[0]=iorder;

    w_mod[nrow-1]=(pentry+nitem-1)->lambda;
    x_mod[nrow-1]=(pentry+nitem-1)->ix;
    y_mod[nrow-1]=(pentry+nitem-1)->iy;
    o_mod[nrow-1]=iorder;
    //xsh_msg("edges=%g %g",pentry->lambda,(pentry+nitem-1)->lambda);
    /* add remaining points */
    xsh_random_init();

    for(int i = 1; i < nrow-1 ; i++ ) {
        j=xsh_get_random_int_window(0, nitem);

        w_mod[i]=(pentry+j)->lambda;
        x_mod[i]=(pentry+j)->ix;
        y_mod[i]=(pentry+j)->iy;
        o_mod[i]=iorder;
    }

    check(xsh_sort_table_3(sampl_tab,"WAVELENGTH","Y","X",CPL_TRUE,CPL_TRUE,CPL_TRUE));
#if REGDEBUG_BSPLINE
    xsh_dump_table_on_region_file(sampl_tab, rname,1);
#endif
    //xsh_msg("No sampling points on order %d %d",wlist->list[iorder].order,nrow);
    //cpl_table_save(sampl_tab,NULL,NULL,tname,CPL_IO_DEFAULT);
    cleanup:
    if(cpl_error_get_code() != CPL_ERROR_NONE) {
         xsh_print_rec_status(0);
         return NULL;
     } else {
         return sampl_tab;
     }


}

static cpl_table*
xsh_create_sampl_uniform_continuum(const int iorder,
                         xsh_wavemap_list* wlist,const int nsampl,
                         const char* prefix)
{

    double * w_mod = NULL;
    double * x_mod = NULL;
    double * y_mod = NULL;
    int    * o_mod =NULL;
    int nrow=nsampl+2;
    wavemap_item* pentry = NULL;
    wavemap_item* pentry_start = NULL;
    pentry_start = wlist->list[iorder].sky;
    int nitem = wlist->list[iorder].sky_size;
    double wtol = 0.01;
    char rname[80];
    char tname[80];

    cpl_table* sampl_tab=cpl_table_new(nrow);
    cpl_table_new_column(sampl_tab,"WAVELENGTH",CPL_TYPE_DOUBLE);
    cpl_table_new_column(sampl_tab,"X",CPL_TYPE_DOUBLE);
    cpl_table_new_column(sampl_tab,"Y",CPL_TYPE_DOUBLE);
    cpl_table_new_column(sampl_tab,"ORDER",CPL_TYPE_INT);
    cpl_table_fill_column_window_double(sampl_tab,"WAVELENGTH",0,nrow,0.);
    cpl_table_fill_column_window_double(sampl_tab,"X",0,nrow,0.);
    cpl_table_fill_column_window_double(sampl_tab,"Y",0,nrow,0.);
    cpl_table_fill_column_window_int(sampl_tab,"ORDER",0,nrow,0.);
    w_mod=cpl_table_get_data_double(sampl_tab,"WAVELENGTH");
    x_mod=cpl_table_get_data_double(sampl_tab,"X");
    y_mod=cpl_table_get_data_double(sampl_tab,"Y");
    o_mod=cpl_table_get_data_int(sampl_tab,"ORDER");

    int k=0;
    int j=0;

    sprintf(rname,"%s.reg",prefix);
    sprintf(tname,"%s.fits",prefix);
    xsh_msg("rname=%s",rname);
    xsh_msg("tname=%s",tname);

    double wstep=(wlist->list[iorder].lambda_max-wlist->list[iorder].lambda_min)/nsampl;
    double wmin=wlist->list[iorder].lambda_min;
    double wave=0;

    pentry=pentry_start;
    /*add start and end wave and corresponding x,y points */
    w_mod[0]=pentry->lambda;
    x_mod[0]=pentry->ix;
    y_mod[0]=pentry->iy;
    o_mod[0]=iorder;

    w_mod[nrow-1]=(pentry+nitem-1)->lambda;
    x_mod[nrow-1]=(pentry+nitem-1)->ix;
    y_mod[nrow-1]=(pentry+nitem-1)->iy;
    o_mod[nrow-1]=iorder;
    xsh_msg("edges=%g %g",pentry->lambda,(pentry+nitem-1)->lambda);
    for(int i = 1; i < nrow-1 ; i++ ) {
        wave=wmin+i*wstep;
        for(j = 0, pentry = pentry_start ; j < nitem ; j++, pentry++) {
            if( fabs( wave - pentry->lambda ) <= wtol ) {
                w_mod[i]=(pentry-k)->lambda;
                x_mod[i]=(pentry-k)->ix;
                y_mod[i]=(pentry-k)->iy;
                o_mod[i]=iorder;
            }
        }
    }

    check(xsh_sort_table_3(sampl_tab,"WAVELENGTH","Y","X",CPL_TRUE,CPL_TRUE,CPL_TRUE));
    //xsh_dump_table_on_region_file(sampl_tab, rname,0);
    //cpl_table_save(sampl_tab,NULL,NULL,tname,CPL_IO_DEFAULT);
    cleanup:
    if(cpl_error_get_code() != CPL_ERROR_NONE) {
         xsh_print_rec_status(0);
         return NULL;
     } else {
         return sampl_tab;
     }


}

static cpl_table*
xsh_create_sampl_uniform_line(const int iorder,
                         xsh_wavemap_list* wlist,const int nsampl,
                         const char* prefix)
{

    double * w_mod = NULL;
    double * x_mod = NULL;
    double * y_mod = NULL;
    int    * o_mod =NULL;
    int nrow=nsampl+2;
    wavemap_item* pentry = NULL;
    wavemap_item* pentry_start = NULL;
    pentry_start = wlist->list[iorder].sky;
    int nitem = wlist->list[iorder].sky_size;
    double wtol = 0.01;
    char rname[80];
    char tname[80];

    cpl_table* sampl_tab=cpl_table_new(nrow);
    cpl_table_new_column(sampl_tab,"WAVELENGTH",CPL_TYPE_DOUBLE);
    cpl_table_new_column(sampl_tab,"X",CPL_TYPE_DOUBLE);
    cpl_table_new_column(sampl_tab,"Y",CPL_TYPE_DOUBLE);
    cpl_table_new_column(sampl_tab,"ORDER",CPL_TYPE_INT);
    cpl_table_fill_column_window_double(sampl_tab,"WAVELENGTH",0,nrow,0.);
    cpl_table_fill_column_window_double(sampl_tab,"X",0,nrow,0.);
    cpl_table_fill_column_window_double(sampl_tab,"Y",0,nrow,0.);
    cpl_table_fill_column_window_int(sampl_tab,"ORDER",0,nrow,0.);
    w_mod=cpl_table_get_data_double(sampl_tab,"WAVELENGTH");
    x_mod=cpl_table_get_data_double(sampl_tab,"X");
    y_mod=cpl_table_get_data_double(sampl_tab,"Y");
    o_mod=cpl_table_get_data_int(sampl_tab,"ORDER");

    int k=0;
    int j=0;

    sprintf(rname,"%s.reg",prefix);
    sprintf(tname,"%s.fits",prefix);
    xsh_msg("rname=%s",rname);
    xsh_msg("tname=%s",tname);

    double wstep=(wlist->list[iorder].lambda_max-wlist->list[iorder].lambda_min)/nsampl;
    double wmin=wlist->list[iorder].lambda_min;
    double wave=0;

    pentry=pentry_start;
    /*add start and end wave and corresponding x,y points */
    w_mod[0]=pentry->lambda;
    x_mod[0]=pentry->ix;
    y_mod[0]=pentry->iy;
    o_mod[0]=iorder;

    w_mod[nrow-1]=(pentry+nitem-1)->lambda;
    x_mod[nrow-1]=(pentry+nitem-1)->ix;
    y_mod[nrow-1]=(pentry+nitem-1)->iy;
    o_mod[nrow-1]=iorder;
    xsh_msg("edges=%g %g",pentry->lambda,(pentry+nitem-1)->lambda);
    for(int i = 1; i < nrow-1 ; i++ ) {
        wave=wmin+i*wstep;
        for(j = 0, pentry = pentry_start ; j < nitem ; j++, pentry++) {
            if( fabs( wave - pentry->lambda ) <= wtol ) {
                w_mod[i]=(pentry-k)->lambda;
                x_mod[i]=(pentry-k)->ix;
                y_mod[i]=(pentry-k)->iy;
                o_mod[i]=iorder;
            }
        }
    }

    check(xsh_sort_table_3(sampl_tab,"WAVELENGTH","Y","X",CPL_TRUE,CPL_TRUE,CPL_TRUE));
    xsh_dump_table_on_region_file(sampl_tab, rname,0);
    cpl_table_save(sampl_tab,NULL,NULL,tname,CPL_IO_DEFAULT);
    cleanup:
    if(cpl_error_get_code() != CPL_ERROR_NONE) {
         xsh_print_rec_status(0);
         return NULL;
     } else {
         return sampl_tab;
     }


}

static cpl_error_code
xsh_wavemap_list_rms_sky_image_save( xsh_wavemap_list * smap,
                                     const double ron2, const double gain,
                             xsh_instrument * instr,
                             const int iter)
{

    cpl_image* sky_rms = NULL;
    cpl_image* sky_dif = NULL;
    cpl_image* sky_rat = NULL;
    float* psky_rms =NULL;
    float* psky_dif =NULL;
    float* psky_rat =NULL;
    int order;
    int sx, sy ;
    char fname[80];
    XSH_ASSURE_NOT_NULL( smap ) ;
    XSH_ASSURE_NOT_NULL( instr ) ;

    sx = instr->config->nx / instr->binx;
    sy = instr->config->ny / instr->biny;
    xsh_msg( "Image size:%d,%d", sx, sy ) ;

    /* associate to object and sky the corresponding wave and slit maps */
    sky_rms  = cpl_image_new(sx,sy,XSH_PRE_DATA_TYPE);
    psky_rms = cpl_image_get_data_float(sky_rms);

    sky_dif  = cpl_image_new(sx,sy,XSH_PRE_DATA_TYPE);
    psky_dif = cpl_image_get_data_float(sky_dif);

    sky_rat  = cpl_image_new(sx,sy,XSH_PRE_DATA_TYPE);
    psky_rat = cpl_image_get_data_float(sky_rat);

    int ix, iy,pix;
    double flux=0;
    double fit=0;
    double sigma2=0;

    double arg;
    double dif;

    for( order = 0 ; order < smap->size ; order++ ) {

        wavemap_item * psky = smap->list[order].sky;
        int sky_size = smap->list[order].sky_size;

        for (int k = 0; k < sky_size; k++) {
            ix = psky->ix;
            iy = psky->iy;
            pix = iy*sx+ix;
            flux = psky->flux;
            fit  = psky->fitted;
            sigma2 =ron2+(fabs(fit)/gain);
            dif = (flux-fit);
            arg = (flux-fit);
            arg *= arg;
            arg /= sigma2;
            psky_rms[pix]=arg;
            psky_dif[pix]=dif;
            psky_rat[pix]=dif/flux;
            psky++;
        }
    }
#if REGDEBUG_BSPLINE
    sprintf(fname,"sky_model_rms_%2.2d.fits",iter);
    cpl_image_save(sky_rms, fname, CPL_TYPE_FLOAT, NULL, CPL_IO_DEFAULT);
    sprintf(fname,"sky_model_dif_%2.2d.fits",iter);
    cpl_image_save(sky_dif, fname, CPL_TYPE_FLOAT, NULL, CPL_IO_DEFAULT);
    sprintf(fname,"sky_model_rat_%2.2d.fits",iter);
    cpl_image_save(sky_rat, fname, CPL_TYPE_FLOAT, NULL, CPL_IO_DEFAULT);
#endif

    cleanup:
    xsh_free_image(&sky_rms);
    xsh_free_image(&sky_dif);
    xsh_free_image(&sky_rat);
    return cpl_error_get_code() ;
}

static double
xsh_skycorr_rms(xsh_wavemap_list* wlist,const int iorder,
                const double ron2,const double gain)
{

    double rms;
    int nitem = wlist->list[iorder].sky_size;
    wavemap_item* psky = NULL;
    wavemap_item* psky_start = NULL;

    double flux=0;
    double fit=0;
    double sigma2=0;
    int i=0;
    double arg;
    int npix=0;
    double line_ratio=0;
    int decode_bp=wlist->instrument->decode_bp;
    psky_start = wlist->list[iorder].sky;
    for(i = 0, psky = psky_start ; i < nitem ; i++, psky++) {
        if( ( ( psky->qual & decode_bp ) == 0) ) {
            /* we compute rms only on good pixels */
            flux = psky->flux;
            fit  = psky->fitted;
            sigma2 =ron2+(fabs(fit)/gain);
            arg = (flux-fit);
            line_ratio += fabs(arg/flux);
            arg *= arg;
            arg /=sigma2;
            rms += arg;
            npix++;
        }
    }
    rms /= npix;
    line_ratio /=npix;
    xsh_msg("Average spectrum ratio: %g",line_ratio);
    return rms;

}

static double
xsh_skycorr_gen_rms_image(xsh_wavemap_list* wlist,const int iorder)
{

    double rms=0;
    int nitem = wlist->list[iorder].sky_size;
    wavemap_item* psky = NULL;
    wavemap_item* psky_start = NULL;
    double flux=0;
    double fit=0;
    double sigma=0;
    int i=0;
    double arg;
    int npix=0;
    int decode_bp=wlist->instrument->decode_bp;
    for(i = 0, psky = psky_start ; i < nitem ; i++, psky++) {
        if( ( ( psky->qual & decode_bp ) == 0) ) {
        /* we compute rms only on good pixels */
        flux = psky->flux;
        fit  = psky->fitted;
        sigma =psky->sigma;
        arg = (flux-fit)/sigma;
        arg *= arg;
        rms += arg;
        npix++;
        }
    }
    rms /= npix;

    return rms;

}
static cpl_table*
xsh_table_unique_column(cpl_table* tab_inp,const char* col_wav){
    XSH_ASSURE_NOT_NULL( tab_inp ) ;
    cpl_table* tab_out=NULL;
    double* pcol_w;


    tab_out=cpl_table_duplicate(tab_inp);
    int nrow=cpl_table_get_nrow(tab_out);
    pcol_w=cpl_table_get_data_double(tab_inp,col_wav);

    nrow=cpl_table_get_nrow(tab_out);

    for(int i=1;i<nrow;i++) {
        if(pcol_w[i]==pcol_w[i-1]) {
            //xsh_msg("found invalid value %g",pcol_inp[i]);
           cpl_table_set_column_invalid(tab_out,col_wav,i,1);
        }
    }

    cpl_table_erase_invalid(tab_out);
    nrow=cpl_table_get_nrow(tab_out);
    check(xsh_sort_table_1(tab_out,col_wav,CPL_TRUE));

    cleanup:
    return tab_out;
}
static cpl_table*
xsh_skycorr_sample_continuum(xsh_wavemap_list* wlist,
                             const int ord, const int n){
    int ord_abs=wlist->list[ord].order;
    int i_step=1;
    double wmin=wlist->list[ord].lambda_min;
    double wmax=wlist->list[ord].lambda_max;
    double* wave=cpl_calloc(n, sizeof(double));
    double wstep=(wmax-wmin)/n;
    xsh_msg("order=%d wmin=%g wmax=%g",ord,wmin,wmax);
    for(int i = 0; i < n ; i+=i_step ) {
        wave[i]=wmin+i*wstep;
    }
    char tname[80];
    sprintf(tname, "points_sampl_cont_%02d", ord);
    xsh_msg("nbkpts_ord=%d",n);
    cpl_free(wave);
    cpl_table* ctab=xsh_create_sampl_uniform_continuum(ord, wlist, n, tname);
    int nrow=cpl_table_get_nrow(ctab);
    xsh_msg("Total sampling points sky continuum on order %d(%d) is =%d",
            ord,ord_abs,nrow);

    return ctab;
 }



static cpl_table*
xsh_skycorr_sampl_lines_from_static_tab(xsh_wavemap_list* wlist,
                                        const cpl_table* tab_ord,const int ord)
{
    int hsampl_line=169;//199;//129//169
    cpl_table* xtab=cpl_table_duplicate(tab_ord);
    int nrow = cpl_table_get_nrow(tab_ord);

    xsh_msg("Total number of sky lines in reference table on order %d is =%d",ord,nrow);
    cpl_table_cast_column(xtab,"FLUX","DFLUX",CPL_TYPE_DOUBLE);
    cpl_table_cast_column(xtab,"WAVELENGTH","DWAVE",CPL_TYPE_DOUBLE);
    double* wref = cpl_table_get_data_double(xtab,"DWAVE");
    char tname[80];
    sprintf(tname, "points_sampl_lines_%02d", ord);
    cpl_table* ltab=xsh_create_sampl_table(ord, wref, nrow, wlist, hsampl_line, tname);
    nrow=cpl_table_get_nrow(ltab);
    xsh_msg("Total sampling points ref sky line on order %d is =%d",ord,nrow);
    xsh_free_table(&xtab);

    return ltab;
}



static cpl_table*
xsh_model2tab(xsh_wavemap_list* wlist, int order){
cpl_table* res=NULL;
xsh_msg("model2tab");
int sky_ndata = wlist->list[order].sky_size;
res=cpl_table_new(sky_ndata);
cpl_table_new_column(res,"WAVE",CPL_TYPE_DOUBLE);
cpl_table_new_column(res,"FLUX",CPL_TYPE_DOUBLE);
cpl_table_fill_column_window_double(res,"WAVE",0,sky_ndata,0.);
cpl_table_fill_column_window_double(res,"FLUX",0,sky_ndata,0.);
double* pwave=cpl_table_get_data_double(res,"WAVE");
double* pflux=cpl_table_get_data_double(res,"FLUX");
int i_step=1;
int decode_bp=wlist->instrument->decode_bp;
int check_data_quality=0;
wavemap_item * psky = wlist->list[order].sky;

for(int i=0;i<sky_ndata;i+=i_step,psky+=i_step) {

    /* the following is condition to have good pixel quality
     * (or saturated line) */

    check_data_quality = (( ( psky->qual & decode_bp ) == 0  ) ||
                    ( psky->qual == QFLAG_SATURATED_DATA )) ? 0:1;

    //check_data_quality = (psky->qual == QFLAG_SKY_MODEL_BAD_PIX) ? 0:1;
    //check_data_quality=0;
    if( check_data_quality == 0) {
        /* if good pixel sample it */
        pwave[i]=psky->lambda;
        pflux[i]=psky->flux;
    } else {
        cpl_table_set_invalid(res,"WAVE",i);
    }

}

xsh_msg("elements before check %d ",cpl_table_get_nrow(res));
xsh_msg("invalid elements %d",cpl_table_count_invalid(res,"WAVE"));
cpl_table_erase_invalid(res);
cpl_table_and_selected_double(res,"WAVE",CPL_EQUAL_TO,0);

cpl_table_erase_selected(res);
xsh_msg("elements after check %d ",cpl_table_get_nrow(res));
return res;

}


static cpl_table*
xsh_skycorr_sampl_lines_from_data(xsh_wavemap_list* wlist,const int ord)
{
    cpl_table* ltab=NULL;

    /*
    cpl_table* tab_clean=xsh_detect_raw_data_outliers_1d(wlist,3,ord, 1);
    int n=cpl_table_get_nrow(tab_clean);
    cpl_table* xtab=xsh_table_unique_wave(tab_clean,"WAVE","FLUX");
    xsh_free_table(&tab_clean);
    n=cpl_table_get_nrow(xtab);
    cpl_table_duplicate_column(xtab,"SWAVE",xtab,"WAVE");
    cpl_table_duplicate_column(xtab,"SFLUX",xtab,"FLUX");
    double * wave=cpl_table_get_data_double(xtab,"WAVE");
    double * flux=cpl_table_get_data_double(xtab,"FLUX");
    double * swave=cpl_table_get_data_double(xtab,"SVAVE");
    double * sflux=cpl_table_get_data_double(xtab,"SFLUX");
    int sno=n;
    double gain=1;
    double ron2=1;

    sflux=xsh_bspline_fit_smooth(wave, flux,n,swave, sno,ron2, gain,wlist->instrument);
    cpl_table_wrap_double(xtab,"SFLUX",sflux);
    cpl_table_save(xtab,NULL,NULL,"smooth_fit.fits",CPL_IO_DEFAULT);


    */
    cpl_table* xtab=NULL;
    cpl_table* dtab=xsh_model2tab(wlist,ord);
    int nrow=cpl_table_get_nrow(dtab);




    int hsize=1000;
    /* smooth data to later find bad pixels */
    cpl_table_duplicate_column(dtab,"FLUX_SMOOTH",dtab,"FLUX");
    cpl_vector* v_flux=NULL;
    cpl_vector* v_smooth=NULL;
    v_flux = cpl_vector_wrap(nrow,cpl_table_get_data_double(dtab,"FLUX"));
    v_smooth=cpl_vector_filter_lowpass_create(v_flux,CPL_LOWPASS_LINEAR,hsize);
    cpl_vector_unwrap(v_flux);
    double* pts=cpl_table_get_data_double(dtab,"FLUX_SMOOTH");
    double* pvs=cpl_vector_get_data(v_smooth);
    for (int i = 0; i < nrow; i++) {
        pts[i]=pvs[i];
    }
    cpl_vector_delete(v_smooth);

    cpl_table_save(dtab,NULL,NULL,"tab_sel.fits",CPL_IO_DEFAULT);







    int niter=5;
    double kappa=5;
    double med=cpl_table_get_column_median(dtab,"FLUX");
    double rms=cpl_table_get_column_stdev(dtab,"FLUX");
    cpl_table_and_selected_double(dtab,"FLUX",CPL_LESS_THAN,med+kappa*rms);
    cpl_table_and_selected_double(dtab,"FLUX",CPL_GREATER_THAN,med-kappa*rms);
    xtab=cpl_table_extract_selected(dtab);
    med=cpl_table_get_column_median(xtab,"FLUX");
    rms=cpl_table_get_column_stdev(xtab,"FLUX");

    for(int i=0;i<niter;i++){
        cpl_table_select_all(dtab);
        cpl_table_and_selected_double(dtab,"FLUX",CPL_LESS_THAN,med+kappa*rms);
        cpl_table_and_selected_double(dtab,"FLUX",CPL_GREATER_THAN,med-kappa*rms);
        cpl_table* xtab=cpl_table_extract_selected(dtab);
        med=cpl_table_get_column_median(xtab,"FLUX");
        rms=cpl_table_get_column_stdev(xtab,"FLUX");
        xsh_msg("med=%g rms=%g",med,rms);
        xsh_free_table(&xtab);
    }
    nrow=cpl_table_get_nrow(dtab);
    cpl_table_new_column(dtab,"FLAG",CPL_TYPE_INT);
    cpl_table_fill_column_window_int(dtab,"FLAG",1,nrow,0);
    int* pflag=cpl_table_get_data_int(dtab,"FALG");
    double* pflux=cpl_table_get_data_double(dtab,"FLUX");
    for(int i=0;i<nrow;i++){
        if( pflux[i] < med-kappa*rms &&
            pflux[i] > med+kappa*rms ) {
            pflag[i]=1;
        }
    }
    cpl_table_save(dtab,NULL,NULL,"tab_sel.fits",CPL_IO_DEFAULT);
    return ltab;
}
static cpl_table*
xsh_skycorr_wave_sampling_create(xsh_wavemap_list* wlist,const int ord,
                                 const int nbkpts_ord,
                                 const cpl_table* bkpts_tab)
{
    cpl_table* rtab=NULL;
    cpl_table* ltab=NULL;
    cpl_table* ctab=NULL;
    int nrow=0;
    char rname[80];
    XSH_ASSURE_NOT_NULL( wlist ) ;
    XSH_ASSURE_NOT_NULL( bkpts_tab ) ;


    double sky_slit_size= (wlist->sky_slit_max-wlist->sky_slit_min)-
                     (wlist->obj_slit_max-wlist->obj_slit_min);
    xsh_msg("sky_slit_size=%g",sky_slit_size);

    /* creates continuum sampling points table */

    ctab=xsh_skycorr_sample_continuum(wlist, ord, nbkpts_ord);

    /* creates line sampling points table */
    ltab=xsh_skycorr_sampl_lines_from_static_tab(wlist,bkpts_tab,ord);
    //ltab=xsh_skycorr_sampl_lines_from_data(wlist,ord);
    nrow=cpl_table_get_nrow(ltab);
    cpl_table_insert(ltab,ctab,nrow);

    /* add order wmin, wmax to be sure the full range is covered */
    nrow=cpl_table_get_nrow(ltab);
    check(xsh_sort_table_3(ltab,"WAVELENGTH","Y","X",CPL_TRUE,CPL_TRUE,CPL_TRUE));
    cpl_table_and_selected_double(ltab,"WAVELENGTH",CPL_EQUAL_TO,0);
    cpl_table_erase_selected(ltab);
    //cpl_table_save(ltab,NULL,NULL,"pippo.fits",CPL_IO_DEFAULT);
    rtab=xsh_table_unique_column(ltab,"WAVELENGTH");
    //rtab=cpl_table_duplicate(ltab);
    //cpl_table_save(rtab,NULL,NULL,tname,CPL_IO_DEFAULT);
    //cpl_table_dump_structure(rtab,stdout);
    sprintf(rname, "points_sampl_order_%02d.reg", ord);
#if REGDEBUG_BSPLINE
    xsh_dump_table_on_region_file(rtab, rname,3);
#endif
    cpl_table_erase_column(rtab,"ORDER");
    nrow=cpl_table_get_nrow(rtab);

    xsh_msg("Total sampling points for sky spectrum on order %d is =%d",ord,nrow);
    xsh_free_table(&ctab);
    xsh_free_table(&ltab);

    cleanup:
    if(cpl_error_get_code() != CPL_ERROR_NONE) {
        xsh_print_rec_status(0);
        return NULL;
    } else {
        return rtab;
    }

}

/*-----------------------------------------------------------------------------
  Implementation
  ---------------------------------------------------------------------------*/	

/*
gsl_bspline_knots_uniform()
  Construct uniformly spaced knots on the interval [a,b] using
the previously specified number of breakpoints. 'a' is the position
of the first breakpoint and 'b' is the position of the last
breakpoint.

Inputs: a - left side of interval
        b - right side of interval
        w - bspline workspace

Return: success or error

Notes: 1) w->knots is modified to contain the uniformly spaced
          knots

       2) The knots vector is set up as follows (using octave syntax):

          knots(1:k) = a
          knots(k+1:k+l-1) = a + i*delta, i = 1 .. l - 1
          knots(n+1:n+k) = b
*/

static int
xsh_bspline_knots_not_uniform (const double* value,
               gsl_bspline_workspace * w)
{
  size_t i,k;         /* looping */
  //double delta;         /* interval spacing */
  //double x;
  double a = value[w->l];
  double b = value[0];
  xsh_msg("workspace size=%d",(int)w->l);
  xsh_msg("a=%g",a);
  xsh_msg("b=%g",b);
  //delta = (b - a) / (double) w->l;

  for (i = 0; i < w->k; i++)
    gsl_vector_set (w->knots, i, a);

  for (i = 0, k=w->l-1 ; i < w->l - 1; i++,k--)
    {
      gsl_vector_set (w->knots, w->k + i, value[k]);

    }

  for (i = w->n; i < w->n + w->k; i++)
    gsl_vector_set (w->knots, i, b);

  return GSL_SUCCESS;
}               /* gsl_bspline_knots_uniform() */


static int
xsh_bspline_knots_not_uniform2 (const double* value,
               gsl_bspline_workspace * w)
{
  size_t i,k;         /* looping */
  //double delta;         /* interval spacing */
  //double x;
  double a = value[w->l];
  double b = value[0];
  xsh_msg("workspace size=%d",(int)w->l);
  xsh_msg("a=%g",a);
  xsh_msg("b=%g",b);
  //delta = (b - a) / (double) w->l;

  for (i = 0; i < w->k; i++)
    gsl_vector_set (w->knots, i, a);

  for (i = 0, k=w->l-1 ; i < w->l - 1; i++)
    {
      gsl_vector_set (w->knots, w->k + i, value[i]);

    }

  for (i = w->n; i < w->n + w->k; i++)
    gsl_vector_set (w->knots, i, b);

  return GSL_SUCCESS;
}               /* gsl_bspline_knots_uniform() */

/*****************************************************************************/
/*****************************************************************************/

static void
sinfo_matrix_Bn_debug(const int i, const int ord, const int nitem, double* Bn_ptr)
{
    {
        /* we check the metric: dumping the matrix Bn into an image (ord,nitem)*/
        char name[80];
        sprintf(name, "Bn_%d.fits", i);
        cpl_image* ima = cpl_image_wrap_double(ord, nitem, Bn_ptr);
        cpl_image_save(ima, name, CPL_TYPE_FLOAT, NULL, CPL_IO_DEFAULT);
        cpl_image_unwrap(ima);
    }
    return;
}

/*

{

    cpl_image* sky_rms = NULL;
    cpl_image* sky_dif = NULL;
    float* psky_rms =NULL;
    float* psky_dif =NULL;
    int order;
    int nx, ny ;

    XSH_ASSURE_NOT_NULL( smap ) ;
    XSH_ASSURE_NOT_NULL( prefix ) ;
    XSH_ASSURE_NOT_NULL( instr ) ;

    nx = instr->config->nx ;
    ny = instr->config->ny ;
    xsh_msg( "Image size:%d,%d", nx, ny ) ;

    // associate to object and sky the corresponding wave and slit maps
    sky_rms  = cpl_image_new(nx,ny,XSH_PRE_DATA_TYPE);
    psky_rms = cpl_image_get_data_float(sky_rms);

    sky_dif  = cpl_image_new(nx,ny,XSH_PRE_DATA_TYPE);
    psky_dif = cpl_image_get_data_float(sky_dif);

    int ix, iy,pix;
    double flux=0;
    double fit=0;
    double sigma=0;
    int i=0;
    double arg;
    double dif;

    for( order = 0 ; order < smap->size ; order++ ) {

        wavemap_item * psky = smap->list[order].sky;
        int sky_size = smap->list[order].sky_size;

        for (int k = 0; k < sky_size; k++) {
            ix = psky->ix;
            iy = psky->iy;
            pix = iy*nx+ix;
            flux = psky->flux;
            fit  = psky->fitted;
            sigma =psky->sigma;
            dif = (flux-fit);
            arg = (flux-fit)/sigma;
            arg *= arg;

            psky_rms[pix]=arg;
            psky_dif[pix]=dif;
            psky++;
        }
    }

    cpl_image_save(sky_rms, "sky_model_rms.fits", CPL_TYPE_FLOAT, NULL, CPL_IO_DEFAULT);
    cpl_image_save(sky_dif, "sky_model_dif.fits", CPL_TYPE_FLOAT, NULL, CPL_IO_DEFAULT);

    cleanup:
    xsh_free_image(&sky_rms);
    xsh_free_image(&sky_dif);

    return cpl_error_get_code() ;
}
*/

static cpl_error_code
xsh_detect_outliers_image(xsh_wavemap_list * wlist, int iorder)
{

    wavemap_item * pentry ;
    //wavemap_item * pstart ;
    pentry = wlist->list[iorder].sky;
    //pstart = wlist->list[iorder].sky;
    //cpl_mask* mask_order=NULL;
    int nitems=wlist->list[iorder].sky_size;


    /* we initially search for CRHs (possibly not previously detected)
     * Then we search for bad pixels in an image using HDRL bpm_2d
     * Finally we need to identify what are real CRHs, what are just
     * not detected lines, what are line residuals, and what are real bad pixels
     */

    /* we initially prepare the images where to flag CRH and BP */
    cpl_size nx=wlist->instrument->config->nx;
    cpl_size ny=wlist->instrument->config->ny;
    int decode_bp=wlist->instrument->decode_bp;
    cpl_image* resi_ima=cpl_image_new(nx,ny,XSH_PRE_DATA_TYPE);
    cpl_image* errs_ima=cpl_image_new(nx,ny,XSH_PRE_ERRS_TYPE);
    cpl_image* qual_ima=cpl_image_new(nx,ny,XSH_PRE_QUAL_TYPE);
    /* we initialise the qualifier to QFLAG_MISSING_DATA to flag as bad
     * pixels any other order except the one we are going now to fill with
     * the real bad pixel quality. In this way later the computations
     * are performed only on the good pixels of the currently analised order
     */

    cpl_image_add_scalar(qual_ima,QFLAG_MISSING_DATA);
    //cpl_image_save(qual_ima, "qual.fits", XSH_PRE_QUAL_BPP, NULL, CPL_IO_DEFAULT);
    float* presi=cpl_image_get_data_float(resi_ima);
    float* perrs=cpl_image_get_data_float(errs_ima);
    int* pqual=cpl_image_get_data_int(qual_ima);
    int pix=0;
    //int pad=0;

    for (int i = 0;i < nitems; i++,pentry++) {
        pix=pentry->ix+nx*(pentry->iy);
        presi[pix]=fabs(pentry->flux/pentry->fitted);
        /*
        perrs[pix]=sqrt(pentry->sigma*pentry->sigma+
                        pentry->fit_err*pentry->fit_err);
                        */
        perrs[pix]=1;
        pqual[pix]=pentry->qual;
    }
    //cpl_image_save(qual_ima, "qual_after.fits", XSH_PRE_QUAL_BPP, NULL, CPL_IO_DEFAULT);
    cpl_mask* qual_mask=xsh_qual_to_cpl_mask(qual_ima, decode_bp);
    //cpl_mask_save(qual_mask,"qual_mask.fits",NULL,CPL_IO_DEFAULT);

    cpl_image_reject_from_mask(resi_ima,qual_mask);
    hdrl_image * hima;
    char fname[80];
    /* CRH detection */
    {

        double sigma_lim=20;
        double f_lim=2;
        int max_iter=4;

        hdrl_parameter * lacosmic_params =
                        hdrl_lacosmic_parameter_create(sigma_lim, f_lim, max_iter);

        hima = hdrl_image_create(resi_ima, errs_ima);

        cpl_mask* crh_mask = hdrl_lacosmic_edgedetect(hima, lacosmic_params);

        sprintf(fname,"mask_crh_%2.2d.fits",iorder);
        cpl_mask_save(crh_mask,fname,NULL,CPL_IO_DEFAULT);

        hdrl_parameter_delete(lacosmic_params);
        xsh_free_mask(&crh_mask);
    }
    /* BPM_2D */
    {
       hdrl_parameter* bpm_params =
                       hdrl_bpm_2d_parameter_create_filtersmooth(10., 19., 3,
                    CPL_FILTER_MEDIAN, CPL_BORDER_FILTER, 5, 3) ;

       cpl_mask* bpm2d_mask = hdrl_bpm_2d_compute(hima, bpm_params);

       sprintf(fname,"mask_bpm2d_%2.2d.fits",iorder);
       cpl_mask_save(bpm2d_mask,fname,NULL,CPL_IO_DEFAULT);

       hdrl_parameter_delete(bpm_params);
       xsh_free_mask(&bpm2d_mask);
    }

    xsh_free_mask(&qual_mask);
    xsh_free_image(&resi_ima);
    xsh_free_image(&errs_ima);
    xsh_free_image(&qual_ima);
    hdrl_image_delete(hima);

    return cpl_error_get_code();
}



static cpl_error_code
xsh_bspline_matrix_transpose(const int nfit, const int ord_minus_1,
                             const int ord, const int Bn_strd, const int* Border,
                             double* Bn_ptr, float* Xtp, double* px, double* py)
{
    /* Filling Transpose matrix before model coefficients determination
     * See GSL doc section 39.1: f(x)=Sum_i c_i B_{i,k}(x)
     * This means: Y=B*C==> C = Xt.Y = Trans(B).B.Y
     * Xtp=Transpose(Bn).Bn and c=Bn.y */
    size_t orderb=0;
    int jbi=0, kbi=0;
    for (int i = 0; i < nfit; i++) {
        orderb = Border[i] - ord_minus_1;
        for (int j = 0; j < ord; j++) {
            jbi = j * Bn_strd+i;

            for (int k = j; k < ord; k++) {
                kbi = k * Bn_strd +i;
                if (!isnan(Bn_ptr[jbi] * Bn_ptr[kbi])) {
                    Xtp[((orderb + k) * ord) + ord_minus_1 - k + j] +=
                       Bn_ptr[jbi] * Bn_ptr[kbi];
                }
                else {
                    xsh_msg("bad matrix coef at %f %f", px[i], py[i]);
                }
            }
        }
    }
    return cpl_error_get_code();
}


static cpl_error_code
xsh_bspline_get_model_coeffs(const int ncoeffs, const int nfit,
                             const int ord_minus_1, const int ord,
                             const int Bn_strd, const int* Border,
                             const double* Bn_ptr, const float* y, float* c)
{
    /* DEBUG WE DUMP IN A TABLE THE VALUES OF THE C COEFFS */
    cpl_table * tab = cpl_table_new(ncoeffs);
    cpl_table_new_column(tab, "C", CPL_TYPE_DOUBLE);
    cpl_table_fill_column_window_double(tab, "C", 0, ncoeffs, 0);
    double* pc = cpl_table_get_data_double(tab, "C");
    for (int ii = 0; ii < nfit; ii++) {
        size_t orderb = Border[ii] - ord_minus_1;
        for (int jj = 0; jj < ord; jj++) {
            //if(!isnan(Bn_ptr[jj*Bn_strd+ii])) {
            c[orderb + jj] += Bn_ptr[jj * Bn_strd + ii] * y[ii];
            pc[orderb + jj] += Bn_ptr[jj * Bn_strd + ii] * y[ii];
            //}
        }
    }
    cpl_table_save(tab, NULL, NULL, "c.fits", CPL_IO_DEFAULT);
    xsh_free_table(&tab);
    return cpl_error_get_code();
}

static cpl_error_code
xsh_bspline_get_sky_model(const int sky_ndata, const int ord_minus_1,
                          const int ord, const int Bn_full_strd,
                          const int* Border_full, const float* c,
                          const double* Bn_full_ptr, float* yf)
{
    /* Fill y with the predicted flux from the fit, over all values (using Bn_full) */
    /* Next not needed as yf was initialised with gsl_vector_calloc */
    int i;
    for (i = 0; i < sky_ndata; i++) {
        yf[i] = 0;
    }
    /* Define the model: fill-up yf */
    for (i = 0; i < sky_ndata; i++) {
        size_t orderb = Border_full[i] - ord_minus_1;
        for (int j = 0; j < ord; j++) {
            if (!isnan(c[orderb + j])) {
                yf[i] += Bn_full_ptr[j * Bn_full_strd + i] * c[orderb + j];
            }
            else {
                //xsh_msg("found NAN");
                //exit(0);
            }
        }
    }
    return cpl_error_get_code();
}


/*****************************************************************************/

static cpl_table*
xsh_detect_outliers_thres_new(xsh_wavemap_list * wlist, const double kappa2,
                              const double ron2, const double gain,
                              const int order,const int iter,
                              cpl_table** tab_line_res)
{


    wavemap_item * psky ;
    int sky_ndata = wlist->list[order].sky_size;
    cpl_table* tab_bad_fit=cpl_table_new(sky_ndata);

    cpl_table_new_column(tab_bad_fit,"WAVE",CPL_TYPE_DOUBLE);
    cpl_table_new_column(tab_bad_fit,"FLUX",CPL_TYPE_FLOAT);
    cpl_table_new_column(tab_bad_fit,"FIT",CPL_TYPE_DOUBLE);
    cpl_table_new_column(tab_bad_fit,"SIGMA",CPL_TYPE_DOUBLE);
    cpl_table_new_column(tab_bad_fit,"X",CPL_TYPE_DOUBLE);
    cpl_table_new_column(tab_bad_fit,"Y",CPL_TYPE_DOUBLE);
    cpl_table_new_column(tab_bad_fit,"FLAG",CPL_TYPE_INT);

    cpl_table_fill_column_window_double(tab_bad_fit, "WAVE", 0, sky_ndata, 0);
    cpl_table_fill_column_window_float(tab_bad_fit, "FLUX", 0, sky_ndata, 0);
    cpl_table_fill_column_window_double(tab_bad_fit, "FIT", 0, sky_ndata, 0);
    cpl_table_fill_column_window_double(tab_bad_fit, "SIGMA", 0, sky_ndata, 0);
    cpl_table_fill_column_window_double(tab_bad_fit, "X", 0, sky_ndata, 0);
    cpl_table_fill_column_window_double(tab_bad_fit, "Y", 0, sky_ndata, 0);
    cpl_table_fill_column_window_int(tab_bad_fit, "FLAG", 0, sky_ndata, 0);

    double* pbwav=cpl_table_get_data_double(tab_bad_fit,"WAVE");
    float* pbflux=cpl_table_get_data_float(tab_bad_fit,"FLUX");
    double* pbfit=cpl_table_get_data_double(tab_bad_fit,"FIT");
    double* pbsigma=cpl_table_get_data_double(tab_bad_fit,"SIGMA");
    double* pbx=cpl_table_get_data_double(tab_bad_fit,"X");
    double* pby=cpl_table_get_data_double(tab_bad_fit,"Y");
    int* pflag=cpl_table_get_data_int(tab_bad_fit,"FLAG");
    int nbad=0;
    int decode_bp=wlist->instrument->decode_bp;
    psky = wlist->list[order].sky;
    xsh_msg("kappa2=%g ron2=%g gain=%g",kappa2,ron2,gain);

    for ( int i = 0 ; i<sky_ndata ; i++, psky++ ) {
        double tmp_val = psky->flux-psky->fitted;
        double tmp_err = psky->sigma;
        //double tmp_var = ( ron2 + (fabs(psky->fitted)/gain));
        double tmp_var = tmp_err * tmp_err;
        int check_data_value = (( tmp_val * tmp_val ) < kappa2 * tmp_var) ? 0:1;

        /* check if pixel is bad */
        int check_data_quality = (( ( psky->qual & decode_bp ) > 0  ) &&
                        ( psky->qual != QFLAG_SATURATED_DATA )) ? 1:0;
        // define sky values used to evaluate the sky model

        if( check_data_value == 1){
            /* these must be only a few points */
            //xsh_msg("check_value=%d",check_data_value);
            //xsh_msg("val1 %g val2 %g",tmp_val * tmp_val, kappa2 * ( psky->sigma * psky->sigma));
            //xsh_msg("val1 %g val2 %g",tmp_val * tmp_val, kappa2 * ( ron2 + (fabs(yf[i])/gain)));
            pbwav[nbad]=psky->lambda;
            pbflux[nbad]=psky->flux;
            pbfit[nbad]=psky->fitted;
            //pbsigma[nbad]=sqrt(tmp_var);
            pbsigma[nbad]=tmp_err;
            pbx[nbad]=psky->ix;
            pby[nbad]=psky->iy;
            pflag[nbad]=1;
            nbad++;
            psky->qual=QFLAG_SKY_MODEL_BAD_FIT;

        }

    }
    cpl_table_set_size(tab_bad_fit,nbad);
    xsh_msg("Flagging sky line residuals order %d iter %d ndata %d nbad_thresh %d",
            order,iter,sky_ndata,nbad);






    char bname[80];
#if REGDEBUG_BSPLINE
    sprintf(bname,"bad_fit_thres_ord_%2.2d_%2.2d.fits",order,iter);
    //cpl_table_save(tab_bad_fit,NULL,NULL,bname,CPL_IO_DEFAULT);
    sprintf(bname,"bad_fit_thres_ord_%2.2d_%2.2d.reg",order,iter);
    xsh_dump_table_on_region_file(tab_bad_fit,bname, 4);
#endif
    int nsel=cpl_table_and_selected_int(tab_bad_fit,"FLAG",CPL_EQUAL_TO,1);
    cpl_table_name_column(tab_bad_fit,"WAVE","WAVELENGTH");

    if (nsel == 0) {
        return NULL;
    }
    double hmin=cpl_table_get_column_min(tab_bad_fit,"WAVELENGTH");
    double hmax=cpl_table_get_column_max(tab_bad_fit,"WAVELENGTH");

    int nbins=(int)(hmax-hmin);
    if (nbins == 0) {
        return NULL;
    }
    double hstp=(hmax-hmin)/(nbins-1);
    xsh_msg("nbins=%d",nbins);
    cpl_table* h = xsh_histogram(tab_bad_fit,"WAVELENGTH",nbins,hmin,hmax);

    double histo_min=0;
    double histo_max=0;
    double histo_mean=0;
    double histo_median=0;
    double histo_stdev=0;

    histo_min=cpl_table_get_column_min(h,"HY");
    histo_max=cpl_table_get_column_max(h,"HY");

    histo_mean=cpl_table_get_column_mean(h,"HY");
    histo_median=cpl_table_get_column_median(h,"HY");
    histo_stdev=cpl_table_get_column_stdev(h,"HY");
    xsh_msg("histo info: min=%g max=%g mean=%g median=%g,stdev=%g",
                 histo_min,histo_max,histo_mean,histo_median,histo_stdev);

    double kappa=3;
    int* phy=cpl_table_get_data_int(h,"HY");
    double* phw=cpl_table_get_data_double(h,"HL");
    double* pbw=cpl_table_get_data_double(tab_bad_fit,"WAVELENGTH");
    pflag=cpl_table_get_data_int(tab_bad_fit,"FLAG");
    pbx=cpl_table_get_data_double(tab_bad_fit,"X");
    pby=cpl_table_get_data_double(tab_bad_fit,"Y");
    *tab_line_res = cpl_table_new(nbins);
    cpl_table_new_column(*tab_line_res,"WAVELENGTH",CPL_TYPE_FLOAT);
    cpl_table_new_column(*tab_line_res,"FLUX",CPL_TYPE_FLOAT);

    cpl_table_fill_column_window_float(*tab_line_res, "WAVELENGTH", 0, nbad, 0);
    cpl_table_fill_column_window_float(*tab_line_res, "FLUX", 0, nbad, 999);
    float* plwav=cpl_table_get_data_float(*tab_line_res,"WAVELENGTH");

#if REGDEBUG_BSPLINE
    sprintf(bname,"residual_lines_thres_ord_%2.2d_%2.2d.reg",order,iter);
    FILE* fout = NULL;
    fout = fopen(bname, "w");
#endif
    int k=0;
    for(int i=0;i<nbins;i++) {
        if(phy[i]>histo_median+kappa*histo_stdev) {
            plwav[k]=phw[i];
            for(int j=0;j<nbad;j++) {
                if( pbw[j] > phw[i] - hstp &&
                    pbw[j] < phw[i] + hstp ) {
                    pflag[j]=2;
#if REGDEBUG_BSPLINE
                    fprintf(fout,"x point (%g %g) #color=green \n",
                            pbx[j]+1,pby[j]+1);
#endif
                }
            }
            k++;
        }
    }

#if REGDEBUG_BSPLINE
    fclose(fout);
#endif
    cpl_table_set_size(*tab_line_res,k);
    cpl_table_set_column_unit(*tab_line_res,"WAVELENGTH","nm");
    cpl_table_set_column_unit(*tab_line_res,"FLUX","rel-flux");
    xsh_msg("Flagging outliers found %d line outliers",k);


    char hname[80];
    sprintf(hname,"histo_ord_%2.2d_%2.2d.fits",order,iter);
    //cpl_table_save(h, NULL, NULL, hname, CPL_IO_DEFAULT);
    xsh_free_table(&h);

    cpl_table_erase_column(tab_bad_fit,"FLUX");
    cpl_table_erase_column(tab_bad_fit,"FIT");
    cpl_table_erase_column(tab_bad_fit,"SIGMA");
    //cpl_table_erase_column(tab_bad_fit,"FLAG");
    cpl_table_and_selected_double(tab_bad_fit,"WAVELENGTH",CPL_EQUAL_TO,0);
    cpl_table_erase_selected(tab_bad_fit);

    return tab_bad_fit;
}



/*****************************************************************************/



static cpl_table*
xsh_detect_outliers_thres(xsh_wavemap_list * wlist, const float* yf,
                          const double kappa2,const double ron2,
                          const double gain, const int order,const int iter,
                          const int decode_bp,cpl_table** tab_line_res)
{


    wavemap_item * psky ;
    int sky_ndata = wlist->list[order].sky_size;
    cpl_table* tab_bad_fit=cpl_table_new(sky_ndata);

    cpl_table_new_column(tab_bad_fit,"WAVE",CPL_TYPE_DOUBLE);
    cpl_table_new_column(tab_bad_fit,"FLUX",CPL_TYPE_FLOAT);
    cpl_table_new_column(tab_bad_fit,"FIT",CPL_TYPE_DOUBLE);
    cpl_table_new_column(tab_bad_fit,"SIGMA",CPL_TYPE_DOUBLE);
    cpl_table_new_column(tab_bad_fit,"X",CPL_TYPE_DOUBLE);
    cpl_table_new_column(tab_bad_fit,"Y",CPL_TYPE_DOUBLE);
    cpl_table_new_column(tab_bad_fit,"FLAG",CPL_TYPE_INT);

    cpl_table_fill_column_window_double(tab_bad_fit, "WAVE", 0, sky_ndata, 0);
    cpl_table_fill_column_window_float(tab_bad_fit, "FLUX", 0, sky_ndata, 0);
    cpl_table_fill_column_window_double(tab_bad_fit, "FIT", 0, sky_ndata, 0);
    cpl_table_fill_column_window_double(tab_bad_fit, "SIGMA", 0, sky_ndata, 0);
    cpl_table_fill_column_window_double(tab_bad_fit, "X", 0, sky_ndata, 0);
    cpl_table_fill_column_window_double(tab_bad_fit, "Y", 0, sky_ndata, 0);
    cpl_table_fill_column_window_int(tab_bad_fit, "FLAG", 0, sky_ndata, 0);

    double* pbwav=cpl_table_get_data_double(tab_bad_fit,"WAVE");
    float* pbflux=cpl_table_get_data_float(tab_bad_fit,"FLUX");
    double* pbfit=cpl_table_get_data_double(tab_bad_fit,"FIT");
    double* pbsigma=cpl_table_get_data_double(tab_bad_fit,"SIGMA");
    double* pbx=cpl_table_get_data_double(tab_bad_fit,"X");
    double* pby=cpl_table_get_data_double(tab_bad_fit,"Y");
    int* pflag=cpl_table_get_data_int(tab_bad_fit,"FLAG");
    int nbad=0;
    psky = wlist->list[order].sky;
    xsh_msg("kappa2=%g ron2=%g gain=%g",kappa2,ron2,gain);

    for ( int i = 0 ; i<sky_ndata ; i++, psky++ ) {
        double tmp_val = yf[i] - psky->flux;
        double tmp_err = psky->sigma;

        //xsh_msg("yf=%g",yf[i]);
        /*
        int check_data_value = (( tmp_val * tmp_val ) <
           kappa2 * ( ron2 + (fabs(yf[i])/gain))) ? 0:1;
        */
        int check_data_value = (( tmp_val * tmp_val ) <
                   kappa2 * (tmp_err * tmp_err)) ? 0:1;

        int check_data_quality = (( ( psky->qual & decode_bp ) == 0  ) ||
                        ( psky->qual == QFLAG_SATURATED_DATA )) ? 0:1;
        // define sky values used to evaluate the sky model

        if( check_data_value ){
            /* these must be only a few points */
            //xsh_msg("check_value=%d",check_data_value);
            //xsh_msg("val1 %g val2 %g",tmp_val * tmp_val, kappa2 * ( psky->sigma * psky->sigma));
            //xsh_msg("val1 %g val2 %g",tmp_val * tmp_val, kappa2 * ( ron2 + (fabs(yf[i])/gain)));
            pbwav[nbad]=psky->lambda;
            pbflux[nbad]=psky->flux;
            pbfit[nbad]=yf[i];
            //pbsigma[nbad]=sqrt(ron2 + (fabs(yf[i])/gain));
            pbsigma[nbad]=tmp_err;
            pbx[nbad]=psky->ix;
            pby[nbad]=psky->iy;
            pflag[nbad]=1;
            nbad++;
            psky->qual=QFLAG_SKY_MODEL_BAD_FIT;

        }

    }
    cpl_table_set_size(tab_bad_fit,nbad);
    xsh_msg("order %d iter %d ndata %d nbad_thresh %d",
            order,iter,sky_ndata,nbad);
    char bname[80];
#if REGDEBUG_BSPLINE
    sprintf(bname,"bad_fit_thres_ord_%2.2d_%2.2d.fits",order,iter);
    cpl_table_save(tab_bad_fit,NULL,NULL,bname,CPL_IO_DEFAULT);
    sprintf(bname,"bad_fit_thres_ord_%2.2d_%2.2d.reg",order,iter);
    xsh_dump_table_on_region_file(tab_bad_fit,bname, 4);
#endif
    cpl_table_and_selected_int(tab_bad_fit,"FLAG",CPL_EQUAL_TO,1);
    cpl_table_name_column(tab_bad_fit,"WAVE","WAVELENGTH");


    double hmin=cpl_table_get_column_min(tab_bad_fit,"WAVELENGTH");
    double hmax=cpl_table_get_column_max(tab_bad_fit,"WAVELENGTH");

    int nbins=(int)(hmax-hmin);
    xsh_msg("nbins=%d",nbins);
    cpl_table* h=xsh_histogram(tab_bad_fit,"WAVELENGTH",nbins,hmin,hmax);
    double hstp=(hmax-hmin)/(nbins-1);
    xsh_msg("hstp=%g",hstp);
    double histo_min=0;
    double histo_max=0;
    double histo_mean=0;
    double histo_median=0;
    double histo_stdev=0;
    histo_min=cpl_table_get_column_min(h,"HY");
    histo_max=cpl_table_get_column_max(h,"HY");
    histo_mean=cpl_table_get_column_mean(h,"HY");
    histo_median=cpl_table_get_column_median(h,"HY");
    histo_stdev=cpl_table_get_column_stdev(h,"HY");
    xsh_msg("histo info: min=%g max=%g mean=%g median=%g,stdev=%g",
                 histo_min,histo_max,histo_mean,histo_median,histo_stdev);

    double kappa=3;
    int* phy=cpl_table_get_data_int(h,"HY");
    double* phw=cpl_table_get_data_double(h,"HL");
    double* pbw=cpl_table_get_data_double(tab_bad_fit,"WAVELENGTH");
    pflag=cpl_table_get_data_int(tab_bad_fit,"FLAG");
    pbx=cpl_table_get_data_double(tab_bad_fit,"X");
    pby=cpl_table_get_data_double(tab_bad_fit,"Y");


    *tab_line_res = cpl_table_new(nbins);
    cpl_table_new_column(*tab_line_res,"WAVELENGTH",CPL_TYPE_FLOAT);
    cpl_table_new_column(*tab_line_res,"FLUX",CPL_TYPE_FLOAT);

    cpl_table_fill_column_window_float(*tab_line_res, "WAVELENGTH", 0, nbad, 0);
    cpl_table_fill_column_window_float(*tab_line_res, "FLUX", 0, nbad, 999);
    float* plwav=cpl_table_get_data_float(*tab_line_res,"WAVELENGTH");



    int k=0;
#if REGDEBUG_BSPLINE
    sprintf(bname,"residual_lines_thres_ord_%2.2d_%2.2d.reg",order,iter);
    FILE* fout = NULL;
    fout = fopen(bname, "w");
#endif
    for(int i=0;i<nbins;i++) {
        if(phy[i]>histo_median+kappa*histo_stdev) {
            for(int j=0;j<nbad;j++) {
                plwav[k]=phw[i];
                if( pbw[j] > phw[i] - hstp &&
                    pbw[j] < phw[i] + hstp ) {
                    pflag[j]=2;
#if REGDEBUG_BSPLINE
                    fprintf(fout,"x point (%g %g) #color=green \n",
                            pbx[j]+1,pby[j]+1);
#endif
                }
            }
            k++;
        }
    }
#if REGDEBUG_BSPLINE
    fclose(fout);
#endif
    cpl_table_set_size(*tab_line_res,k);
    cpl_table_set_column_unit(*tab_line_res,"WAVELENGTH","nm");
    cpl_table_set_column_unit(*tab_line_res,"FLUX","rel-flux");
    xsh_msg("found %d line outliers",k);
#if REGDEBUG_BSPLINE
    char hname[80];
    sprintf(hname,"histo_ord_%2.2d_%2.2d.fits",order,iter);
    cpl_table_save(h, NULL, NULL, hname, CPL_IO_DEFAULT);
#endif
    xsh_free_table(&h);

    cpl_table_erase_column(tab_bad_fit,"FLUX");
    cpl_table_erase_column(tab_bad_fit,"FIT");
    cpl_table_erase_column(tab_bad_fit,"SIGMA");
    //cpl_table_erase_column(tab_bad_fit,"FLAG");
    cpl_table_and_selected_double(tab_bad_fit,"WAVELENGTH",CPL_EQUAL_TO,0);
    cpl_table_erase_selected(tab_bad_fit);

    return tab_bad_fit;
}
/*****************************************************************************/


static cpl_error_code
xsh_model_fill_fit(xsh_wavemap_list* wlist,const int order, cpl_table* stab,
                   const float ron2, const float gain){


    int sky_ndata = wlist->list[order].sky_size;
    int all_ndata = wlist->list[order].all_size;

    double* pwave=cpl_table_get_data_double(stab,"WAVE");
    double* pfit=cpl_table_get_data_double(stab,"FIT");

    //cpl_table_dump(stab,0,5,stdout);
    wavemap_item * psky = wlist->list[order].sky;
    int nrow=cpl_table_get_nrow(stab);
    int j_min=0;

    cpl_table_dump_structure(stab,stdout);
    for(int i=0;i<sky_ndata;i++,psky++) {
        for(int j=j_min;j<nrow;j++) {
            if(pwave[j]==psky->lambda) {
                psky->fitted=pfit[j];
                /* we assume error free model */
                //psky->fit_err=sqrt(ron2+fabs(pfit[j]/gain));
                j_min=j;
                break;
            }
        }
    }

    wavemap_item * pall = wlist->list[order].all;
    j_min=0;
    for(int i=0;i<all_ndata;i++,pall++) {
         for(int j=j_min;j<nrow;j++) {
             if(pwave[j]==pall->lambda) {
                 pall->fitted=pfit[j];
                 /* we assume error free model */
                 //psky->fit_err=sqrt(ron2+fabs(pfit[j]/gain));
                 j_min=j;
                 break;
             }
         }
     }


    return cpl_error_get_code();
}


static cpl_error_code
xsh_model_fill_obj(xsh_wavemap_list* wlist,const int order, const double s1,
                   const double s2,cpl_table* stab,
                   const float ron2, const float gain){


    int sky_ndata = wlist->list[order].sky_size;
    int all_ndata = wlist->list[order].all_size;
    //int obj_ndata = wlist->list[order].object_size;

    double* pwave=cpl_table_get_data_double(stab,"WAVE");
    double* pfit=cpl_table_get_data_double(stab,"FIT");

    //cpl_table_dump(stab,0,5,stdout);
    wavemap_item * psky = wlist->list[order].sky;
    wavemap_item * pall = wlist->list[order].all;
    int nrow=cpl_table_get_nrow(stab);
    int j_min=0;

    cpl_table_dump_structure(stab,stdout);

    for(int i=0;i<all_ndata;i++,pall++) {
        wavemap_item * psky = wlist->list[order].sky;
        for(int j=0;j<sky_ndata;j++,psky++) {
            if(pall->lambda==psky->lambda) {
                pall->fitted=psky->fitted;
                //psky->fit_err=sqrt(ron2+fabs(pfit[j]/gain));
                break;
            }
        }
    }
    pall = wlist->list[order].all;
    j_min=0;
    for(int i=0;i<all_ndata;i++,pall++) {
         for(int j=j_min;j<nrow;j++) {
             if(pwave[j]==pall->lambda && pall->slit >= s1 && pall->slit<=s2) {
                 pall->fitted=pfit[j];
                 //psky->fit_err=sqrt(ron2+fabs(pfit[j]/gain));
                 j_min=j;
                 break;
             }
         }
     }


    return cpl_error_get_code();
}

static cpl_error_code
xsh_model_fill_fit_bfit(xsh_wavemap_list* wlist,const int order, cpl_table* stab,
                   const float ron2, const float gain){

    int sky_ndata = wlist->list[order].sky_size;
    int all_ndata = wlist->list[order].all_size;
    double* pwave=cpl_table_get_data_double(stab,"WAVE");
    double* pfit=cpl_table_get_data_double(stab,"BFIT");
    //cpl_table_dump(stab,0,5,stdout);
    wavemap_item * psky = wlist->list[order].sky;
    int nrow=cpl_table_get_nrow(stab);
    int j_min=0;

    for(int i=0;i<sky_ndata;i++,psky++) {
        for(int j=j_min;j<nrow;j++) {
            if(pwave[j]==psky->lambda) {
                psky->fitted=pfit[j];
                //psky->fit_err=sqrt(ron2+fabs(pfit[j]/gain));
                j_min=j;
                break;
            }
        }
    }

    wavemap_item * pall = wlist->list[order].all;
    j_min=0;
    for(int i=0;i<all_ndata;i++,pall++) {
         for(int j=j_min;j<nrow;j++) {
             if(pwave[j]==pall->lambda) {
                 pall->fitted=pfit[j];
                 //psky->fit_err=sqrt(ron2+fabs(pfit[j]/gain));
                 j_min=j;
                 break;
             }
         }
     }


    return cpl_error_get_code();
}


static cpl_error_code
xsh_model_fill_fit_slice(xsh_wavemap_list* wlist,const int order,
                         const double s1, const double s2, cpl_table* stab,
                         const float ron2, const float gain){

    int sky_ndata = wlist->list[order].sky_size;
    int all_ndata = wlist->list[order].all_size;
    double* pwave=cpl_table_get_data_double(stab,"WAVE");
    double* pfit=cpl_table_get_data_double(stab,"FIT");
    //cpl_table_dump(stab,0,5,stdout);
    wavemap_item * psky = wlist->list[order].sky;
    int nrow=cpl_table_get_nrow(stab);
    int j_min=0;

    for(int i=0;i<sky_ndata;i++,psky++) {
        for(int j=j_min;j<nrow;j++) {
            if(pwave[j]==psky->lambda &&
                         psky->slit > s1 &&
                         psky->slit < s2) {
                psky->fitted=pfit[j];
                //psky->fit_err=sqrt(ron2+fabs(pfit[j]/gain));
                j_min=j;
                break;
            }
        }
    }

    wavemap_item * pall = wlist->list[order].all;
    j_min=0;
    for(int i=0;i<all_ndata;i++,pall++) {
         for(int j=j_min;j<nrow;j++) {
             if(pwave[j]==pall->lambda) {
                 pall->fitted=pfit[j];
                 //psky->fit_err=sqrt(ron2+fabs(pfit[j]/gain));
                 j_min=j;
                 break;
             }
         }
     }

    return cpl_error_get_code();
}


static cpl_error_code
xsh_model_fill_fit_slice_bfit(xsh_wavemap_list* wlist,const int order,
                         const double s1, const double s2, cpl_table* stab,
                         const float ron2, const float gain){

    int sky_ndata = wlist->list[order].sky_size;
    int all_ndata = wlist->list[order].all_size;
    double* pwave=cpl_table_get_data_double(stab,"WAVE");
    double* pfit=cpl_table_get_data_double(stab,"BFIT");
    //cpl_table_dump(stab,0,5,stdout);
    wavemap_item * psky = wlist->list[order].sky;
    int nrow=cpl_table_get_nrow(stab);
    int j_min=0;

    for(int i=0;i<sky_ndata;i++,psky++) {
        for(int j=j_min;j<nrow;j++) {
            if(pwave[j]==psky->lambda &&
                         psky->slit >= s1 &&
                         psky->slit < s2) {
                psky->fitted=pfit[j];
                //psky->fit_err=sqrt(ron2+fabs(pfit[j]/gain));
                j_min=j;
                break;
            }
        }
    }

    wavemap_item * pall = wlist->list[order].all;
    j_min=0;
    for(int i=0;i<all_ndata;i++,pall++) {
         for(int j=j_min;j<nrow;j++) {
             if(pwave[j]==pall->lambda &&
                             pall->slit >= s1 &&
                             pall->slit < s2) {
                 pall->fitted=pfit[j];
                 //psky->fit_err=sqrt(ron2+fabs(pfit[j]/gain));
                 j_min=j;
                 break;
             }
         }
     }

    return cpl_error_get_code();
}


static
int test_gsl_example(double* data_x, double* data_y,double* data_e,double* yfit,
                     const int i_start, const size_t n, const size_t ncoeffs) {

  const size_t nbreak = ncoeffs-2;
  size_t i, j;
  gsl_bspline_workspace *bw;
  gsl_vector *B;
  double dy;
  gsl_rng *r;
  gsl_vector *c, *w;
  gsl_vector *x, *y;
  gsl_matrix *X, *cov;
  gsl_multifit_linear_workspace *mw;
  //gsl_multifit_robust_workspace *mw;
  double chisq, dof;
  //double tss, Rsq;


  gsl_rng_env_setup();
  r = gsl_rng_alloc(gsl_rng_default);

  /* allocate a cubic bspline workspace (k = 4) */
  bw = gsl_bspline_alloc(4, nbreak);
  B = gsl_vector_alloc(ncoeffs);

  x = gsl_vector_alloc(n);
  y = gsl_vector_alloc(n);
  X = gsl_matrix_alloc(n, ncoeffs);
  c = gsl_vector_alloc(ncoeffs);
  w = gsl_vector_alloc(n);
  cov = gsl_matrix_alloc(ncoeffs, ncoeffs);
  mw = gsl_multifit_linear_alloc(n, ncoeffs);
  //mw = gsl_multifit_robust_alloc(gsl_multifit_robust_bisquare, n, ncoeffs);

  //printf("#m=0,S=0 n=%d\n",n);
  /* Fill vectors with data to be fitted */
  for (i = 0; i < n; ++i)
    {
      double sigma;
      sigma = data_e[i];

      gsl_vector_set(x, i, data_x[i]);
      gsl_vector_set(y, i, data_y[i]);
      gsl_vector_set(w, i, 1.0 / (sigma * sigma));

      //printf("%f %f\n", data_x[i], data_y[i]);
    }

  /* use uniform breakpoints on [0, 15]. bw is sized nord*nbreak=4*nbreak */
  gsl_bspline_knots_uniform(data_x[0], data_x[n-1], bw);

  /* construct the fit matrix X */
  for (i = 0; i < n; ++i)
    {
      double xi = gsl_vector_get(x, i);

      /* compute B_j(xi) for all j */
      gsl_bspline_eval(xi, B, bw);

      /* fill in row i of X */
      for (j = 0; j < ncoeffs; ++j)
        {
          double Bj = gsl_vector_get(B, j);
          gsl_matrix_set(X, i, j, Bj);
        }
    }

  /* do the fit */
  gsl_multifit_wlinear(X, w, y, c, cov, &chisq, mw);

  dof = n - ncoeffs;
  /*
  tss = gsl_stats_wtss(w->data, 1, y->data, 1, y->size);
  Rsq = 1.0 - chisq / tss;

  fprintf(stderr, "chisq/dof = %e, Rsq = %f\n",
                   chisq / dof, Rsq);
  */
  /* output the smoothed curve */
  //printf("fit\n");
  {
    double xi, yi, yerr;

    //printf("#m=1,S=0\n");
    for (i = 0; i < n; i++)
      {
        gsl_bspline_eval(data_x[i], B, bw);
        gsl_multifit_linear_est(B, c, cov, &yi, &yerr);
        //gsl_multifit_robust_est(B, c, cov, &yi, &yerr);
        yfit[i_start+i]=yi;
        //printf("%f %f\n", data_x[i], yi);
      }
  }

  gsl_rng_free(r);
  gsl_bspline_free(bw);
  gsl_vector_free(B);
  gsl_vector_free(x);
  gsl_vector_free(y);
  gsl_matrix_free(X);
  gsl_vector_free(c);
  gsl_vector_free(w);
  gsl_matrix_free(cov);
  gsl_multifit_linear_free(mw);
  //gsl_multifit_robust_free(mw);

  //exit(0);
}

static
int xsh_gsl_bspline_non_uniform(double* data_x, double* data_y,double* data_e,
                                double* bkpts, double* yfit,
                     const int i_start, const size_t n, const size_t ncoeffs) {

  const size_t nbreak = ncoeffs-2;
  size_t i, j;
  gsl_bspline_workspace *bw;
  gsl_vector *B;
  double dy;
  gsl_rng *r;
  gsl_vector *c, *w;
  gsl_vector *x, *y;
  gsl_matrix *X, *cov;
  gsl_multifit_linear_workspace *mw;
  //gsl_multifit_robust_workspace *mw;
  double chisq, dof;
  //double tss, Rsq;

  gsl_rng_env_setup();
  r = gsl_rng_alloc(gsl_rng_default);

  /* allocate a cubic bspline workspace (k = 4) */
  bw = gsl_bspline_alloc(4, nbreak);
  //B = gsl_vector_alloc(n);
  B = gsl_vector_alloc(ncoeffs);
  gsl_vector* bkpts_v = gsl_vector_alloc(nbreak);
  x = gsl_vector_alloc(n);
  y = gsl_vector_alloc(n);
  X = gsl_matrix_alloc(n, ncoeffs);
  c = gsl_vector_alloc(ncoeffs);
  w = gsl_vector_alloc(n);
  cov = gsl_matrix_alloc(ncoeffs, ncoeffs);
  mw = gsl_multifit_linear_alloc(n, ncoeffs);

  //mw = gsl_multifit_robust_alloc(gsl_multifit_robust_ols,n, ncoeffs);
  //mw->maxiter=200;
  //printf("#m=0,S=0 n=%d\n",n);
  /* Fill gsl vector with defined break points */
  //printf("data\n");
  /*force start and end break points to be start/end data points */
  bkpts[0]=data_x[0];
  bkpts[nbreak-1]=data_x[n-1];
  for (i = 0; i < nbreak; ++i){
      gsl_vector_set(bkpts_v, i, bkpts[i]);
      //xsh_msg("knots=%13.8g",bkpts[i]);
  }

  /* Fill gsl vector with data points */
  for (i = 0; i < n; ++i)
    {
      double sigma;
      sigma = data_e[i];

      gsl_vector_set(x, i, data_x[i]);
      gsl_vector_set(y, i, data_y[i]);
      gsl_vector_set(w, i, 1.0 / (sigma * sigma));

      //printf("%f %f\n", data_x[i], data_y[i]);
    }

  /* use non uniform breakpoints  */
  /*
  xsh_msg("ncoeffs=%d nbkpts=%d ndata=%d",ncoeffs,nbreak,n);
  xsh_msg("ok1 xmin=%13.8g xmax=%13.8g",data_x[0],data_x[n-1]);
  xsh_msg("ok1 bkg_min=%13.8g bkg_max=%13.8g",bkpts[0],bkpts[nbreak-1]);
  xsh_msg("breakpts->size %d", bkpts_v->size);
  xsh_msg("w->nbreak %d",bw->nbreak);
  */
  gsl_bspline_knots(bkpts_v, bw);
  /* construct the fit matrix X */
  int k=0;
  int flat=0;
  double vx;
  for (i = 0; i < n; ++i)
    {

      double xi = gsl_vector_get(x, i);
      //xsh_msg("ok22 B->size=%d bw->n=%d",B->size,bw->n);
      /* compute B_j(xi) for all j */
      /*
      xsh_msg("i=%d xi=%13.8g bkg_min=%g bkg_max=%13.8g xmin=%13.8g xmax=%13.8g",
              i,xi,bkpts[0],bkpts[nbreak+1],data_x[0],data_x[n-1]);
              */

      gsl_bspline_eval(xi, B, bw);

      /* fill in row i of X */
      for (j = 0; j < ncoeffs; ++j)
        {
          double Bj = gsl_vector_get(B, j);
          gsl_matrix_set(X, i, j, Bj);
        }

    }

  /* do the fit */
  gsl_multifit_wlinear(X, w, y, c, cov, &chisq, mw);
  //gsl_multifit_robust(X, y, c, cov, mw);

  dof = n - ncoeffs;
  /*
  tss = gsl_stats_wtss(w->data, 1, y->data, 1, y->size);
  Rsq = 1.0 - chisq / tss;

  fprintf(stderr, "chisq/dof = %e, Rsq = %f\n",
                   chisq / dof, Rsq);
  */
  /* compute the smoothed curve */
  {
    double xi, yi, yerr;
    for (i = 0; i < n; i++)
      {
        gsl_bspline_eval(data_x[i], B, bw);
        gsl_multifit_linear_est(B, c, cov, &yi, &yerr);
        //gsl_multifit_robust_est(B, c, cov, &yi, &yerr);
        yfit[i_start+i]=yi;
      }
  }

  gsl_rng_free(r);
  gsl_bspline_free(bw);
  gsl_vector_free(B);
  gsl_vector_free(x);
  gsl_vector_free(y);
  gsl_matrix_free(X);
  gsl_vector_free(c);
  gsl_vector_free(w);
  gsl_matrix_free(cov);
  //gsl_multifit_robust_free(mw);

  gsl_multifit_linear_free(mw);
}

cpl_error_code
xsh_bspline_smooth_uniform_sl(cpl_table** tab,xsh_instrument* instrument,
                              const double s1,const double s2){


    int nrow=cpl_table_get_nrow(*tab);
    int nchunks=20;

    int npart=(int)(nrow/nchunks+0.5);
    double* pwave=cpl_table_get_data_double(*tab,"WAVE");
    double* pflux=cpl_table_get_data_double(*tab,"FLUX_SMOOTH");
    double* perr=cpl_table_get_data_double(*tab,"ERR");
    double* pbfit=cpl_table_get_data_double(*tab,"BFIT");
    cpl_table* xtab=NULL;
    int i_min=0;
    int i_max=0;
    xsh_msg("npart=%d",npart);
    int npoints=0;
    for(int i=0;i<=nchunks-1;i++) {
        i_min=i*npart;
        i_max=((i+1)*npart<nrow) ?  (i+1)*npart: nrow-1;
        cpl_table_and_selected_double(*tab,"SLIT",CPL_NOT_LESS_THAN,s1);
        cpl_table_and_selected_double(*tab,"SLIT",CPL_LESS_THAN,s2);
        cpl_table_and_selected_double(*tab,"WAVE",CPL_NOT_LESS_THAN,pwave[i_min]);
        cpl_table_and_selected_double(*tab,"WAVE",CPL_LESS_THAN,pwave[i_max]);
        xtab=cpl_table_extract_selected(*tab);
        xsh_sort_table_1(*tab,"WAVE",CPL_FALSE);
        npoints=cpl_table_get_nrow(xtab);
        //xsh_msg("npoints=%d",npoints);
        double* pw=cpl_table_get_data_double(xtab,"WAVE");
        double* pf=cpl_table_get_data_double(xtab,"FLUX_SMOOTH");
        double* pbf=cpl_table_get_data_double(xtab,"BFIT");
        //cpl_table_save(xtab,NULL,NULL,"pippo_xtab.fits",CPL_IO_DEFAULT);

        xsh_msg("nscan=%d",i_max-i_min+1);
        test_gsl_example(pw, pf,perr,pbfit,i_min,npoints,51);
        cpl_table_save(*tab,NULL,NULL,"pippo.fits",CPL_IO_DEFAULT);
        //exit(0);
        cpl_table_select_all(*tab);
        xsh_free_table(&xtab);
    }
    cpl_table_save(*tab,NULL,NULL,"pippo.fits",CPL_IO_DEFAULT);
    //exit(0);
    return cpl_error_get_code();
}

cpl_error_code
xsh_bspline_smooth_uniform(cpl_table** tab,xsh_instrument* instrument){


    int nrow=cpl_table_get_nrow(*tab);
    int nchunks=20;
    cpl_table_new_column(*tab,"BFIT",CPL_TYPE_DOUBLE);
    cpl_table_fill_column_window_double(*tab,"BFIT",0,nrow,0);
    int npart=(int)(nrow/nchunks+0.5);
    double* pwave=cpl_table_get_data_double(*tab,"WAVE");
    double* pflux=cpl_table_get_data_double(*tab,"FLUX_SMOOTH");
    double* perr=cpl_table_get_data_double(*tab,"ERR");
    double* pbfit=cpl_table_get_data_double(*tab,"BFIT");
    cpl_table* xtab=NULL;
    int i_min=0;
    int i_max=0;
    xsh_msg("npart=%d",npart);
    int npoints=0;
    int nbkpts=151;//131; //151;
    for(int i=0;i<=nchunks-1;i++) {
        i_min=i*npart;
        i_max=((i+1)*npart<nrow) ?  (i+1)*npart: nrow-1;
        cpl_table_and_selected_double(*tab,"WAVE",CPL_NOT_LESS_THAN,pwave[i_min]);
        cpl_table_and_selected_double(*tab,"WAVE",CPL_LESS_THAN,pwave[i_max]);
        xtab=cpl_table_extract_selected(*tab);
        npoints=cpl_table_get_nrow(xtab);
        //xsh_msg("npoints=%d",npoints);
        double* pw=cpl_table_get_data_double(xtab,"WAVE");
        double* pf=cpl_table_get_data_double(xtab,"FLUX_SMOOTH");
        double* pbf=cpl_table_get_data_double(xtab,"BFIT");
        /*
        cpl_table_save(xtab,NULL,NULL,"pippo_xtab.fits",CPL_IO_DEFAULT);
        if(pwave[i_min]<2272) {
            xsh_msg("wave[i_min]=%g pw[0]=%g",pwave[i_min],pw[0]);
            xsh_msg("wave[i_max]=%g pw[npoints-1]=%g",pwave[i_max],pw[npoints-1]);
        }

         */
        //xsh_msg("nscan=%d nbkpts=%d",i_max-i_min+1,nbkpts);
        if( i_max-i_min+1 <= nbkpts) {
            nbkpts=i_max-i_min-1;
            if(nbkpts < 4) {
                break;
            }
        }

        //xsh_msg("nscan=%d nbkpts=%d",i_max-i_min+1,nbkpts);
        test_gsl_example(pw, pf,perr,pbfit,i_min,npoints,nbkpts);
        cpl_table_select_all(*tab);
        xsh_free_table(&xtab);
    }
    //cpl_table_save(*tab,NULL,NULL,"pippo.fits",CPL_IO_DEFAULT);

    return cpl_error_get_code();
}


static cpl_error_code
xsh_bspline_smooth_non_uniform(cpl_table* bkpts, cpl_table** tab,
                              xsh_instrument* instrument){


    int nrow=cpl_table_get_nrow(*tab);
    int nchunks=50;
    cpl_table_new_column(*tab,"BFIT",CPL_TYPE_DOUBLE);
    cpl_table_fill_column_window_double(*tab,"BFIT",0,nrow,0);
    int npart=(int)(nrow/nchunks+0.5);
    double* pwave=cpl_table_get_data_double(*tab,"WAVE");
    double* pflux=cpl_table_get_data_double(*tab,"FLUX_SMOOTH");
    double* perr=cpl_table_get_data_double(*tab,"ERR");
    double* pbfit=cpl_table_get_data_double(*tab,"BFIT");
    cpl_table* xtab=NULL;
    cpl_table* btab=NULL;
    int i_min=0;
    int i_max=0;
    //xsh_msg("npart=%d",npart);
    int npoints=0;

    for(int i=0;i<=nchunks-1;i++) {
        i_min=i*npart;
        i_max=((i+1)*npart<nrow) ?  (i+1)*npart: nrow-1;
        //xsh_msg("wmin=%13.8g wmax=%13.8g",pwave[i_min],pwave[i_max]);
        cpl_table_and_selected_double(bkpts,"WAVELENGTH",CPL_NOT_LESS_THAN,pwave[i_min]);
        cpl_table_and_selected_double(bkpts,"WAVELENGTH",CPL_LESS_THAN,pwave[i_max]);
        btab=cpl_table_extract_selected(bkpts);
        int nbkpts=cpl_table_get_nrow(btab);
        double* pbkpts=cpl_table_get_data_double(btab,"WAVELENGTH");
        //xsh_msg("bkg_min=%13.8g bkg_max=%13.8g",pbkpts[0],pbkpts[nbkpts-1]);
        //xsh_msg(">>>>>>>nbkpts=%d",nbkpts);

        cpl_table_and_selected_double(*tab,"WAVE",CPL_NOT_LESS_THAN,pwave[i_min]);
        cpl_table_and_selected_double(*tab,"WAVE",CPL_LESS_THAN,pwave[i_max]);
        xtab=cpl_table_extract_selected(*tab);
        npoints=cpl_table_get_nrow(xtab);
        //xsh_msg("npoints=%d",npoints);

        double* pw=cpl_table_get_data_double(xtab,"WAVE");
        double* pf=cpl_table_get_data_double(xtab,"FLUX_SMOOTH");
        double* pbf=cpl_table_get_data_double(xtab,"BFIT");

        cpl_table_save(xtab,NULL,NULL,"pippo_xtab.fits",CPL_IO_DEFAULT);
        cpl_table_save(btab,NULL,NULL,"pippo_btab.fits",CPL_IO_DEFAULT);
        //xsh_msg("nscan=%d",i_max-i_min+1);

        /*
        xsh_msg("i_min=%d i_max=%d wmin=%13.8g wmax=%13.8g xmin=%13.8g xmax=%13.8g",
                i_min, i_min+npoints-1, pwave[i_min],pwave[i_min+npoints-1],
                pw[0],pw[npoints-1]);
                */

        xsh_gsl_bspline_non_uniform(pw, pf,perr,pbkpts,pbfit,i_min,npoints,nbkpts);
        cpl_table_save(*tab,NULL,NULL,"pippo.fits",CPL_IO_DEFAULT);
        cpl_table_select_all(*tab);
        cpl_table_select_all(bkpts);
        xsh_free_table(&xtab);
        xsh_free_table(&btab);
    }
    //cpl_table_save(*tab,NULL,NULL,"pippo.fits",CPL_IO_DEFAULT);

    return cpl_error_get_code();
}

static cpl_error_code
xsh_bspline_smooth_non_uniform2(cpl_table* bkpts, cpl_frame* sky_orders_chunks,
                                cpl_table** tab,
                                const int iorder, xsh_instrument* instrument){


    xsh_msg("Spline smooth of non uniformly sampled & chopped profile");
    const char* name=cpl_frame_get_filename(sky_orders_chunks);
    int order_nb=0;
    /* load proper table to specify order chunk locations */
    if(instrument->arm == XSH_ARM_UVB) {
        order_nb=instrument->uvb_orders_nb;
    } else if(instrument->arm == XSH_ARM_VIS) {
        order_nb=instrument->vis_orders_nb;
    } else if(instrument->arm == XSH_ARM_NIR) {
        order_nb=instrument->nir_orders_nb;
    }
    xsh_msg("name=%s iorder=%d order_nb=%d",name,iorder,order_nb);
    cpl_table* tab_chunks=cpl_table_load(name,order_nb-iorder,1);

    int nchunks=cpl_table_get_nrow(tab_chunks);

    int nrow=cpl_table_get_nrow(*tab);
    double* pchunk=cpl_table_get_data_double(tab_chunks,"col1");

    /* prepare table with results */
    cpl_table_new_column(*tab,"BFIT",CPL_TYPE_DOUBLE);
    cpl_table_fill_column_window_double(*tab,"BFIT",0,nrow,0);
    int npart=(int)(nrow/nchunks+0.5);
    double* pwave=cpl_table_get_data_double(*tab,"WAVE");
    double* pflux=cpl_table_get_data_double(*tab,"FLUX_SMOOTH");
    double* perr=cpl_table_get_data_double(*tab,"ERR");
    double* pbfit=cpl_table_get_data_double(*tab,"BFIT");
    cpl_table* xtab=NULL;
    cpl_table* btab=NULL;
    double wmin=0;
    double wmax=0;
    double chunk_wmin=0;
    double chunk_wmax=0;
    int i_min=0;
    int i_max=0;
    //xsh_msg("npart=%d",npart);
    int npoints=0;

    for(int i=0;i<nchunks-1;i++) {

        i_max=((i+1)*npart<nrow) ?  (i+1)*npart: nrow-1;
        /* restrict break points to given chunk region */
        chunk_wmin=pchunk[i];
        chunk_wmax=pchunk[i+1];
        //wmin = ( pwave[i_min] < chunk_wmin ) ? pwave[i_min] : chunk_wmin;
        wmin = pwave[i_min];
        xsh_msg("order splitting wmin=%13.8g wmax=%13.8g",chunk_wmin,chunk_wmax);
        //cpl_table_and_selected_double(bkpts,"WAVELENGTH",CPL_NOT_LESS_THAN,chunk_wmin);
        cpl_table_and_selected_double(bkpts,"WAVELENGTH",CPL_NOT_LESS_THAN,wmin);
        cpl_table_and_selected_double(bkpts,"WAVELENGTH",CPL_LESS_THAN,chunk_wmax);
        btab=cpl_table_extract_selected(bkpts);
        int nbkpts=cpl_table_get_nrow(btab);
        double* pbkpts=cpl_table_get_data_double(btab,"WAVELENGTH");
        //xsh_msg("bkg_min=%13.8g bkg_max=%13.8g",pbkpts[0],pbkpts[nbkpts-1]);
        //xsh_msg(">>>>>>>nbkpts=%d",nbkpts);


        xsh_msg("wmin=%g chunk_wmin=%g pwave[i_min]=%g chunk_wmax=%g",
                wmin,chunk_wmin,pwave[i_min],chunk_wmax);
        /* restrict fit of data to region delimited by chunk region */
        cpl_table_and_selected_double(*tab,"WAVE",CPL_NOT_LESS_THAN,wmin);
        cpl_table_and_selected_double(*tab,"WAVE",CPL_LESS_THAN,chunk_wmax);
        xtab=cpl_table_extract_selected(*tab);
        npoints=cpl_table_get_nrow(xtab);
        //xsh_msg("npoints=%d",npoints);
         if(npoints > 0 )  {
        double* pw=cpl_table_get_data_double(xtab,"WAVE");
        double* pf=cpl_table_get_data_double(xtab,"FLUX_SMOOTH");
        double* pbf=cpl_table_get_data_double(xtab,"BFIT");

        cpl_table_save(xtab,NULL,NULL,"pippo_xtab.fits",CPL_IO_DEFAULT);
        cpl_table_save(btab,NULL,NULL,"pippo_btab.fits",CPL_IO_DEFAULT);
        //xsh_msg("nscan=%d",i_max-i_min+1);
        /*
        xsh_msg("i_min=%d i_max=%d wmin=%13.8g wmax=%13.8g xmin=%13.8g xmax=%13.8g",
                i_min, i_min+npoints-1, pwave[i_min],pwave[i_min+npoints-1],
                pw[0],pw[npoints-1]);
        */
        //xsh_msg("ok1 nbkpts=%d npoints=%d",nbkpts,npoints);
        if( i_max-i_min+1 <= nbkpts) {
            nbkpts=i_max-i_min-1;
            if(nbkpts < 4) {
                break;
            }
        }


        xsh_gsl_bspline_non_uniform(pw, pf,perr,pbkpts,pbfit,i_min,npoints,nbkpts);

        i_min+=npoints;
        cpl_table_save(*tab,NULL,NULL,"pippo.fits",CPL_IO_DEFAULT);
         } else {
             break;
         }
        cpl_table_select_all(*tab);
        cpl_table_select_all(bkpts);
        xsh_free_table(&xtab);
        xsh_free_table(&btab);
    }
    //cpl_table_save(*tab,NULL,NULL,"pippo.fits",CPL_IO_DEFAULT);

    return cpl_error_get_code();
}


/**S
  @brief This routine performs the B-SPLINE fit of a set of sky data
  non-uniformly sampled.

  @param[in] wlist i/o structure with sky data and its B-spline model info
  @param[in] order extraction order (relative no)
  @param[in] sampl_pts_no number of sampling points (NOT USED: parameter to be removed)
  @param[in] iter_no model iteration ID
  @param[in] sky_par sky extraction parameters
  @param[in] ron2  square of RON
  @param[in] gain gain value
  @param[out] tab_line_res i/o table with location of line sampling points

 @return table with location of (bad model fit) outliers.

 */
static cpl_table*
xsh_fit_spline1( xsh_wavemap_list * wlist, int order,cpl_table* sampl_tab,
                 cpl_frame* sky_orders_chunks,int sampl_pts_no, int iter_no,
                 xsh_subtract_sky_single_param *sky_par,
                 float ron2, float gain, cpl_table** tab_line_res)
{
    /* init variables and log some message */
    xsh_msg("xsh_fit_spline1");
    cpl_table* tab_bad_fit=NULL;
    float kappa=sky_par->kappa;
    int bs_order = sky_par->bezier_spline_order;
    int i=0;
    wavemap_item * psky = wlist->list[order].sky;
    int abs_order = wlist->list[order].order;
    int sky_ndata = wlist->list[order].sky_size;
    int n=sky_ndata;
    double wmin=wlist->list[order].lambda_min;
    double wmax=wlist->list[order].lambda_max;
    cpl_table* tab_mod=NULL;

    tab_mod=xsh_model_to_table(wlist,order);


    cpl_table* tab_clean=NULL;
    //xsh_msg("total sky data %d",sky_ndata);


    /* remove outliers (as Kelson recommends) and determined spectrum to be fit
     * as clean median filter of sky data.
     */

    int decode_bp = wlist->instrument->decode_bp;
    tab_clean=xsh_detect_raw_data_outliers_1d(wlist,sky_par->median_hsize,
                    ron2,gain,order, iter_no, decode_bp);

    n=cpl_table_get_nrow(tab_clean);
    //xsh_msg("After outlier flagging %d",n);

    /* make sure to remove duplicate wavelength entries */
    cpl_table* xtab=xsh_table_unique_wave(tab_clean,"WAVE","FLUX");
    xsh_free_table(&tab_clean);

    /* make sure wavelength are decreasing (as original input) */
    xsh_sort_table_1(sampl_tab,"WAVELENGTH",CPL_FALSE);
    if( sky_par->method == BSPLINE_METHOD ||
        sky_par->method == BSPLINE_METHOD1 ||
                    wlist->instrument->arm == XSH_ARM_UVB) {

      xsh_bspline_smooth_uniform(&xtab,wlist->instrument);
    } else {
      //xsh_bspline_smooth_non_uniform(sampl_tab,&xtab,wlist->instrument);
      check(xsh_bspline_smooth_non_uniform2(sampl_tab,sky_orders_chunks,
                                          &xtab,order,wlist->instrument));
    }

    char sname[80];
    sprintf(sname,"tab_bspline_smooth_ord_%2.2d_%2.2d.fits",abs_order,iter_no);
    //cpl_table_save(xtab,NULL,NULL,sname,CPL_IO_DEFAULT);
    sprintf(sname,"tab_bspline_bkpts_ord_%2.2d_%2.2d.fits",abs_order,iter_no);
    //cpl_table_save(sampl_tab,NULL,NULL,sname,CPL_IO_DEFAULT);
    n=cpl_table_get_nrow(xtab);
    xsh_msg("After removal wave duplicates %d",n);




    sprintf(sname,"tab_data_ord_%2.2d_%2.2d.fits",abs_order,iter_no);
    //cpl_table_save(sampl_tab,NULL,NULL,sname,CPL_IO_DEFAULT);

    //double* swave=cpl_table_get_data_double(sampl_tab,"WAVELENGTH");
    double* wave=cpl_table_get_data_double(xtab,"WAVE");
    double* flux=NULL;
    /* We always fit to the BSPLINE SMOOTH FIT value, never to the data
     * to prevent spikes
        flux=cpl_table_get_data_double(xtab,"FLUX");
    */
        //flux=cpl_table_get_data_double(xtab,"FLUX_SMOOTH");
        flux=cpl_table_get_data_double(xtab,"BFIT");

#if REGDEBUG_BSPLINE
    sprintf(sname,"xtab_ord_%2.2d_%2.2d.fits",abs_order,iter_no);
    cpl_table_save(xtab,NULL,NULL,sname,CPL_IO_DEFAULT);
#endif

    /* put wlist structure info in a table, make sure it is sorted properly */
    cpl_table* mtab=NULL;
    mtab=xsh_model2tab(wlist,order);
    xsh_sort_table_1(mtab,"WAVE",CPL_FALSE);

    /* remove wavelength duplicates */
    cpl_table* stab=NULL;
    stab=xsh_table_unique_wave(mtab,"WAVE","FLUX");
    xsh_free_table(&mtab);

    /* (B-spline) interpolate model data (swave,sflux) to get back full
     * resolution (wave,flux)
     */
    /*
    double* swave=cpl_table_get_data_double(stab,"WAVE");
    int sno=cpl_table_get_nrow(stab);
    double* sflux=xsh_bspline_interpolate_data_at_pos(wave,flux,n,swave, sno);
    */
    double* swave = cpl_calloc(wlist->list[order].all_size,sizeof(double));
    wavemap_item* pall=NULL;
    pall = wlist->list[order].all;
    int sno=wlist->list[order].all_size;
    for(i=0;i<wlist->list[order].all_size;i++,pall++){
        swave[i]=pall->lambda;
    }
    cpl_table* atab=cpl_table_new(wlist->list[order].all_size);
    cpl_table_wrap_double(atab,swave,"WAVE");
    double* sflux=xsh_bspline_interpolate_data_at_pos(wave,flux,n,swave, sno);
    /*
    double* sflux=xsh_bspline_fit_smooth(wave, flux, n, swave, sno, ron2,
                    gain, wlist->instrument);
                    */

    //cpl_table_wrap_double(sampl_tab,sflux,"FIT");
    //cpl_table_wrap_double(stab,sflux,"FIT");
    cpl_table_wrap_double(atab,sflux,"FIT");

#if REGDEBUG_BSPLINE
    sprintf(sname,"tab_sampl_ord_%2.2d_%2.2d.fits",abs_order,iter_no);
    cpl_table_save(atab,NULL,NULL,sname,CPL_IO_DEFAULT);
#endif

    //xsh_msg("abs order %d",abs_order);
    //xsh_dump_sky_fit(abs_order, order, n, wlist, yf,iter_no);

    /* copy model solution back to wlist structure */
    //xsh_model_fill_fit(wlist,order,stab,ron2,gain);
    xsh_model_fill_fit(wlist,order,atab,ron2,gain);

    /* identify model outliers (bad pixels and lines) */
    check(tab_bad_fit=xsh_detect_outliers_thres_new(wlist,kappa*kappa, ron2,
                     gain,order,iter_no,tab_line_res));

    cleanup:
    xsh_free_table(&atab);
    xsh_free_table(&stab);
    xsh_free_table(&xtab);
    xsh_free_table(&tab_mod);
    return tab_bad_fit;
}

static cpl_table*
xsh_fit_spline11( xsh_wavemap_list * wlist, int order,cpl_table* sampl_tab,
                 cpl_frame* sky_orders_chunks,int sampl_pts_no, int iter_no,
                 xsh_subtract_sky_single_param *sky_par,
                 float ron2, float gain, cpl_table** tab_line_res)
{
    /* init variables and log some message */
    xsh_msg("xsh_fit_spline11");
    cpl_table* tab_bad_fit=NULL;
    float kappa=sky_par->kappa;
    int bs_order = sky_par->bezier_spline_order;
    int i=0;
    wavemap_item * psky = wlist->list[order].sky;
    int abs_order = wlist->list[order].order;
    int sky_ndata = wlist->list[order].sky_size;
    int n=sky_ndata;
    double wmin=wlist->list[order].lambda_min;
    double wmax=wlist->list[order].lambda_max;
    cpl_table* tab_mod=NULL;

    tab_mod=xsh_model_to_table(wlist,order);


    cpl_table* tab_clean=NULL;
    //xsh_msg("total sky data %d",sky_ndata);


    /* remove outliers (as Kelson recommends) and determined spectrum to be fit
     * as clean median filter of sky data.
     */

    int decode_bp = wlist->instrument->decode_bp;
    tab_clean=xsh_detect_raw_data_outliers_1d(wlist,sky_par->median_hsize,
                    ron2,gain,order, iter_no, decode_bp);

    n=cpl_table_get_nrow(tab_clean);
    //xsh_msg("After outlier flagging %d",n);

    /* make sure to remove duplicate wavelength entries */
    cpl_table* xtab=xsh_table_unique_wave(tab_clean,"WAVE","FLUX");
    xsh_free_table(&tab_clean);

    /* make sure wavelength are decreasing (as original input) */
    xsh_sort_table_1(sampl_tab,"WAVELENGTH",CPL_FALSE);
    if( sky_par->method == BSPLINE_METHOD ||
        sky_par->method == BSPLINE_METHOD1 ||
                    wlist->instrument->arm == XSH_ARM_UVB) {
      xsh_bspline_smooth_uniform(&xtab,wlist->instrument);
    } else {
         exit(0);
      //xsh_bspline_smooth_non_uniform(sampl_tab,&xtab,wlist->instrument);
      check(xsh_bspline_smooth_non_uniform2(sampl_tab,sky_orders_chunks,
                                          &xtab,order,wlist->instrument));
    }

    char sname[80];
    sprintf(sname,"tab_bspline_smooth_ord_%2.2d_%2.2d.fits",abs_order,iter_no);
    //cpl_table_save(xtab,NULL,NULL,sname,CPL_IO_DEFAULT);
    sprintf(sname,"tab_bspline_bkpts_ord_%2.2d_%2.2d.fits",abs_order,iter_no);
    //cpl_table_save(sampl_tab,NULL,NULL,sname,CPL_IO_DEFAULT);
    n=cpl_table_get_nrow(xtab);
    xsh_msg("After removal wave duplicates %d",n);



    sprintf(sname,"tab_data_ord_%2.2d_%2.2d.fits",abs_order,iter_no);
    //cpl_table_save(sampl_tab,NULL,NULL,sname,CPL_IO_DEFAULT);

    //double* swave=cpl_table_get_data_double(sampl_tab,"WAVELENGTH");
    double* wave=cpl_table_get_data_double(xtab,"WAVE");
    double* flux=NULL;
    /* we always fit to the B-spline smooth of the data never to the data
     * to prevent spikes
        flux=cpl_table_get_data_double(xtab,"FLUX");
        */
        //flux=cpl_table_get_data_double(xtab,"FLUX_SMOOTH");
        flux=cpl_table_get_data_double(xtab,"BFIT");
    //}
#if REGDEBUG_BSPLINE
    sprintf(sname,"xtab_ord_%2.2d_%2.2d.fits",abs_order,iter_no);
    cpl_table_save(xtab,NULL,NULL,sname,CPL_IO_DEFAULT);
#endif


    /* copy model solution back to wlist structure */
    //xsh_model_fill_fit(wlist,order,stab,ron2,gain);
    xsh_model_fill_fit_bfit(wlist,order,xtab,ron2,gain);

    /* identify model outliers (bad pixels and lines) */
    check(tab_bad_fit=xsh_detect_outliers_thres_new(wlist,kappa*kappa, ron2,
                     gain,order,iter_no,tab_line_res));

    cleanup:
    return tab_bad_fit;
}

/**S
  @brief This routine performs the B-SPLINE fit of a set of sky data
  non-uniformly sampled.

  @param[in] wlist i/o structure with sky data and its B-spline model info
  @param[in] order extraction order (relative no)
  @param[in] sampl_pts_no number of sampling points (NOT USED: parameter to be removed)
  @param[in] iter_no model iteration ID
  @param[in] sky_par sky extraction parameters
  @param[in] ron2  square of RON
  @param[in] gain gain value
  @param[out] tab_line_res i/o table with location of line sampling points

 @return table with location of (bad model fit) outliers.

 */
static cpl_table*
xsh_fit_spline6( xsh_wavemap_list * wlist, int order,cpl_table* sampl_tab,
                 cpl_frame* sky_orders_chunks,int sampl_pts_no, int iter_no,
                 xsh_subtract_sky_single_param *sky_par,
                 float ron2, float gain, cpl_table** tab_line_res)
{
    /* init variables and log some message */
    xsh_msg("xsh_fit_spline6");
    cpl_table* tab_bad_fit=NULL;
    float kappa=sky_par->kappa;
    int bs_order = sky_par->bezier_spline_order;
    int i=0;
    wavemap_item * psky = wlist->list[order].sky;
    int abs_order = wlist->list[order].order;
    int sky_ndata = wlist->list[order].sky_size;
    int n=sky_ndata;
    double wmin=wlist->list[order].lambda_min;
    double wmax=wlist->list[order].lambda_max;

    //xsh_msg("sky slit min=%g max=%g",wlist->sky_slit_min,wlist->sky_slit_max);


    double smin=wlist->sky_slit_min;
    double smax=wlist->sky_slit_max;
    //xsh_msg("sky slit min=%g max=%g",smax);
    double omin=wlist->obj_slit_min;
    double omax=wlist->obj_slit_max;
    int ns=5;
    double s_step=(smax-smin)/ns;
    double s_extra=0.0; //s_step;
    double s, s1,s2;
    int sid=0;
    cpl_table* xtab=NULL;
    cpl_table* tab_clean=NULL;
    char sname[80];
    //xsh_msg("total sky data %d",sky_ndata);

    /* remove outliers (as Kelson recommends) and determined spectrum to be fit
     * as clean median filter of sky data.
     */
    //xsh_msg("ok1 s_extra s_extra=%g",s_extra);
    int decode_bp = wlist->instrument->decode_bp;
    int med_hsize=sky_par->median_hsize;
    cpl_table* ttab=cpl_table_new(0);

    int ntot=0;
    for(s=smin, sid=0; s<smax-0.0001;s+=s_step,sid++) {
        xsh_msg("sid=%d start s=%10.8g smax=%10.8g",sid,s,smax);
        if(s>=omin && s <=omax) {
            xsh_msg("sid=%d obj slit",sid);
            if(s+s_step<omax) {
                xsh_msg("sid=%d s=%g continue",sid,s);
                continue;
            } else {
              s1=omax;
              s2=s+s_step;
              xsh_msg("sid=%d s=%g s1=%g s2=%g",sid,s,s1,s2);
              if(s1==s2) {
                    xsh_msg("sid=%d s=%g s1=%g s2=%g continue",sid,s,s1,s2);
                    break;
              }
            }
        } else {

            if(s<omin) {
                xsh_msg("sid=%d sky slit small",sid);
                s1=(s>smin) ? s: smin-s_extra;
                s2=(s+s_step<=omin) ?s+s_step: omin+s_extra;

            } else {
                xsh_msg("sid=%d sky slit large",sid);
                s1=(s>=omin) ? s: omin-s_extra;
                s2=(s+s_step<=smax) ?s+s_step: smax+s_extra;
                if(s1==s2) {
                      xsh_msg("sid=%d s=%g s1=%g s2=%g continue",sid,s,s1,s2);
                      break;
                }
            }
            xsh_msg("sid=%d s1=%g s2=%g",sid,s1,s2);
            tab_clean=xsh_detect_raw_data_outliers_1d_slice(wlist,med_hsize,
                            ron2, gain, order, iter_no, decode_bp,s1,s2,sid);

            n=cpl_table_get_nrow(tab_clean);
            if(sid==0) {
                cpl_table_copy_structure(ttab,tab_clean);
                cpl_table_new_column(ttab,"BFIT",CPL_TYPE_DOUBLE);
            }

            //xsh_msg("After outlier flagging %d",n);

            /* make sure to remove duplicate wavelength entries */
            cpl_table* xtab=xsh_table_unique_wave(tab_clean,"WAVE","FLUX");
            xsh_free_table(&tab_clean);

            /* make sure wavelength are decreasing (as original input) */
            check(xsh_sort_table_1(sampl_tab,"WAVELENGTH",CPL_FALSE));

            if( sky_par->method == BSPLINE_METHOD1 ||
                sky_par->method == BSPLINE_METHOD2 ||
                wlist->instrument->arm == XSH_ARM_UVB) {
                xsh_bspline_smooth_uniform(&xtab,wlist->instrument);
            } else {
                //xsh_bspline_smooth_non_uniform(sampl_tab,&xtab,wlist->instrument);
                xsh_bspline_smooth_non_uniform2(sampl_tab,sky_orders_chunks,
                                &xtab,order,wlist->instrument);
            }

            cpl_table_insert(ttab,xtab,ntot);
            ntot += cpl_table_get_nrow(xtab);
            sprintf(sname,"tab_bspline_smooth_ord_%2.2d_slice_%2.2d_iter_%2.2d.fits",
                    abs_order,sid,iter_no);
            //cpl_table_save(xtab,NULL,NULL,sname,CPL_IO_DEFAULT);

            sprintf(sname,"tab_bspline_bkpts_ord_%2.2d_slice_%2.2d_iter_%2.2d.fits",
                    abs_order,sid,iter_no);
            //cpl_table_save(sampl_tab,NULL,NULL,sname,CPL_IO_DEFAULT);

            n=cpl_table_get_nrow(xtab);
            xsh_msg("After removal wave duplicates %d",n);


            sprintf(sname,"tab_data_ord_%2.2d_slice_%2.2d_iter_%2.2d.fits",
                    abs_order,sid,iter_no);
            //cpl_table_save(sampl_tab,NULL,NULL,sname,CPL_IO_DEFAULT);

            //double* swave=cpl_table_get_data_double(sampl_tab,"WAVELENGTH");



            double* wave=cpl_table_get_data_double(xtab,"WAVE");
            double* flux=NULL;
            /* we always fit to the B-spline smooth of the data, never to the
             * data to prevent spikes


                flux=cpl_table_get_data_double(xtab,"FLUX");
            */
                //flux=cpl_table_get_data_double(xtab,"FLUX_SMOOTH");
                flux=cpl_table_get_data_double(xtab,"BFIT");

        #if REGDEBUG_BSPLINE
            sprintf(sname,"xtab_ord_%2.2d_%2.2d.fits",abs_order,iter_no);
            cpl_table_save(xtab,NULL,NULL,sname,CPL_IO_DEFAULT);
        #endif

            /* put wlist structure info in a table, make sure it is sorted properly */
            cpl_table* mtab=NULL;
            mtab=xsh_model2tab(wlist,order);
            xsh_sort_table_1(mtab,"WAVE",CPL_FALSE);

            /* remove wavelength duplicates */
            cpl_table* stab=NULL;
            stab=xsh_table_unique_wave(mtab,"WAVE","FLUX");
            xsh_free_table(&mtab);

            /* (B-spline) interpolate model data (swave,sflux) to get back full
               * resolution (wave,flux)
               */
              /*
              double* swave=cpl_table_get_data_double(stab,"WAVE");
              int sno=cpl_table_get_nrow(stab);
              double* sflux=xsh_bspline_interpolate_data_at_pos(wave,flux,n,swave, sno);
              */
              double* swave = cpl_calloc(wlist->list[order].all_size,sizeof(double));

              wavemap_item* pall=NULL;
              pall = wlist->list[order].all;
              int sno=wlist->list[order].all_size;
              for(i=0;i<wlist->list[order].all_size;i++,pall++){
                  swave[i]=pall->lambda;
              }

              cpl_table* atab=cpl_table_new(wlist->list[order].all_size);
              cpl_table_wrap_double(atab,swave,"WAVE");
              int nxtab=cpl_table_get_nrow(xtab);
              int xsno=cpl_table_and_selected_double(atab,"WAVE",CPL_NOT_LESS_THAN,wave[0]);
              xsno=cpl_table_and_selected_double(atab,"WAVE",CPL_NOT_GREATER_THAN,wave[nxtab-1]);
              //xsno=cpl_table_and_selected_double(atab,"WAVE",CPL_NOT_GREATER_THAN,wave[nxtab]);
              cpl_table* xstab=cpl_table_extract_selected(atab);
              double* xswave=cpl_table_get_data_double(xstab,"WAVE");
              //xsh_msg("ok8 wave[0]=%g swave[0]=%g",wave[0],swave[0]);
              //xsh_msg("ok8 wave[0]=%g xswave[0]=%g",wave[0],xswave[0]);

              double* sflux=NULL;
              sflux=xsh_bspline_interpolate_data_at_pos(wave,flux,n,xswave, xsno);


              /*
              double* sflux=xsh_bspline_fit_smooth(wave, flux, n, swave, sno, ron2,
                              gain, wlist->instrument);
                              */

              //cpl_table_wrap_double(sampl_tab,sflux,"FIT");
              //cpl_table_wrap_double(stab,sflux,"FIT");
              cpl_table_wrap_double(xstab,sflux,"FIT");

          #if REGDEBUG_BSPLINE
              sprintf(sname,"tab_sampl_ord_%2.2d_slice_%2.2d_iter_%2.2d.fits",
                      abs_order,sid,iter_no);
              cpl_table_save(xstab,NULL,NULL,sname,CPL_IO_DEFAULT);
          #endif

              //xsh_msg("abs order %d",abs_order);
              //xsh_dump_sky_fit(abs_order, order, n, wlist, yf,iter_no);

              /* copy model solution back to wlist structure */
              //xsh_model_fill_fit(wlist,order,stab,ron2,gain);

              xsh_model_fill_fit_slice(wlist,order,s1,s2,xstab,ron2,gain);
              sprintf(sname,"model_ord_%2.2d_slice_%2.2d_iter_%2.2d.fits",
                                    abs_order,sid,iter_no);

              xsh_wavemap_list_sky_image_save( wlist, wlist->instrument, abs_order,sid,444);
              xsh_wavemap_list_rms_sky_image_save(wlist,ron2,gain,wlist->instrument,555);


        }
    }
    //exit(0);
    sprintf(sname,"tab_bspline_smooth_ord_%2.2d_iter_%2.2d.fits",
            abs_order,iter_no);
    //cpl_table_save(ttab,NULL,NULL,sname,CPL_IO_DEFAULT);
    sprintf(sname,"tab_bspline_bkpts_ord_%2.2d_iter_%2.2d.fits",
            abs_order,iter_no);
    //cpl_table_save(sampl_tab,NULL,NULL,sname,CPL_IO_DEFAULT);
    //exit(0);





    /* identify model outliers (bad pixels and lines) */
    check(tab_bad_fit=xsh_detect_outliers_thres_new(wlist,kappa*kappa, ron2,
                     gain,order,iter_no,tab_line_res));

    cleanup:
    return tab_bad_fit;
}


static cpl_table*
xsh_fit_spline2( xsh_wavemap_list * wlist, int order,cpl_table* sampl_tab,
                 cpl_frame* sky_orders_chunks,int sampl_pts_no, int iter_no,
                 xsh_subtract_sky_single_param *sky_par,
                 float ron2, float gain, cpl_table** tab_line_res)
{
    /* init variables and log some message */
    xsh_msg("xsh_fit_spline2");
    cpl_table* tab_bad_fit=NULL;
    float kappa=sky_par->kappa;
    int bs_order = sky_par->bezier_spline_order;
    int i=0;
    wavemap_item * psky = wlist->list[order].sky;
    int abs_order = wlist->list[order].order;
    int sky_ndata = wlist->list[order].sky_size;
    int n=sky_ndata;
    double wmin=wlist->list[order].lambda_min;
    double wmax=wlist->list[order].lambda_max;

    //xsh_msg("sky slit min=%g max=%g",wlist->sky_slit_min,wlist->sky_slit_max);


    double smin=wlist->sky_slit_min;
    double smax=wlist->sky_slit_max;
    //xsh_msg("sky slit min=%g max=%g",smax);
    double omin=wlist->obj_slit_min;
    double omax=wlist->obj_slit_max;
    int ns=5;
    double s_step=(smax-smin)/ns;
    double s_extra=0.0; //s_step;
    double s, s1,s2;
    int sid=0;
    cpl_table* xtab=NULL;
    cpl_table* tab_clean=NULL;
    char sname[80];
    //xsh_msg("total sky data %d",sky_ndata);

    /* remove outliers (as Kelson recommends) and determined spectrum to be fit
     * as clean median filter of sky data.
     */
    //xsh_msg("ok1 s_extra s_extra=%g",s_extra);
    int decode_bp = wlist->instrument->decode_bp;
    int med_hsize=sky_par->median_hsize;
    cpl_table* ttab=cpl_table_new(0);

    double s_margin=0.001;

    int ntot=0;
    for(s=smin, sid=0; s<smax-0.0001;s+=s_step,sid++) {
        xsh_msg("sid=%d start s=%10.8g smax=%10.8g splus=%10.8g",sid,s,smax,s+s_step);
        if(s < omin) {
            xsh_msg("sid=%d sky slit small",sid);
            //if(s+s_step < omin-s_margin) {
            s1=s;
            s2=( s+s_step < omin-s_margin ) ? s+s_step: omin;
            /*
            } else {
                xsh_msg("sid=%d sky slit small continue",sid);
                continue;
            }
            */
        } else if (s > omin && s < omax) {

            if( s+s_step < omax+s_margin) {
              xsh_msg("sid=%d object slit continue",sid);
              continue;
            } else {
                xsh_msg("sid=%d obj slit special",sid);
                s1 = omax;
                s2 = s+s_step;
            }
        } else {
                xsh_msg("sid=%d sky slit large",sid);
                s1 = s;
                s2 = (s+s_step < smax) ? s+s_step: smax;

        }
        xsh_msg("sid=%d s1=%g s2=%g",sid,s1,s2);
        tab_clean=xsh_detect_raw_data_outliers_1d_slice(wlist,med_hsize,
                        ron2, gain, order, iter_no, decode_bp,s1,s2,sid);

        n=cpl_table_get_nrow(tab_clean);
        if(sid==0) {
            cpl_table_copy_structure(ttab,tab_clean);
            cpl_table_new_column(ttab,"BFIT",CPL_TYPE_DOUBLE);
        }

        //xsh_msg("After outlier flagging %d",n);

        /* make sure to remove duplicate wavelength entries */
        cpl_table* xtab=xsh_table_unique_wave(tab_clean,"WAVE","FLUX");
        xsh_free_table(&tab_clean);

        /* make sure wavelength are decreasing (as original input) */
        check(xsh_sort_table_1(sampl_tab,"WAVELENGTH",CPL_FALSE));

        if( sky_par->method == BSPLINE_METHOD2 ||
            wlist->instrument->arm == XSH_ARM_UVB) {
            xsh_bspline_smooth_uniform(&xtab,wlist->instrument);
        } else {

            //xsh_bspline_smooth_non_uniform(sampl_tab,&xtab,wlist->instrument);
            xsh_bspline_smooth_non_uniform2(sampl_tab,sky_orders_chunks,
                            &xtab,order,wlist->instrument);
        }

        cpl_table_insert(ttab,xtab,ntot);
        ntot += cpl_table_get_nrow(xtab);
        sprintf(sname,"tab_bspline_smooth_ord_%2.2d_slice_%2.2d_iter_%2.2d.fits",
                abs_order,sid,iter_no);
        //cpl_table_save(xtab,NULL,NULL,sname,CPL_IO_DEFAULT);

        sprintf(sname,"tab_bspline_bkpts_ord_%2.2d_slice_%2.2d_iter_%2.2d.fits",
                abs_order,sid,iter_no);
        //cpl_table_save(sampl_tab,NULL,NULL,sname,CPL_IO_DEFAULT);

        n=cpl_table_get_nrow(xtab);
        xsh_msg("After removal wave duplicates %d",n);


        sprintf(sname,"tab_data_ord_%2.2d_slice_%2.2d_iter_%2.2d.fits",
                abs_order,sid,iter_no);


        xsh_model_fill_fit_slice_bfit(wlist,order,s1,s2,xtab,ron2,gain);

        sprintf(sname,"model_ord_%2.2d_slice_%2.2d_iter_%2.2d.fits",
                abs_order,sid,iter_no);
        xsh_msg("order=%d sid=%d s1=%g s2=%g omin=%g omax=%g",
                order,sid,s1,s2,omin,omax);

        xsh_wavemap_list_sky_image_save( wlist, wlist->instrument, abs_order,sid,444);

        xsh_wavemap_list_rms_sky_image_save(wlist,ron2,gain,wlist->instrument,555);
        xsh_free_table(&xtab);

    }
    /* at this point we have the sky model on the sky region and we need to get
     * the sky model on the obj region. The best approximation is to
     * spline-interpolate the data from the slice next to the object edges.
     */
    /* */
    s1=(omin-s_step<smin)? smin: omin-s_step;
    s2=(omax+s_step>smax)? smax: omax+s_step;

    //s1=smin;
    //s2=smax;
    cpl_table_and_selected_double(ttab,"SLIT",CPL_GREATER_THAN,s1);
    cpl_table_and_selected_double(ttab,"SLIT",CPL_LESS_THAN,s2);
    xtab=cpl_table_extract_selected(ttab);

    xsh_sort_table_1(xtab,"WAVE",CPL_FALSE);
    cpl_table* utab=xsh_table_unique_wave(xtab,"WAVE", "BFIT");
    xsh_free_table(&xtab);
    cpl_table_save(utab,NULL,NULL,"utab.fits",CPL_IO_DEFAULT);

    //xsh_model_fill_fit_slice_bfit(wlist,order,s1,s2,utab,ron2,gain);


    n=cpl_table_get_nrow(utab);
    double* flux=cpl_table_get_data_double(utab,"BFIT");
    double* wave=cpl_table_get_data_double(utab,"WAVE");
    wmin=cpl_table_get_column_min(utab,"WAVE");
    wmax=cpl_table_get_column_max(utab,"WAVE");
    //cpl_table_save(xtab,NULL,NULL,"utab.fits",CPL_IO_DEFAULT);
    double* swave = cpl_calloc(wlist->list[order].object_size,sizeof(double));
    wavemap_item* pobj=NULL;

    pobj = wlist->list[order].object;
    int sno=wlist->list[order].object_size;
    for(i=0;i<wlist->list[order].object_size;i++,pobj++){
        swave[i]=pobj->lambda;
    }

    cpl_table* otab=cpl_table_new(wlist->list[order].object_size);
    cpl_table_wrap_double(otab,swave,"WAVE");
    cpl_table_and_selected_double(otab,"WAVE",CPL_NOT_LESS_THAN,wmin);
    cpl_table_and_selected_double(otab,"WAVE",CPL_NOT_GREATER_THAN,wmax);
    cpl_table* stab=cpl_table_extract_selected(otab);
    cpl_table_save(stab,NULL,NULL,"stab.fits",CPL_IO_DEFAULT);
    sno=cpl_table_get_nrow(stab);
    swave=cpl_table_get_data_double(stab,"WAVE");
    double* sflux=xsh_bspline_interpolate_data_at_pos(wave,flux,n,swave, sno);
    xsh_free_table(&utab);
    check(cpl_table_wrap_double(stab,sflux,"FIT"));

#if REGDEBUG_BSPLINE
    sprintf(sname,"tab_sampl_ord_%2.2d_%2.2d.fits",abs_order,iter_no);
    cpl_table_save(stab,NULL,NULL,sname,CPL_IO_DEFAULT);
#endif
    sprintf(sname,"tab_sampl_ord_%2.2d_%2.2d.fits",abs_order,iter_no);
    //cpl_table_save(stab,NULL,NULL,sname,CPL_IO_DEFAULT);

    xsh_model_fill_obj(wlist, order, s1, s2, stab, ron2, gain);

    //xsh_model_fill_fit(wlist,order,atab,ron2,gain);
    //xsh_model_fill_fit_slice_bfit(wlist,order,omin,omax,ttab,ron2,gain);




    sprintf(sname,"tab_bspline_smooth_ord_%2.2d_iter_%2.2d.fits",
            abs_order,iter_no);
    //cpl_table_save(ttab,NULL,NULL,sname,CPL_IO_DEFAULT);
    sprintf(sname,"tab_bspline_bkpts_ord_%2.2d_iter_%2.2d.fits",
            abs_order,iter_no);
    //cpl_table_save(sampl_tab,NULL,NULL,sname,CPL_IO_DEFAULT);
    //exit(0);





    /* identify model outliers (bad pixels and lines) */
    check(tab_bad_fit=xsh_detect_outliers_thres_new(wlist,kappa*kappa, ron2,
                     gain,order,iter_no,tab_line_res));

    cleanup:
    xsh_free_table(&otab);
    xsh_free_table(&stab);
    xsh_free_table(&ttab);
    return tab_bad_fit;
}


/*****************************************************************************/
/*****************************************************************************/
static int xsh_wave_compare( const void * un, const void * deux )
{
  wavemap_item * one = (wavemap_item *)un ;
  wavemap_item * two = (wavemap_item *)deux ;

  if ( one->lambda < two->lambda ) return -1 ;
  else if ( one->lambda == two->lambda ) return 0 ;
  return 1 ;
}
/*****************************************************************************/

/*****************************************************************************/
/**
  @brief
    Create the wavemap list
*/
/*****************************************************************************/
double xsh_nbkpts_uvb[XSH_ORDERS_UVB]={2.0,2.0,2.0,2.0,
                                       2.0,2.0,2.0,2.0,
                                       2.0,2.0,2.0,2.0}; //XSH_ORDERS_UVB=12



                             /* order= 16   17  18  19  */
double xsh_nbkpts_vis[XSH_ORDERS_VIS]={1.5,0.8,0.8,0.7,
                                       0.8,0.8,0.7,0.7,
                                       0.7,0.75,0.7,0.6,
                                       0.6,0.6,0.6};     //XSH_ORDERS_VIS=15


/*
double xsh_nbkpts_vis[XSH_ORDERS_VIS]={1.5,0.8,0.8,0.8,
                                       0.8,0.8,0.9,0.9,
                                       0.9,0.8,0.9,0.9,
                                       0.8,0.8,0.6};     //XSH_ORDERS_VIS=15

*/
                               /* right */
//0.28,0.28,0.28,0.3, STD
//0.30,0.28,0.28,0.7
/* REF
double xsh_nbkpts_nir[XSH_ORDERS_NIR]={0.30,0.28,0.28,0.7,
                                       0.9,0.8,0.9,0.9,
                                       0.5,0.7,1.0,0.5,
                                       0.7,0.7,0.7,0.4}; //XSH_ORDERS_NIR=16
*/
double xsh_nbkpts_nir[XSH_ORDERS_NIR]={0.30,0.28,0.28,0.7,
                                       0.9,0.8,0.9,0.9,
                                       0.5,0.7,1.0,0.5,
                                       0.7,0.7,0.7,0.4}; //XSH_ORDERS_NIR=16


static cpl_error_code
xsh_get_obj_and_sky_extraction_slits(xsh_subtract_sky_single_param *sky_par,
                                     xsh_localization* list,
                                     double* obj_slit_min,
                                     double* obj_slit_max,
                                     double* sky_slit_min,
                                     double* sky_slit_max)
{

    double pos1 = sky_par->pos1;
    double hheight1 = sky_par->hheight1;
    double pos2 = sky_par->pos2;
    double hheight2 = sky_par->hheight2;

    /* if hheight1 or hheight2 is defined */
    if ((hheight1 == 0) && (hheight2 == 0) && (list != NULL)) {
        *obj_slit_min = cpl_polynomial_eval_1d(list->edglopoly, 600, NULL);
        *obj_slit_max = cpl_polynomial_eval_1d(list->edguppoly, 600, NULL);

        xsh_msg_dbg_medium("Limit in slit given by localization %f => %f",
                        *obj_slit_min, *obj_slit_max);

    }
    else {
        if (hheight1 > 0) {
            *sky_slit_min = pos1 - hheight1;
            *sky_slit_max = pos1 + hheight1;

            if (hheight2 > 0) {
                double s2_min, s2_max, s1_min, s1_max;

                s1_min = pos1 - hheight1;
                s1_max = pos1 + hheight1;

                s2_min = pos2 - hheight2;
                s2_max = pos2 + hheight2;

                if (s2_min < s1_min) {
                    *sky_slit_min = s2_min;
                }
                else {
                    *sky_slit_min = s1_min;
                }

                if (s2_max > s1_max) {
                    *sky_slit_max = s2_max;
                }
                else {
                    *sky_slit_max = s1_max;
                }

                if (s2_min > s1_max) {
                    *obj_slit_min = s1_max;
                    *obj_slit_max = s2_min;
                }

                if (s1_min > s2_max) {
                    *obj_slit_min = s2_max;
                    *obj_slit_max = s1_min;
                }
            }
        }
        else {
            *sky_slit_min = pos2 - hheight2;
            *sky_slit_max = pos2 + hheight2;
        }
    }

    return cpl_error_get_code();
}

static cpl_error_code
xsh_get_sky_edges_min_max_and_size(xsh_subtract_sky_single_param *sky_par,
                                   const int miny, const int maxy,
                                   const int iorder, const int order,
                                   const int nx, const int wmap_xsize_diff,
                                   xsh_order_list* ord_tab,
                                   cpl_image* slitmap,
                                   double* sky_slit_min,
                                   double* sky_slit_max,
                                   int *max_size, int*max_size_x)
{
    /* NOTE that because sky_par->slit_edges_mask (=0.5arcsec)>0 not all order
     * space is sampled.
     */
    /*
     For each Y calculate the nb of pixels in X.
     */
    *max_size = 0;
    *max_size_x = 0;
    double* pslit = cpl_image_get_data_double( slitmap);
    double sky_slit_min_edge=0;
    double sky_slit_max_edge=0;
    for (int iy = miny; iy < maxy; iy++) {
        int minx=xsh_get_edge_x_min(ord_tab, iorder, iy);
        int maxx=xsh_get_edge_x_max(ord_tab, iorder, iy);

        int offset = iy * (nx - wmap_xsize_diff);
        float slit_val = pslit[minx + offset];

        if (slit_val > sky_slit_max_edge) {
            sky_slit_max_edge = slit_val;
        }
        if (slit_val < sky_slit_min_edge) {
            sky_slit_min_edge = slit_val;
        }

        slit_val = pslit[maxx + offset];

        if (slit_val > sky_slit_max_edge) {
            sky_slit_max_edge = slit_val;
        }
        if (slit_val < sky_slit_min_edge) {
            sky_slit_min_edge = slit_val;
        }
        /* Add nb of pixels in the X range */
        *max_size += (maxx - minx + 1);
        if(maxx - minx + 1 > *max_size_x) {
           *max_size_x = (maxx - minx + 1);
        }
    }
    //xsh_msg("max_size=%d",*max_size);

    XSH_ASSURE_NOT_ILLEGAL( sky_slit_min_edge <= sky_slit_max_edge);
    XSH_CMP_INT( *max_size, >, 0, "Not enough points for the sky order %d",
                 ,order);
    double slit_edges_mask = sky_par->slit_edges_mask;
    slit_edges_mask=0;
    //xsh_msg("sky_edges=%g %g",sky_slit_min_edge,sky_slit_max_edge);
    sky_slit_min_edge = sky_slit_min_edge+slit_edges_mask;
    sky_slit_max_edge = sky_slit_max_edge-slit_edges_mask;
    //xsh_msg("sky_edges=%g %g",sky_slit_min_edge,sky_slit_max_edge);

    double hheight1 = sky_par->hheight1;
    double hheight2 = sky_par->hheight2;
    //xsh_msg("sky height=%g %g",hheight1,hheight2);
    if ( (hheight1 == 0) && (hheight2 == 0)){
        *sky_slit_min = sky_slit_min_edge;
        *sky_slit_max = sky_slit_max_edge;
    }
    else{
        if (sky_slit_min_edge > *sky_slit_min){
            *sky_slit_min = sky_slit_min_edge;
        }
        if (sky_slit_max_edge < *sky_slit_max){
            *sky_slit_max = sky_slit_max_edge;
        }
    }
    //xsh_msg("sky_edges=%g %g",*sky_slit_min,*sky_slit_max);

    cleanup:

    return cpl_error_get_code();
}

static cpl_error_code
xsh_wavecal_list_pupulate(xsh_pre *pre_sci, const int iorder, const int miny,
                          const int maxy, const int wmap_xsize_diff,
                          const int decode_bp, const double obj_slit_min,
                          const double obj_slit_max, const double sky_slit_min,
                          const double sky_slit_max, const int order,
                          xsh_wavemap_list* wave_list, int* sky_size,
                          xsh_order_list* ord_tab, cpl_image *wavemap,
                          cpl_image *slitmap)
{
    /* Populate the wavecal list */
    wavemap_item * psky = wave_list->list[iorder].sky;
    wavemap_item * pobject = wave_list->list[iorder].object;
    wavemap_item * pall = wave_list->list[iorder].all;
    double* plambda = cpl_image_get_data_double( wavemap);
    double* pslit = cpl_image_get_data_double( slitmap);

    float* pflux = cpl_image_get_data_float( xsh_pre_get_data( pre_sci));
    float* perrs = cpl_image_get_data_float( xsh_pre_get_errs( pre_sci));
    int* pqual = cpl_image_get_data_int( xsh_pre_get_qual( pre_sci));

    *sky_size = 0;
    int object_size = 0;
    int all_size = 0;
    double derrs;
    int idx;
    int sdx;
    double lambda_min = 10000., lambda_max = 0. ;
    int nx = xsh_pre_get_nx( pre_sci);
    int ny = xsh_pre_get_ny( pre_sci);
    /*
    xsh_msg("nx=%d ny=%d",nx,ny);
    xsh_msg("slit map nx=%d ny=%d",(int)cpl_image_get_size_x(slitmap),(int)cpl_image_get_size_y(slitmap));
    xsh_msg("wave map nx=%d ny=%d",(int)cpl_image_get_size_x(wavemap),(int)cpl_image_get_size_y(wavemap));
     */
    float slit_val;
    /*
    xsh_msg("miny=%d maxy=%d",miny,maxy);
    xsh_msg("obj slit min=%g max=%g",obj_slit_min,obj_slit_max);
    xsh_msg("sky slit min=%g max=%g",sky_slit_min,sky_slit_max);
    */
    for (int iy = miny; iy < maxy; iy++) {
        //xsh_msg("ok0: iy=%d",iy);
        int minx=xsh_get_edge_x_min(ord_tab, iorder, iy);
        int maxx=xsh_get_edge_x_max(ord_tab, iorder, iy);

        int offset_idx = iy * nx;
        int offset_sdx = iy * (nx - wmap_xsize_diff);
        //xsh_msg("ok2: iy=%d",iy);
        //xsh_msg("minx=%d maxx=%d",minx,maxx);
        //exit(0);
        for (int ix = minx; ix <= maxx; ix++) {

            idx = ix + offset_idx;
            //xsh_msg("ok1: ix=%d idx=%d",ix,idx);
            if ((pqual[idx] & decode_bp) == 0) {
                /* good pix */

                sdx = ix + offset_sdx;
                //xsh_msg("ok1: sdx=%d",sdx);
                double lambda = plambda[sdx];
                if (lambda == 0) {
                    continue;
                }
                if (lambda < lambda_min) {
                    lambda_min = lambda;
                }
                if (lambda > lambda_max) {
                    lambda_max = lambda;
                }
                slit_val = pslit[sdx];

                
                pall->lambda = lambda;
                pall->slit = slit_val;
                pall->flux = *(pflux + idx);
                derrs = *(perrs + idx);

                  //pobject->sigma = (double) 1. / (derrs * derrs);

                pall->sigma = (double) derrs;
                pall->qual = *(pqual + idx);
                pall->ix = ix;
                pall->iy = iy;


                pobject->fitted = 0;
                pobject->fit_err =0;
                psky->fitted = 0;
                psky->fit_err =0;
                pall->fitted = 0;
                pall->fit_err =0;


                all_size++;
                pall++;


                if ( ( (slit_val >= obj_slit_min) && (slit_val <= obj_slit_max) ) ||
                       (slit_val <  sky_slit_min) || (slit_val >  sky_slit_max) ) {
                 


/*
                if ( ( (slit_val >= obj_slit_min) && (slit_val <= obj_slit_max) ) ) {
*/
                    //xsh_msg("object");
                    pobject->lambda = lambda;
                    pobject->slit = slit_val;
                    pobject->flux = *(pflux + idx);
                    derrs = *(perrs + idx);
                    //pobject->sigma = (double) 1. / (derrs * derrs);

                    pobject->sigma = (double) derrs;
                    pobject->qual = *(pqual + idx);
                    pobject->ix = ix;
                    pobject->iy = iy;
                    object_size++;
                    pobject++;
                }
                else {
                    //xsh_msg("sky");
                    psky->lambda = lambda;
                    psky->slit = slit_val;
                    psky->flux = *(pflux + idx);
                    derrs = *(perrs + idx);
                    //psky->sigma = (double) 1. / (derrs * derrs);
                    psky->sigma = (double) derrs;
                    psky->qual = *(pqual + idx);
                    psky->ix = ix;
                    psky->iy = iy;


                    (*sky_size)++;
                    psky++;
                }

            }
            //xsh_msg("ok2: ix=%d idx=%d",ix,idx);
        }
        //xsh_msg("ok3: iy=%d",iy);
    }



    assure(*sky_size > 0, CPL_ERROR_ILLEGAL_INPUT,
           "On order %d sky_size 0. Order edge tab may "
           "over-estimate corresponding order size or "
           "localize-slit-hheight is too large or "
           "sky-slit-edges-mask too large or "
           "sky-hheight1 too small or too large ",
           order);

    wave_list->list[iorder].all_size = all_size;
    wave_list->list[iorder].sky_size = *sky_size;
    wave_list->list[iorder].object_size = object_size;
    wave_list->list[iorder].lambda_min = lambda_min;
    wave_list->list[iorder].lambda_max = lambda_max;
    wave_list->sky_slit_min=sky_slit_min;
    wave_list->sky_slit_max=sky_slit_max;
    wave_list->obj_slit_min=obj_slit_min;
    wave_list->obj_slit_max=obj_slit_max;

    /* Sort by lambda */
    qsort(wave_list->list[iorder].all, all_size, sizeof(wavemap_item),
             xsh_wave_compare);
    qsort(wave_list->list[iorder].sky, *sky_size, sizeof(wavemap_item),
          xsh_wave_compare);
    qsort(wave_list->list[iorder].object, object_size, sizeof(wavemap_item),
          xsh_wave_compare);

    cleanup:
    return cpl_error_get_code();
}

static cpl_error_code
xsh_skycor_def_nbkpts_ord(const int iorder, const int nbkpts,
                          cpl_frame* break_pts_frame, xsh_instrument* inst,
                          int bspline_sampling_method, int* nbkpts_ord)
{

    if (break_pts_frame != NULL) {
        /* if the user specifies different factors use those */
        const char * name = cpl_frame_get_filename(break_pts_frame);
        cpl_table* break_pts_tab = cpl_table_load(name, 1, 0);
        int nrow = cpl_table_get_nrow(break_pts_tab);
        const double* pfactor = cpl_table_get_data_double_const(break_pts_tab,
                                        "FACTOR");
        switch (xsh_instrument_get_arm(inst))
        {
        case XSH_ARM_UVB:
            XSH_ASSURE_NOT_ILLEGAL_MSG(nrow==XSH_ORDERS_UVB,
                            "Wrong number of factors for single frame sky subtraction table");
            xsh_nbkpts_uvb[iorder] = pfactor[iorder];
            break;
        case XSH_ARM_VIS:
            XSH_ASSURE_NOT_ILLEGAL_MSG(nrow==XSH_ORDERS_VIS,
                            "Wrong number of factors for single frame sky subtraction table");
            xsh_msg("iorder=%d factor=%f", iorder, pfactor[iorder]);
            xsh_nbkpts_vis[iorder] = pfactor[iorder];
            break;
        case XSH_ARM_NIR:
            XSH_ASSURE_NOT_ILLEGAL_MSG(nrow==XSH_ORDERS_NIR,
                            "Wrong number of factors for single frame sky subtraction table");
            xsh_nbkpts_nir[iorder] = pfactor[iorder];
            break;
        default:
            xsh_msg("Arm not supported");
            break;
        }
        xsh_free_table(&break_pts_tab);
    }
    double nbkpts_arm=0;
    switch (xsh_instrument_get_arm(inst))
    {
    case XSH_ARM_UVB:
        nbkpts_arm = xsh_nbkpts_uvb[iorder];break;
    case XSH_ARM_VIS:
        nbkpts_arm = xsh_nbkpts_vis[iorder];break;
    case XSH_ARM_NIR:
        nbkpts_arm = xsh_nbkpts_nir[iorder];break;
    default:
        xsh_msg("Arm not supported");break;
    }
    if (bspline_sampling_method == FINE) {
        *nbkpts_ord = (int) (nbkpts * nbkpts_arm + 0.5);
    }
    else {
        *nbkpts_ord = nbkpts;
    }

    cleanup:
    return cpl_error_get_code();
}


static xsh_wavemap_list*
xsh_wavemap_list_new( cpl_frame *wavemap_frame, cpl_frame *slitmap_frame,
                      xsh_localization *list, cpl_frame *order_table_frame,
                      xsh_pre *pre_sci, int nbkpts,
                      cpl_frame* break_points_frame,
                      cpl_frame* ref_sky_list, cpl_frame* sky_orders_chunks,
                      xsh_subtract_sky_single_param *sky_par,
                      xsh_instrument *instrument)
{
    xsh_wavemap_list *wave_list = NULL;
    cpl_image *wavemap=NULL;
    cpl_image *slitmap=NULL;
    const char* wavemap_name=NULL;
    const char* slitmap_name=NULL;

    int wmap_xsize_diff=0;
    double obj_slit_min=0, obj_slit_max=0;
    double sky_slit_min=0, sky_slit_max=0;
    int decode_bp=instrument->decode_bp;
    double gain=sky_par->gain;
    double ron2=sky_par->ron*sky_par->ron;
    xsh_order_list* order_table;
    int nbkpts_ord_max=50000;
    /* Check input parameters */
    XSH_ASSURE_NOT_NULL( wavemap_frame);
    XSH_ASSURE_NOT_NULL( slitmap_frame);
    XSH_ASSURE_NOT_NULL( order_table_frame);
    XSH_ASSURE_NOT_NULL( pre_sci);
    XSH_ASSURE_NOT_NULL( sky_par);
    XSH_ASSURE_NOT_NULL( instrument);

    int nx = xsh_pre_get_nx( pre_sci);
    check( wavemap_name = cpl_frame_get_filename( wavemap_frame));
    check( wavemap = cpl_image_load( wavemap_name, CPL_TYPE_DOUBLE, 0, 0));
    check( slitmap_name = cpl_frame_get_filename( slitmap_frame));
    check( slitmap = cpl_image_load( slitmap_name, CPL_TYPE_DOUBLE,0, 0));
    if( xsh_instrument_get_arm(instrument) == XSH_ARM_NIR){
        wmap_xsize_diff=pre_sci->nx-cpl_image_get_size_x(slitmap);
    }
    if( xsh_instrument_get_arm(instrument) == XSH_ARM_UVB){
        nbkpts_ord_max=90000;
    }
    if( xsh_instrument_get_arm(instrument) == XSH_ARM_VIS){
        nbkpts_ord_max=90000;
    }
    check( wave_list = xsh_wavemap_list_create( instrument));

    xsh_get_obj_and_sky_extraction_slits(sky_par,list,&obj_slit_min,
                                         &obj_slit_max,&sky_slit_min,
                                         &sky_slit_max);

    xsh_msg("ob slit: min=%g max=%g sky slit: min=%g max=%g",
            obj_slit_min,obj_slit_max,sky_slit_min,sky_slit_max);

    //TODO: wmap_xsize_diff seems to be always 0 should be removed?
    check( order_table = xsh_order_list_load (order_table_frame, instrument));
    cpl_table* bkpts_tab;

    const char* fname;
    if ( sky_par->method != MEDIAN_METHOD){
       fname = cpl_frame_get_filename(ref_sky_list);

    }

    if ( sky_par->method == BSPLINE_METHOD ||
         sky_par->method == BSPLINE_METHOD1 ||
         sky_par->method == BSPLINE_METHOD2 ||
         sky_par->method == BSPLINE_METHOD3 ||
         sky_par->method == BSPLINE_METHOD4 )
    {
        /* convert input table frame into cpl table */
         bkpts_tab = cpl_table_load(fname, 1, 0);
         if (cpl_table_get_column_type(bkpts_tab, "WAVELENGTH") == CPL_TYPE_DOUBLE) {
             cpl_table_duplicate_column(bkpts_tab,"TWAVELENGTH",bkpts_tab,"WAVELENGTH");
             cpl_table_erase_column(bkpts_tab,"WAVELENGTH");
             cpl_table_cast_column(bkpts_tab,"TWAVELENGTH","WAVELENGTH",CPL_TYPE_FLOAT);
         }
         if (cpl_table_get_column_type(bkpts_tab, "FLUX") == CPL_TYPE_DOUBLE) {
             cpl_table_duplicate_column(bkpts_tab,"TFLUX",bkpts_tab,"FLUX");
             cpl_table_erase_column(bkpts_tab,"FLUX");
             cpl_table_cast_column(bkpts_tab,"TFLUX","FLUX",CPL_TYPE_FLOAT);
         }
         cpl_table_erase_column(bkpts_tab,"NAME");
         cpl_table_erase_column(bkpts_tab,"COMMENT");

    }

    /* Loop over orders */
    for ( int iorder = 0; iorder < order_table->size; iorder++){

        /* Calculate x,y limits --> minx, maxx, miny, maxy */
        int miny = xsh_order_list_get_starty( order_table, iorder);
        int maxy = xsh_order_list_get_endy( order_table, iorder);

        int starty = order_table->list[iorder].starty/instrument->biny;

        int endy = order_table->list[iorder].endy/instrument->biny;

        miny = ( miny > starty ) ? miny : starty;
        maxy = ( maxy < endy ) ? maxy : endy;

        int order = order_table->list[iorder].absorder ;
        xsh_msg_dbg_medium( "Order %d, miny %d, maxy %d", order, miny, maxy ) ;
        int max_size=0;
        int max_size_x=0;

        xsh_get_sky_edges_min_max_and_size(sky_par, miny, maxy, iorder, order,
                                           nx, wmap_xsize_diff, order_table,
                                           slitmap, &sky_slit_min,
                                           &sky_slit_max, &max_size, &max_size_x);
        /*
        xsh_msg( "SKY data slit range : [%f,%f],[%f,%f]",
                 sky_slit_min, obj_slit_min, obj_slit_max, sky_slit_max);
        */
        check(xsh_wavemap_list_set_max_size( wave_list, iorder, order, max_size));

        int sky_size = 0;
        xsh_wavecal_list_pupulate(pre_sci, iorder, miny, maxy, wmap_xsize_diff,
                                  decode_bp, obj_slit_min, obj_slit_max,
                                  sky_slit_min, sky_slit_max, order, wave_list,
                                  &sky_size, order_table, wavemap, slitmap);

        if ( sky_par->method != MEDIAN_METHOD){
            /* Fit spline */

            int nbkpts_ord=0;
            double* wsampl=NULL;
            cpl_table* wsampl_tab=NULL;
            check(xsh_skycor_def_nbkpts_ord(iorder, nbkpts, break_points_frame,
                            instrument, sky_par->bspline_sampling, &nbkpts_ord));

            double wmin=wave_list->list[iorder].lambda_min;
            double wmax=wave_list->list[iorder].lambda_max;


            cpl_table* tab_ord=NULL;

            if ( sky_par->method == BSPLINE_METHOD ||
                 sky_par->method == BSPLINE_METHOD1 ||
                 sky_par->method == BSPLINE_METHOD2 ||
                 sky_par->method == BSPLINE_METHOD3 ||
                 sky_par->method == BSPLINE_METHOD4 ){

                //xsh_msg("ref sky list: %s nrows=%d",fname,(int)cpl_table_get_nrow(bkpts_tab));
                cpl_table_and_selected_float(bkpts_tab,"WAVELENGTH",CPL_NOT_GREATER_THAN,wmax);
                cpl_table_and_selected_float(bkpts_tab,"WAVELENGTH",CPL_GREATER_THAN,wmin);
                tab_ord=cpl_table_extract_selected(bkpts_tab);
                cpl_table_select_all(bkpts_tab);

            } else if ( sky_par->method == BSPLINE_METHOD5) {

                /* load sampling points from proper extension */
                int orders_nb;
                const char* name;
                if(instrument->arm == XSH_ARM_UVB) {
                    orders_nb=instrument->uvb_orders_nb;
                    name="xsh_kelson_becker_bkpts_uvb.fits";
                } else if (instrument->arm == XSH_ARM_VIS) {
                    orders_nb=instrument->vis_orders_nb;
                    name="xsh_kelson_becker_bkpts_vis.fits";
                } else {
                    orders_nb=instrument->nir_orders_nb;
                    name="xsh_kelson_becker_bkpts_nir.fits";
                }
                xsh_msg("iorder=%d wmin=%g wmax=%g orders=%d",
                                iorder,wmin,wmax,orders_nb);

                wsampl_tab = cpl_table_load(name, orders_nb-iorder, 0);
                cpl_table_name_column(wsampl_tab,"col1","WAVELENGTH");
                cpl_table_divide_scalar(wsampl_tab,"WAVELENGTH",10.);

            }

            cpl_table* tab_line_res=NULL;
            wavemap_item * psky=NULL;
            wavemap_item * psky_start=NULL;

            for(int i=0;i<sky_par->niter;i++) {
                if ( sky_par->method == BSPLINE_METHOD ||
                     sky_par->method == BSPLINE_METHOD1 ||
                     sky_par->method == BSPLINE_METHOD2 ||
                     sky_par->method == BSPLINE_METHOD3 ||
                     sky_par->method == BSPLINE_METHOD4 ){

                    /* correct sampling table if new break points need to be added */
                    int nbkpts=cpl_table_get_nrow(tab_ord);
                    //xsh_msg("selected sampling lines %d wmin=%g wmax=%g",nbkpts,wmin,wmax);
                    if(tab_line_res != NULL) {
                        int nres=cpl_table_get_nrow(tab_line_res);
                        if(nres>0) {
                            if (cpl_table_get_column_type(tab_ord, "FLUX") == CPL_TYPE_INT) {
                                cpl_table_duplicate_column(tab_ord,"FFLUX",tab_ord,"FLUX");
                                cpl_table_erase_column(tab_ord,"FLUX");
                                cpl_table_cast_column(tab_ord,"FFLUX","FLUX",CPL_TYPE_FLOAT);
                                cpl_table_erase_column(tab_ord,"FFLUX");
                                cpl_table_set_column_unit(tab_ord,"FLUX","rel-flux");
                                cpl_table_set_column_unit(tab_ord,"WAVELENGTH","nm");
                            }
                            check(cpl_table_insert(tab_ord,tab_line_res,nbkpts));
                            xsh_sort_table_1(tab_ord,"WAVELENGTH",CPL_FALSE);
                        }
                        xsh_free_table(&tab_line_res);

                    }

                    check(wsampl_tab=xsh_skycorr_wave_sampling_create(wave_list,
                                    iorder,nbkpts_ord,tab_ord));

                }

                check(wsampl=cpl_table_get_data_double(wsampl_tab,"WAVELENGTH"));
                int nrow=0;
                int j=0;
                cpl_table* tab_bad_fit=NULL;
                nbkpts_ord=cpl_table_get_nrow(wsampl_tab);
                xsh_msg("Sampling iter=%d nbkpts_ord=%d",i,nbkpts_ord);
                if(nbkpts_ord > nbkpts_ord_max) {
                    xsh_msg_error("too many break points");
                    exit(0);
                }

                //cpl_table_save(wsampl_tab,NULL,NULL,"line_sampl_tab.fits",CPL_IO_DEFAULT);
                char sname[80];

                int sky_ndata = wave_list->list[iorder].sky_size;
                psky  = wave_list->list[iorder].sky;
                psky_start = wave_list->list[iorder].sky;
                for(j=0, psky=psky_start;j< sky_ndata;j++,psky++) {
                   psky->fitted=0;
                   psky->fit_err=0;
                }
                /*
                xsh_msg("sky slit min=%g max=%g",
                         wave_list->sky_slit_min,wave_list->sky_slit_max);
                         */
                if ( sky_par->method == BSPLINE_METHOD ||
                     sky_par->method == BSPLINE_METHOD1 ||
                     sky_par->method == BSPLINE_METHOD3) {
                    check(tab_bad_fit=xsh_fit_spline1(wave_list,iorder,
                                    wsampl_tab,sky_orders_chunks,nbkpts_ord,i,
                                    sky_par, ron2, gain, &tab_line_res));
                } else if ( sky_par->method == BSPLINE_METHOD2 ||
                            sky_par->method == BSPLINE_METHOD4 ){
                    check(tab_bad_fit=xsh_fit_spline2(wave_list,iorder,
                                    wsampl_tab,sky_orders_chunks,nbkpts_ord,i,
                                    sky_par, ron2, gain, &tab_line_res));


                }

                if(tab_bad_fit!=NULL) {

                      cpl_table_and_selected_int(tab_bad_fit,"FLAG",CPL_EQUAL_TO,1);
                      cpl_table* tab_bp=cpl_table_extract_selected(tab_bad_fit);
                      cpl_table_select_all(tab_bad_fit);
                      cpl_table_and_selected_int(tab_bad_fit,"FLAG",CPL_EQUAL_TO,2);
                      cpl_table* tab_lr=cpl_table_extract_selected(tab_bad_fit);
#if REGDEBUG_BSPLINE
                      cpl_table_save(tab_lr,NULL,NULL,"tab_lr.fits",CPL_IO_DEFAULT);
                      cpl_table_save(tab_bp,NULL,NULL,"tab_bp.fits",CPL_IO_DEFAULT);
#endif
                      xsh_free_table(&tab_bad_fit);
                      cpl_table_erase_column(tab_lr,"FLAG");
                      cpl_table_erase_column(tab_bp,"FLAG");
#if REGDEBUG_BSPLINE
                      sprintf(sname,"tab_lr_ord_%2.2d_%2.2d.reg",order,i);
                      xsh_dump_table_on_region_file(tab_lr,sname,1);
                      sprintf(sname,"tab_lr_ord_%2.2d_%2.2d.fits",order,i);
                      //cpl_table_save(tab_lr,NULL,NULL,sname,CPL_IO_DEFAULT);
                      sprintf(sname,"tab_bp_ord_%2.2d_%2.2d.reg",order,i);
                      xsh_dump_table_on_region_file(tab_bp,sname,2);
                      sprintf(sname,"tab_bp_ord_%2.2d_%2.2d.fits",order,i);
                      //cpl_table_save(tab_bp,NULL,NULL,sname,CPL_IO_DEFAULT);
#endif


                      /* adding sampling points
                      nrow=cpl_table_get_nrow(wsampl_tab);
                      cpl_table_insert(wsampl_tab,tab_lr,nrow);
                      cpl_table* tmp=cpl_table_duplicate(wsampl_tab);
                      xsh_free_table(&wsampl_tab);
                      check(wsampl_tab=xsh_table_unique_column(tmp,"WAVELENGTH"));
                      check(xsh_sort_table_3(wsampl_tab,"WAVELENGTH","Y","X",CPL_TRUE,CPL_TRUE,CPL_TRUE));
                      xsh_free_table(&tmp);
                      */
                      /* removing sampling points */
                      int nbp_row=cpl_table_get_nrow(tab_bp);
                      int nlr_row=cpl_table_get_nrow(tab_lr);
                      int nsrow=cpl_table_get_nrow(wsampl_tab);
                      xsh_msg("Sampling: adding %d removing %d points",nlr_row,nbp_row);
                      xsh_msg("Sampling: original %d points",nbkpts_ord);
                      double* pbw = cpl_table_get_data_double(tab_bp,"WAVELENGTH");
                      double* psw = cpl_table_get_data_double(wsampl_tab,"WAVELENGTH");
                      int nbad=0;
                      for(int k=0;k<nbp_row;k++) {
                          // NOTE that we do not remove the 1st and the last
                          // sampling points in order to have full coverage of
                          // the wavelength range!!

                          for(int j=1;j<nsrow-1;j++) {
                              if(psw[j] == pbw[k]) {
                                  check(cpl_table_set_invalid(wsampl_tab,"WAVELENGTH",j));
                                  nbad++;
                              }
                          }
                      }
                      cpl_table_erase_invalid(wsampl_tab);
                      nbkpts_ord=cpl_table_get_nrow(wsampl_tab);
                      xsh_msg("Sampling: removed %d bad pixels",nbad);
                      xsh_msg("Sampling: After update %d points",nbkpts_ord);
                      //cpl_table_dump_structure(wsampl_tab,stdout);
#if REDDEBUG_SPLINE
                      sprintf(sname,"samplin_ord_%2.2d_%2.2d.reg",order,i);
                      check(xsh_dump_table_on_region_file(wsampl_tab,sname, 4));
                      sprintf(sname,"samplin_ord_%2.2d_%2.2d.fits",order,i);
                      //cpl_table_save(wsampl_tab,NULL,NULL,sname,CPL_IO_DEFAULT);
#endif
                     xsh_free_table(&tab_bp);
                     xsh_free_table(&tab_lr);
                }

                xsh_wavemap_list_rms_sky_image_save(wave_list,ron2,gain,instrument,i);
                check(xsh_wavemap_list_object_image_save( wave_list, instrument, i));
                xsh_wavemap_list_sky_image_save( wave_list, instrument, 0,0,i);

            }
            xsh_free_table(&tab_ord);
            xsh_free_table(&tab_line_res);
            xsh_free_table(&wsampl_tab);
            double rms=0;
            rms=xsh_skycorr_rms(wave_list,iorder,ron2,gain);
            xsh_msg("rms=%g",rms);
            //check(xsh_wavemap_list_sky_image_save(wave_list,instrument,"sky"));

            //exit(0);

            /*
            xsh_msg("sk0");
                      cpl_frame* wlist =
                      xsh_wavemap_list_save(wave_list, order_table_frame, pre_sci, instrument,
                                     "pippo");
                      xsh_msg("sk1");
                      exit(0);
             */
            xsh_msg("Sampling points nbkpts_ord=%d",nbkpts_ord);
        }
        else{
            /* median method */
            int median_hsize = sky_par->median_hsize;
            int nb_points = 2*median_hsize+1;
            double m_pi_2 = 0.5*M_PI;
            cpl_vector* median_vector = cpl_vector_new( nb_points);
            wavemap_item *psky = wave_list->list[iorder].sky;

            for( int i=median_hsize; i< (sky_size-median_hsize); i++){
                double median_err=0.0;
                int pixel = i-median_hsize;
                double wght=0;
                for(int mediani=0; mediani< nb_points; mediani++){
                    cpl_vector_set( median_vector, mediani, psky[pixel+mediani].flux);
                    /* NB: psky[pixel+mediani].sigma contains actually the 1./(sigma*sigma) value
                     * Thus formula after is numerically correct despite "appears" wrong
                     *
                     */
                    //median_err += 1.0/psky[pixel+mediani].sigma;

                    //wght = psky[pixel+mediani].sigma;
                    //median_err += (wght*wght);
                }
                double median = cpl_vector_get_median(median_vector);
                psky[i].fitted = median;
                /* we assume eeror-free model */
                psky[i].fit_err = 0;
            }
            xsh_free_vector( &median_vector);

        }

    }


    xsh_wavemap_list_rms_sky_image_save(wave_list,ron2,gain,instrument,100);
    xsh_wavemap_list_object_image_save( wave_list, instrument, 100);
    xsh_wavemap_list_sky_image_save( wave_list, instrument, 0,0,100);

    //exit(0);
    cleanup :
    xsh_free_image( &slitmap);
    xsh_free_image( &wavemap);
    xsh_order_list_free( &order_table);

    return wave_list ;
}
/*****************************************************************************/

/*****************************************************************************/
/**
  @brief
    Build the sky spectrum list

*/
/*****************************************************************************/
xsh_rec_list*
xsh_wavemap_list_build_sky(xsh_wavemap_list* wave_list, xsh_instrument* instr) {
  xsh_rec_list* sky_list = NULL;
  int i, j;
  int level;

  /* check input parameters */
  XSH_ASSURE_NOT_NULL( wave_list);
  XSH_ASSURE_NOT_NULL( instr);

  xsh_msg("Build sky model");
  level = xsh_debug_level_get();

  check( sky_list = xsh_rec_list_create_with_size( wave_list->size, instr));

  /* Loop over orders */
  for (i = 0; i < wave_list->size; i++) {

    wavemap_item *psky = NULL;
    int order;
    int sky_size;
    double *lambdas = NULL;
    float *sky_flux = NULL;
    float *sky_err = NULL;
    float *sky_qua = NULL;
    float* pflat = NULL;
    int sx = 0;

    psky = wave_list->list[i].sky;
    sky_size = wave_list->list[i].sky_size;
    order = wave_list->list[i].order;

    check( xsh_rec_list_set_data_size( sky_list, i, order, sky_size, 1));
    check( lambdas = xsh_rec_list_get_lambda( sky_list, i));
    check( sky_flux = xsh_rec_list_get_data1( sky_list, i));
    check( sky_err = xsh_rec_list_get_errs1( sky_list, i));
    check( sky_qua = xsh_rec_list_get_qual1( sky_list, i));

    for (j = 0; j < sky_size; j++) {
        lambdas[j] = psky->lambda;
        sky_flux[j] = psky->fitted;
        sky_err[j] = psky->fit_err;
        sky_qua[j] = psky->qual;
        psky++;
    }

    if (level >= XSH_DEBUG_LEVEL_MEDIUM) {
      FILE* debug = NULL;
      char debug_name[256];

      sprintf(debug_name, "fitted_data_sky_%d.log", order);
      debug = fopen(debug_name, "w");
      fprintf(debug, "# lambda flux err x y slit\n");
      psky = wave_list->list[i].sky;

      for (j = 0; j < sky_size; j++) {
        fprintf(debug, "%f %f %f %d %d %f\n", psky->lambda, psky->fitted,
            psky->fit_err, psky->ix, psky->iy, psky->slit);
        psky++;
      }
      fclose(debug);
    }

  }

  cleanup:
  if (cpl_error_get_code() != CPL_ERROR_NONE) {
    xsh_rec_list_free(&sky_list);
  }

  return sky_list;
}

cpl_error_code
xsh_wavemap_list_full_sky_save(xsh_wavemap_list* wave_list, xsh_instrument* instr) {
  xsh_rec_list* sky_list = NULL;
  int i, j;
  int level;

  /* check input parameters */
  XSH_ASSURE_NOT_NULL( wave_list);
  XSH_ASSURE_NOT_NULL( instr);

  xsh_msg("Build sky model");
  level = xsh_debug_level_get();
  int nx=wave_list->instrument->config->nx/wave_list->instrument->binx;
  int ny=wave_list->instrument->config->ny/wave_list->instrument->biny;
  xsh_msg("nx=%d ny=%d",nx,ny);
  check( sky_list = xsh_rec_list_create_with_size( wave_list->size, instr));
  cpl_image* sky_ima=cpl_image_new(nx,ny,CPL_TYPE_FLOAT);
  cpl_image* qua_ima=cpl_image_new(nx,ny,CPL_TYPE_INT);
  float* psky_ima=cpl_image_get_data_float(sky_ima);
  float* pqua_ima=cpl_image_get_data_int(qua_ima);
  /* Loop over orders */
  for (i = 0; i < wave_list->size; i++) {

    wavemap_item * psky = wave_list->list[i].sky;
    int sky_size = wave_list->list[i].sky_size;

    wavemap_item * pobj = wave_list->list[i].object;
    int obj_size = wave_list->list[i].object_size;

    for (j = 0; j < sky_size; j++,psky++) {
        psky_ima[psky->ix+nx*psky->iy]=psky->fitted;
        pqua_ima[psky->ix+nx*psky->iy]=psky->qual;
    }

    for (j = 0; j < obj_size; j++,pobj++) {
        psky_ima[pobj->ix+nx*pobj->iy]=pobj->fitted;
        pqua_ima[pobj->ix+nx*pobj->iy]=pobj->qual;
    }

  }

/*
  cpl_image_save(sky_ima, "sky_model_full.fits",XSH_PRE_DATA_BPP ,wave_list->header, CPL_IO_DEFAULT);
  cpl_image_save(qua_ima, "sky_model_full.fits",XSH_PRE_QUAL_BPP ,wave_list->header, CPL_IO_EXTEND);
*/

  cleanup:
  xsh_rec_list_free(&sky_list);
  xsh_free_image(&sky_ima);
  xsh_free_image(&qua_ima);
  if (cpl_error_get_code() != CPL_ERROR_NONE) {
    xsh_rec_list_free(&sky_list);
  }

  return cpl_error_get_code();
}
static void
xsh_obj_fit_dump(int iorder, xsh_wavemap_list* wave_list)
{
    int level = xsh_debug_level_get();
    if (level >= XSH_DEBUG_LEVEL_MEDIUM) {
        for (iorder = 0; iorder < wave_list->size; iorder++) {
            wavemap_item* pobject = NULL;
            int j, order, object_size;
            FILE* debug = NULL;
            char debug_name[256];

            order = wave_list->list[iorder].order;
            object_size = wave_list->list[iorder].object_size;
            pobject = wave_list->list[iorder].object;

            sprintf(debug_name, "fitted_data_obj_%d.log", order);
            debug = fopen(debug_name, "w");
            fprintf(debug, "# lambda flux err x y slit\n");

            for (j = 0; j < object_size; j++) {
                fprintf(debug, "%f %f %f %d %d %f\n", pobject->lambda,
                                pobject->fitted, pobject->fit_err, pobject->ix,
                                pobject->iy, pobject->slit);
                pobject++;
            }
            fclose(debug);
        }
    }
    return;
}

static cpl_error_code
xsh_subtract_sky_at_pix2(const int all_size,const int sky_size, const int nx,
                         wavemap_item* pall,wavemap_item* psky,
                        float* pflux, float* pflat, float* perrs, int* pqual)
{
    for (int i = 0; i < all_size; i++) {
        int x, y;
        float flux, fitted, err, fitted_err;

        x = pall->ix;
        y = pall->iy;
        int pixel = x + y * nx;
        flux = pflux[pixel];
        fitted = pall->fitted;
        fitted_err = pall->fit_err;
        pflux[pixel] = flux - fitted;
        err = perrs[pixel];
        perrs[pixel] = sqrt(fitted_err * fitted_err + err * err);
        pqual[pixel] = pall->qual;
        pall++;
    }

    for (int i = 0; i < sky_size; i++) {
           int x, y;
           //float flux, fitted, err, fitted_err;
           x = psky->ix;
           y = psky->iy;
           int pixel = x + y * nx;
           //flux = pflux[pixel];
           //fitted = psky->fitted;
           //fitted_err = psky->fit_err;
           //pflux[pixel] /= flux - fitted;
           //err = perrs[pixel];
           //perrs[pixel] = sqrt(fitted_err * fitted_err + err * err);
           pqual[pixel] = psky->qual;
           psky++;
    }
    return cpl_error_get_code();
}

static cpl_error_code
xsh_subtract_sky_at_pix(const int sky_size, const int nx, wavemap_item* psky,
                        float* pflux, float* pflat, float* perrs, int* pqual)
{
    for (int i = 0; i < sky_size; i++) {
        int x, y;
        float flux, fitted, err, fitted_err;

        x = psky->ix;
        y = psky->iy;
        int pixel = x + y * nx;
        flux = pflux[pixel];
        fitted = psky->fitted;
        fitted_err = psky->fit_err;
        pflux[pixel] = flux - fitted;
        err = perrs[pixel];
        perrs[pixel] = sqrt(fitted_err * fitted_err + err * err);
        pqual[pixel] = psky->qual;
        psky++;
    }
    return cpl_error_get_code();
}



static cpl_error_code
xsh_subtract_sky_on_obj_at_pix(xsh_rec_list* sky_list, const int iorder,
                               const int object_size, const int nx,
                               wavemap_item** pobject, float* pflux,
                               float* perrs, int* pqual)
{
    int sky_lambdas_size = xsh_rec_list_get_nlambda( sky_list, iorder);
    double* sky_lambdas = xsh_rec_list_get_lambda( sky_list, iorder);
    float* sky_flux = xsh_rec_list_get_data1( sky_list, iorder);
    float* sky_err = xsh_rec_list_get_errs1( sky_list, iorder);
    int* sky_qua = xsh_rec_list_get_qual1( sky_list, iorder);
    int sky_lambdas_idx=0;
    for (int i = 0; i < object_size; i++) {
        int x, y, obj_qual;
        float flux, err, fitted, fitted_err;
        float lambda, lambda_min, lambda_max;
        float flux_min, flux_max, err_min, err_max;
        float cte_min, cte_max;
        double slope=0;
        x = (*pobject)->ix;
        y = (*pobject)->iy;
        lambda = (*pobject)->lambda;
        obj_qual = (*pobject)->qual;
        int pixel = x + y * nx;
        flux = pflux[pixel];
        while ((sky_lambdas_idx < sky_lambdas_size)
                        && (sky_lambdas[sky_lambdas_idx] <= lambda)) {
            (sky_lambdas_idx)++;
        }
        if (sky_lambdas_idx >= (sky_lambdas_size - 1)) {
            break;
        }
        lambda_max = sky_lambdas[sky_lambdas_idx];
        if (sky_lambdas_idx == 0) {
            (*pobject)++;
            continue;
        }
        lambda_min = sky_lambdas[sky_lambdas_idx - 1];
        flux_min = sky_flux[sky_lambdas_idx - 1];
        flux_max = sky_flux[sky_lambdas_idx];
        err_min = sky_err[sky_lambdas_idx - 1];
        err_max = sky_err[sky_lambdas_idx];
        cte_max = (lambda - lambda_min) / (lambda_max - lambda_min);
        cte_min = 1 - cte_max;


        fitted = flux_min * cte_min + flux_max * cte_max;
        //slope=(flux_max-flux_min)/(lambda_max-lambda_min);
        //fitted = flux_min + slope * (lambda - lambda_min);

        fitted_err = sqrt(
                        err_min * err_min * cte_min
                                        + err_max * err_max * cte_max);
        (*pobject)->fitted = fitted;
        (*pobject)->fit_err = fitted_err;
        pflux[pixel] = flux - fitted;
        err = perrs[pixel];
        perrs[pixel] = sqrt(err * err + fitted_err * fitted_err);
        //pqual[pixel] = obj_qual |= sky_qua[pixel];
        (*pobject)++;
    }

    return cpl_error_get_code();
}

/*****************************************************************************/

/*****************************************************************************/

static void
xsh_image_clean(xsh_pre* pre,const int decode_bp){
    double *spectrum_flux = NULL;
     double *spectrum_errs = NULL;
     int *spectrum_qual = NULL;


     cpl_mask* mask = xsh_qual_to_cpl_mask(pre->qual,decode_bp);

     cpl_mask * filtered = xsh_bpm_filter(mask,5,5,CPL_FILTER_CLOSING);
     xsh_bpmap_mask_bad_pixel(pre->qual,filtered,QFLAG_QUESTIONABLE_PIXEL);
     xsh_free_mask(&mask);
     //cpl_image_save(pre->data, "before.fits", CPL_TYPE_FLOAT, NULL, CPL_IO_CREATE);

     cpl_image_reject_from_mask(pre->data,filtered);

     cpl_image_reject_from_mask(pre->errs,filtered);
     //cpl_detector_interpolate_rejected(pre->data);
     //cpl_detector_interpolate_rejected(pre->errs);

     //cpl_image_save(pre->data, "after.fits", CPL_TYPE_FLOAT, NULL, CPL_IO_CREATE);
     xsh_free_mask(&filtered);

     return;

}

/*****************************************************************************/
/**
  @brief
    subtract sky
*/
/*****************************************************************************/
static cpl_frame*
xsh_wavelist_subtract_sky(xsh_pre * pre_sci, xsh_rec_list* sky_list,
                          xsh_wavemap_list * wave_list, xsh_instrument* instrument,
                          const char* rec_prefix,xsh_subtract_sky_single_param* sky_par) {

  float* perrs = NULL;
  float* pflux = NULL;
  float* pflat = NULL;
  int* pqual = NULL;
  int nx, ny;
  xsh_pre* pre_res = NULL;
  cpl_image* res_image = NULL;
  cpl_frame* res_frame = NULL;
  int iorder;
  char tag[256];
  char fname[256];

  /* check input parameters */
  XSH_ASSURE_NOT_NULL( pre_sci);
  XSH_ASSURE_NOT_NULL( sky_list);
  XSH_ASSURE_NOT_NULL( wave_list);

  check( pre_res = xsh_pre_duplicate( pre_sci));
  check( res_image = xsh_pre_get_data( pre_res));
  check( pflux = cpl_image_get_data_float( res_image));
  check( perrs = cpl_image_get_data_float( xsh_pre_get_errs( pre_res)));
  check( pqual = cpl_image_get_data_int( xsh_pre_get_qual( pre_res)));

  nx = xsh_pre_get_nx(pre_res);
  ny = xsh_pre_get_ny(pre_res);

  for (iorder = 0; iorder < wave_list->size; iorder++) {
    wavemap_item* psky = NULL;
    wavemap_item* pall = NULL;
    wavemap_item* pobject = NULL;
    int order, sky_size, object_size,all_size;

    order = wave_list->list[iorder].order;
    psky = wave_list->list[iorder].sky;
    pall = wave_list->list[iorder].all;
    sky_size = wave_list->list[iorder].sky_size;
    all_size = wave_list->list[iorder].all_size;

    xsh_msg_dbg_medium( "Subtract Sky - Order %d", order);
    if(sky_par->method == BSPLINE_METHOD ||
       sky_par->method == BSPLINE_METHOD1 ||
       sky_par->method == BSPLINE_METHOD2 ||
       sky_par->method == BSPLINE_METHOD3 ||
       sky_par->method == BSPLINE_METHOD4) {  //3
        xsh_subtract_sky_at_pix2(all_size, sky_size, nx, pall, psky, pflux, pflat, perrs,pqual);
    } else {
        xsh_subtract_sky_at_pix(sky_size, nx, psky, pflux, pflat, perrs,pqual);
    }
    object_size = wave_list->list[iorder].object_size;
    pobject = wave_list->list[iorder].object;

    xsh_msg_dbg_medium(
        "Subtract Sky on object - Order %d size %d", order, object_size);
    if(sky_par->method == BSPLINE_METHOD ||
       sky_par->method == BSPLINE_METHOD1 ||
       sky_par->method == BSPLINE_METHOD2 ||
       sky_par->method == BSPLINE_METHOD3 ||
       sky_par->method == BSPLINE_METHOD4) { //3

    } else {

      xsh_subtract_sky_on_obj_at_pix(sky_list,iorder, object_size, nx, &pobject,
                                     pflux, perrs, pqual);

    }
  }

  xsh_obj_fit_dump(iorder, wave_list);

  xsh_wavemap_list_full_sky_save(wave_list, instrument);

  xsh_wavemap_list_object_image_save( wave_list, instrument, 99);
  xsh_wavemap_list_sky_image_save( wave_list, instrument, 0,0,99);
  sprintf(tag, "%s_SUB_SKY_%s", rec_prefix,
      xsh_instrument_arm_tostring(instrument));
  sprintf(fname, "%s.fits", tag);

  /* AMO FILTER filter BP */
  //xsh_image_clean(pre_res,instrument->decode_bp);

  if (strstr(fname, "TMPSKY") != NULL) {
    check( res_frame = xsh_pre_save( pre_res, fname, tag, 1 ));
  } else {
    check( res_frame = xsh_pre_save( pre_res, fname, tag, 0 ));
  }

  check(
      xsh_frame_config(fname,tag,CPL_FRAME_TYPE_IMAGE,CPL_FRAME_GROUP_CALIB, CPL_FRAME_LEVEL_FINAL,&res_frame));
  //xsh_add_temporary_file(fname);

  cleanup:
  xsh_pre_free(&pre_res);


  return res_frame;
}
/*****************************************************************************/
static cpl_error_code
xsh_result_check(cpl_frame* res_frame,cpl_frame* wavemap_frame){
    const char* res_name = cpl_frame_get_filename(res_frame);
    const char* wav_name = cpl_frame_get_filename(wavemap_frame);
    cpl_image* res_ima=cpl_image_load(res_name,CPL_TYPE_DOUBLE,0,0);
    cpl_image* wav_ima=cpl_image_load(wav_name,CPL_TYPE_DOUBLE,0,0);
    double* pres=cpl_image_get_data_double(res_ima);
    double* pwav=cpl_image_get_data_double(wav_ima);
    int nx =cpl_image_get_size_x(res_ima);
    int ny =cpl_image_get_size_y(res_ima);
    int i;
    int nx_ny=nx*ny;
    for(i=0;i<nx_ny;i++) {
        if(pwav[i] == 0.) pres[i]=0.;
    }
    cpl_image_save(res_ima, "obj_result.fits", CPL_BPP_IEEE_FLOAT, NULL, CPL_IO_DEFAULT);

    cpl_image_delete(res_ima);
    cpl_image_delete(wav_ima);
    return cpl_error_get_code();
}

/*****************************************************************************/
/** 
  @brief
    Subtract the sky background for single frame. If sky_spectrum is NOT NULL
    it is saved as a product, of type REC. 
  @param[in] sci_frame 
    Input Science frame after divide by flat
  @param[in] order_table_frame 
    Input Order Table Frame
  @param[in] slitmap_frame 
    Input Wavesolution frame
  @param[in] wavemap_frame 
    Wave map: each pixel content is the wave length
  @param[in] loc_table_frame 
    Localisation table (NULL if not applicable)
  @param[in] usr_defined_break_points_frame 
    user defined break points frame
  @param[in] instrument 
    Pointer to instrument description structure
  @param[in] nbkpts 
    Nb of break points used for Bezier Spline
  @param[in] sky_par
    parameters structure to control the sky subtraction
  @param[out] sky_spectrum  
   The 1D supersampled observed sky spectrum [REC] 
  @param[out] sky_spectrum_eso  
   The 1D supersampled observed sky spectrum [ESO format] 
  @param[in] rec_prefix
   Input recipe prefix

 @return 
   The frame (PRE) after sky subtraction.
 */
/* TODO
 * slitmap_frame,wavemap_frame may be replaced by the physical model
 * usr_defined_break_points_frame should be removed
 * nbkpts should be removed (made automatic optimisation or part of sky_params)
 * why two different parameters for order_table_frame, loc_table_frame?
 */
/*****************************************************************************/
cpl_frame * 
xsh_subtract_sky_single (cpl_frame *sci_frame, 
                         cpl_frame *order_table_frame,
                         cpl_frame *slitmap_frame,
                         cpl_frame *wavemap_frame,
                         cpl_frame *loc_table_frame,
                         cpl_frame *ref_sky_list,
                         cpl_frame *ref_sky_orders_chunks,
                         cpl_frame *usr_def_sampl_points,
                         xsh_instrument *instrument, 
                         int nbkpts, 
                         xsh_subtract_sky_single_param* sky_par,
                         cpl_frame **sky_spectrum,
                         cpl_frame** sky_spectrum_eso,
                         const char* rec_prefix,const int clean_tmp)
{
    cpl_frame *res_frame = NULL ;
    xsh_pre *pre_sci = NULL;
    xsh_wavemap_list *wave_list = NULL;
    xsh_localization *local_list = NULL;
    xsh_rec_list *sky_list = NULL;
    cpl_frame *sky_frame = NULL;
    cpl_frame *sky_frame2 = NULL;
    char fname[256];
    char tag[256];

    /* check input parameters */
    XSH_ASSURE_NOT_NULL_MSG( sci_frame,"Required science frame is missing");
    XSH_ASSURE_NOT_NULL_MSG( order_table_frame,"Required order table frame is missing");
    XSH_ASSURE_NOT_NULL_MSG( slitmap_frame, "Required slitmap frame is missing, provide it or set compute-map to TRUE");
    XSH_ASSURE_NOT_NULL_MSG( wavemap_frame,"Required wavemap frame is missing");
    XSH_ASSURE_NOT_NULL_MSG( instrument,"Instrument setting undefined");
    XSH_ASSURE_NOT_ILLEGAL( nbkpts > 1);
    XSH_ASSURE_NOT_NULL_MSG( sky_par,"Undefined input sky parameters");
    //XSH_ASSURE_NOT_NULL_MSG(ref_sky_list,"Null input reference sky line list");

    xsh_msg_dbg_low( "method %s edges slit mask %f",
                     SKY_METHOD_PRINT( sky_par->method), sky_par->slit_edges_mask);
    if (sky_par->method != MEDIAN_METHOD){
        xsh_msg_dbg_medium("start niter=%d kappa=%f",
                        sky_par->niter, sky_par->kappa);
        if(sky_par->niter>3) {
            sky_par->niter=3;
            xsh_msg_warning("With sky-method=BSPLINE, sky-bspline-niter must be < 4");
        }
    }
    else{
        xsh_msg_dbg_low("median_hsize %d", sky_par->median_hsize);
    }

    if ( loc_table_frame == NULL ) {
        xsh_msg( "Subtract sky single no localization");
    }
    else {
        xsh_msg( "Subtract sky single using localization");
        check( local_list = xsh_localization_load( loc_table_frame));
    }

    check( pre_sci = xsh_pre_load( sci_frame, instrument));

    if (xsh_instrument_get_arm(instrument) == XSH_ARM_NIR) {
        sky_par->gain = 2.12;
        sky_par->ron = 10;/* this should come from a curve: 22 (DIT=2s)- 6 (DIT=100s) */
        double dit = xsh_pfits_get_dit(pre_sci->data_header);
        sky_par->ron = xsh_compute_ron_nir(dit);
    } else {
        sky_par->gain = xsh_pfits_get_gain(pre_sci->data_header);
        sky_par->ron = xsh_pfits_get_ron(pre_sci->data_header);
    }

    /* Create data to build spectrum */
    check( wave_list = xsh_wavemap_list_new( wavemap_frame, slitmap_frame,
                    local_list, order_table_frame, pre_sci, nbkpts,
                    usr_def_sampl_points,ref_sky_list,ref_sky_orders_chunks,
                    sky_par, instrument));

    /* build sky spectrum */
    check (sky_list = xsh_wavemap_list_build_sky(wave_list,instrument));
    /* AMO added to debug: Note that the sky list is a 1D spectrum (order by order
     * with error and qualifier information) */
    //cpl_frame* tmp_frame = xsh_rec_list_save2( sky_list,"sky_list.fits","TEST");

    sprintf(tag,"%s_DRL_SKY_ORD1D_%s", rec_prefix,
            xsh_instrument_arm_tostring(instrument));
    sprintf(fname,"%s.fits",tag);

    check( sky_frame = xsh_rec_list_save( sky_list,fname,tag, CPL_TRUE));
    xsh_add_temporary_file(fname);

    if ( sky_spectrum != NULL){
        *sky_spectrum = cpl_frame_duplicate( sky_frame);
    }

    sprintf( tag,"%s_SKY_ORD1D_%s",  rec_prefix,
             xsh_instrument_arm_tostring(instrument));
    sprintf(fname,"%s.fits",tag);

    check( sky_frame2 = xsh_rec_list_save2( sky_list,fname,tag));

    if( (clean_tmp==1) || (strstr(rec_prefix,"TMPSKY")!=NULL) ) {
        xsh_add_temporary_file(fname);
    }

    if ( sky_spectrum != NULL){
        *sky_spectrum_eso = cpl_frame_duplicate( sky_frame2);
    }

    /* Now subtract sky */
    check( res_frame = xsh_wavelist_subtract_sky( pre_sci, sky_list,
                    wave_list,instrument,
                    rec_prefix,sky_par));

    //check(xsh_result_check(res_frame,wavemap_frame));

    cleanup:
    xsh_free_frame( &sky_frame);
    xsh_free_frame( &sky_frame2);
    xsh_pre_free( &pre_sci);
    xsh_localization_free( &local_list);
    xsh_wavemap_list_free( &wave_list);
    xsh_rec_list_free( &sky_list);

    return res_frame;
}
/*****************************************************************************/
/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
cpl_frame* xsh_save_sky_model( cpl_frame* obj_frame, cpl_frame* sub_sky_frame,
			       const char* sky_tag,xsh_instrument* instrument)
{
   char sky_name[256];
   cpl_frame* result=NULL;
   
   sprintf(sky_name,"%s.fits",sky_tag);
   result=xsh_pre_frame_subtract( obj_frame, sub_sky_frame, sky_name, instrument,0);

   cpl_frame_set_filename(result,sky_name);
   cpl_frame_set_tag(result,sky_tag);

   return result;
}
/*----------------------------------------------------------------------------*/

/*----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
cpl_frame * xsh_add_sky_model( cpl_frame *subsky_frame, cpl_frame *sky_frame,
  xsh_instrument * instrument, const char* prefix)
{
  char result_tag[256];
  char result_name[256];
  cpl_frame *result = NULL;
  xsh_pre *pre_sci = NULL;
  const char *sky_name = NULL;
  cpl_image *sky_img = NULL;

  sprintf( result_tag, "%s_OBJ_AND_SKY_NOCRH_%s", prefix,
    xsh_instrument_arm_tostring(instrument));
  sprintf( result_name, "%s.fits", result_tag);


  check( pre_sci = xsh_pre_load( subsky_frame, instrument));
    
  check( sky_name = cpl_frame_get_filename(sky_frame));
  /* we put back just the sky level: for this reason we
   * correct only the data part, not the associated errors */
  check( sky_img = cpl_image_load( sky_name, CPL_TYPE_FLOAT, 0, 0));
  check( cpl_image_add( pre_sci->data, sky_img));
  check( result = xsh_pre_save( pre_sci, result_name, result_tag, 1));

  cleanup:
    xsh_free_image( &sky_img);
    xsh_pre_free( &pre_sci);

    return result;     
}
/*----------------------------------------------------------------------------*/

/**@}*/
