/*                                                                            *
 *   This file is part of the ESO X-shooter Pipeline                          *
 *   Copyright (C) 2006 European Southern Observatory                         *
 *                                                                            *
 *   This library is free software; you can redistribute it and/or modify     *
 *   it under the terms of the GNU General Public License as published by     *
 *   the Free Software Foundation; either version 2 of the License, or        *
 *   (at your option) any later version.                                      *
 *                                                                            *
 *   This program is distributed in the hope that it will be useful,          *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of           *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the            *
 *   GNU General Public License for more details.                             *
 *                                                                            *
 *   You should have received a copy of the GNU General Public License        *
 *   along with this program; if not, write to the Free Software              *
 *   Foundation, 51 Franklin St, Fifth Floor, Boston, MA  02111-1307  USA     *
 *                                                                            */

/*
 * $Author: amodigli $
 * $Date: 2012-12-01 18:42:28 $
 * $Revision: 1.50 $
 * $Name: not supported by cvs2svn $
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

/*----------------------------------------------------------------------------*/
/**
 * @defgroup xsh_compute_linearity  Compute Linearity (xsh_compute_linearity)
 * @ingroup drl_functions
 *
 * Functions used to compute the linearity bad pixel mask (recipe
 * xsh_linear)
 */
/*----------------------------------------------------------------------------*/
/**@{*/

/*-----------------------------------------------------------------------------
  Includes
  -----------------------------------------------------------------------------*/

#include <math.h>
#include <xsh_drl.h>

#include <xsh_badpixelmap.h>
#include <xsh_data_pre.h>
#include <xsh_dfs.h>
#include <xsh_pfits.h>
#include <xsh_error.h>
#include <xsh_msg.h>
#include <xsh_fit.h>

#include <xsh_compute_linearity.h>

#include <cpl.h>

#define USE_CPL_POLYNOMIAL 
#define OPTIMIZE_IMAGE_HANDLING

/*-----------------------------------------------------------------------------
  Functions prototypes
  -----------------------------------------------------------------------------*/

/*-----------------------------------------------------------------------------
  Implementation
  -----------------------------------------------------------------------------*/

/** 
 * Function used by qsort to order frames by increasing exptime.
 * 
 * @param one Pointer to first item to be compared
 * @param two Pointer to second item ...
 * 
 * @return 1 if first exptime is > than second, -1 otherwise
 */
static int
orderCompare (const void *one, const void *two)
{
  TIME_FRAME *un = (TIME_FRAME *) one;
  TIME_FRAME *deux = (TIME_FRAME *) two;

  if (un->exptime <= deux->exptime)
    return -1;
  else
    return 1;
}

/** 
 * Function used by qsort to order frames by ON/OFF Lamp keyword.
 * 
 * @param one Pointer to first item to be compared
 * @param two Pointer to second item ...
 * 
 * @return 1 if first on_off flag is > than second, -1 otherwise
 */
static int
onoffCompare (const void *one, const void *two)
{
  TIME_FRAME *un = (TIME_FRAME *) one;
  TIME_FRAME *deux = (TIME_FRAME *) two;

  if (un->on_off <= deux->on_off)
    return 1;
  else
    return -1;
}

/** 
 * Subtracts OFF frames from ON frames with same DIT (NIR only)
 * 
 * @param set Frameset of all the frames at same DIT
 * @param instrument Pointer to instrument structure
 * 
 * @return Frameset of the subtracted frames
 */
cpl_frameset * xsh_subtract_on_off( cpl_frameset *set,
				    xsh_instrument *instrument )
{
  cpl_frame *currOn = NULL, *currOff = NULL;
  cpl_frameset * subSet = NULL ;
  int nframes, i ;
  xsh_pre * subtracted = NULL ;
  xsh_pre *preOn = NULL, * preOff = NULL ;

  /*
    Subtraction: 1-2, 3-4, ...
  */
  xsh_msg( "===> Subtract on_off" ) ;

  nframes = cpl_frameset_get_size( set ) ;
  subSet = cpl_frameset_new() ;
  assure( subSet != NULL, cpl_error_get_code(),
	  "Cant create new frameset" ) ;

  for( i = 0 ; i<nframes ; i+=2 ) {
    const char *filename = NULL;
    currOn = cpl_frameset_get_frame(set,i);
    assure( currOn != NULL, cpl_error_get_code(),
	    "Cant get frame of frameset" ) ;
    filename = cpl_frame_get_filename (currOn);
    xsh_msg_dbg_low( " Subtracting ON : %s", filename ) ;
    check_msg( preOn = xsh_pre_load( currOn, instrument ),
	       "Cant load PRE ON" ) ;

    currOff = cpl_frameset_get_frame (set,i+1);
    assure( currOff != NULL, cpl_error_get_code(),
	    "Cant get frame of frameset" ) ;
    filename = cpl_frame_get_filename (currOff);
    xsh_msg_dbg_low( "             OFF: %s", filename ) ;
    check_msg( preOff = xsh_pre_load( currOff, instrument ),
	       "Cant load PRE frame OFF" ) ;

    /* Now substract images, save and add to frameset */
    subtracted = xsh_pre_duplicate(preOn);
    xsh_pre_subtract( subtracted, preOff ) ;
    assure( subtracted != NULL, cpl_error_get_code(),
	    "Cant subtract images" ) ;
    /* save subtracted  */
    {
      cpl_frame * saved = NULL ;
      char outname[128] ;
      const char *tag = XSH_LINEARITY_NIR ;

      sprintf( outname, "linear_sub_set_%d.fits", i ) ;
      saved = xsh_pre_save( subtracted, outname, tag,1 ) ;
      assure( saved != NULL, cpl_error_get_code(),
	      "Cant save subtracted frame" ) ;
      /* Add a valid TAG (missing in the function xsh_pre_save) */
      check_msg( cpl_frame_set_tag( saved, tag ),
		 "Cant set frame tag" ) ;
      /* insert into frameset */
      check_msg( cpl_frameset_insert( subSet, saved ),
		 "Cant insert frame %d into subSet", i ) ;
    }
    /* Cleanup memory */
    xsh_pre_free( &preOn ) ;
    xsh_pre_free( &preOff ) ;
    xsh_pre_free( &subtracted ) ;
  }

 cleanup:
  if ( cpl_error_get_code() != CPL_ERROR_NONE ) {
    xsh_free_frameset( &subSet ) ;
    subSet = NULL ;
  }
  xsh_pre_free( &preOn ) ;
  xsh_pre_free( &preOff ) ;
  xsh_pre_free( &subtracted ) ;
  return subSet ;
}

/** 
 * Create a list of framesets, each frameset containing frames with
 * the same exposure time (+- tolerance).
 * One should find at least 3 groups, each group with 3 frames.
 * If arm is NIR, also spread ON and OFF frames into different framesets.
 * 
 * @param raws Frameset of raw frames (LINEARITY)
 * @param instrument Pointer to instrument description structure
 * @param exp_toler Tolerance on same exposure time in a group
 * @param groupSet Pointer to list of framesets, one frameset per group
 * 
 * @return Number of groups of same exposure time
 */
int
xsh_linear_group_by_exptime (cpl_frameset * raws, xsh_instrument* instrument,
			     double exp_toler, cpl_frameset ** groupSet)
{
  /* for all frames in the frame set, get the EXPTIME or DIT (if NIR)
     group by same EXPTIME (or DIT if NIR) (+- exp_toler)
     and by ON/OFF flags (if NIR)
     returns the number of groups and the list of framesets
  */
  int i, ngroups = 0;
  int nframes = 0;
  TIME_FRAME *expOrdered = NULL;
  TIME_FRAME *porder = NULL;
  cpl_frame *current = NULL;
  double temps = 0.0;
  int onoff = 0;
  cpl_frameset **pgrpset = NULL;

  nframes = cpl_frameset_get_size (raws);
  xsh_msg( "Nb of Frames = %d", nframes ) ;

  expOrdered = cpl_malloc (nframes * sizeof (TIME_FRAME));
  assure (expOrdered != NULL, cpl_error_get_code (),
	  "Cant create tempory time+fframe array");

  porder = expOrdered;

  for ( i = 0 ; i<nframes ; i++, porder++) {
    const char *filename;
    cpl_propertylist *header;
    current = NULL ;

    current = cpl_frameset_get_frame (raws,i);

    assure( current != NULL, CPL_ERROR_ILLEGAL_INPUT,
	    "Cant get Frame #%d from frameset", i ) ;
    /* Load propertylist of current frame */
    filename = cpl_frame_get_filename (current);
    xsh_msg_dbg_low ("load propertylist file %s\n", filename);
    check_msg (header = cpl_propertylist_load (filename, 0),
	       "Could not load header of file '%s'", filename);

    /* Get EXPTIME (or DIT in case of NIR) and store */
    if ( instrument->arm == XSH_ARM_NIR )
      porder->exptime = xsh_pfits_get_dit (header);
    else porder->exptime = xsh_pfits_get_exptime( header ) ;

    porder->frame = current;
    if ( instrument->arm == XSH_ARM_NIR ) {
      bool onoffs = false;
      check(onoffs = xsh_pfits_get_lamp_on_off( header )) ;
      if ( onoffs ) porder->on_off = 1;
      else porder->on_off = 0 ;
    }
    xsh_msg_dbg_low ("Exptime[%d] = %lf", i, porder->exptime);
    xsh_free_propertylist( &header ) ;
  }

  /* Order the array by increasing EXPTIME/DIT */
  xsh_msg_dbg_low ("==== Before qsort");
  qsort (expOrdered, nframes, sizeof (TIME_FRAME), orderCompare);

  /* In addition, if NIR, separate ON and OFF frames */
  {
    TIME_FRAME *first = expOrdered;
    //TIME_FRAME *last ;
    int nb = 0 ;

    temps = expOrdered->exptime ;

    for (i = 0, porder = expOrdered; i < nframes; i++, porder++) {
      if ( porder->exptime == temps ) {
	//last = porder ;
	nb++ ;
      }
      else {
 	xsh_msg_dbg_low( "Call ON/OFF compare (%d frames)", nb ) ;
 	qsort( first, nb, sizeof (TIME_FRAME), onoffCompare ) ;
	first = porder ;
	nb = 1 ;
	temps = porder->exptime ;
      }
    }
    /* Do the last group */
    qsort( first, nb, sizeof (TIME_FRAME), onoffCompare ) ;
  }
    
  /*  if ( cpl_msg_get_level() == CPL_MSG_DEBUG ) */ {
    xsh_msg_dbg_low ("==== After qsort");
    for (i = 0, porder = expOrdered; i < nframes; i++, porder++)
      xsh_msg_dbg_low("    DIT[%d] = %lf (ON/OFF = %d)", i, porder->exptime,
	      porder->on_off);
  }

  /* Now create groups */
  porder = expOrdered;
  temps = porder->exptime;
  onoff = porder->on_off ;

  pgrpset = groupSet;

  xsh_msg_dbg_low ("New frameset 0 starting  at Frame 0" );
  *pgrpset = cpl_frameset_new ();
  check_msg (cpl_frameset_insert (*pgrpset, porder->frame),
	     "Cant insert frame %d in group %d", i, ngroups);
  porder++;
  ngroups = 1;
  for (i = 1; i < nframes; i++, porder++) {
    if (fabs (porder->exptime - temps) > exp_toler || 
	porder->on_off != onoff ) {
      xsh_msg_dbg_low ("New frameset %d starting  at Frame %d", ngroups, i);
      ngroups++;
      temps = porder->exptime;
      onoff = porder->on_off ;
      pgrpset++;
      *pgrpset = cpl_frameset_new ();
      assure( *pgrpset != NULL, cpl_error_get_code(),
	      "Cant create new framest" ) ;
    }
    else
      xsh_msg_dbg_low ("Same group frame %d - %lf vs %lf", i,
		     porder->exptime, temps);
    xsh_msg_dbg_low ("Add frame %d to frameset group %d", i, ngroups);
    check_msg (cpl_frameset_insert (*pgrpset, porder->frame),
	       "Cant insert frame %d in group %d", i, ngroups);
  }

 cleanup:
  if ( expOrdered != NULL ) cpl_free( expOrdered ) ;
  return ngroups;
}

#if !defined(USE_CPL_POLYNOMIAL)

/** 
 * This function calculates, pixel by pixel, the coefficients of the
 * polynomial fits. It replaces (temporarily) the CPL function that is
 * supposed to do the same job because the CPL function (CPL3.0) does not
 * work as expected (no constant coefficient). It shall be replaced by
 * the CPL version when available.
 * 
 * @param medlist List of Images to handle
 * @param dit Array of DIT/EXPTIME
 * @param nframes Number of frames
 * @param nx Image size first axis
 * @param ny Image size second axis
 * @param degree Polynomial degree
 * 
 * @return The "data cube" (ie imagelist) of the polynomial coefficients
 */
static cpl_imagelist *xsh_imagelist_fit_polynomial( cpl_imagelist * medlist,
					     double *dit, int nframes,
					     int nx, int ny,
					     int degree )
{
  int ll, ix, iy ;
  cpl_vector * pixvect = NULL;
  cpl_vector * temps = NULL;
  cpl_polynomial *poly = NULL;
  cpl_imagelist *poly_list = NULL, /**< Temporary polynomial coefficients
				    image planes */
    *final_list = NULL ;	/**< Final .... */
  float **pixbuf = NULL ;	/**< Array of images data */
  cpl_binary **maskbuf = NULL ;	/**< Array of images BPM masks */

  assure( medlist != NULL, CPL_ERROR_NULL_INPUT,
	  "Input list of median images is NUL !" ) ;

  poly_list = cpl_imagelist_new() ;
  assure( poly_list != NULL, cpl_error_get_code(),
	  "Cant create imagelist" ) ;
  /*
    Create the "data cube" (image list) to store the polynomial coefficients
  */
  for ( ll = 0 ; ll<= degree ; ll++ ) {
    cpl_image *newone = NULL ;

    newone = cpl_image_new( nx, ny, CPL_TYPE_DOUBLE ) ;
    assure( newone != NULL, cpl_error_get_code(),
	    "Cant create new image" ) ;
    cpl_imagelist_set( poly_list, newone, ll ) ;
  }

  /* 
     Loop over pixels.
     Populate a data vector with the content of same pixel of all
       the images
     Fit polynomial
     Populate the resulting imagelist with the polynome coefficients
  */
  xsh_msg( "Polynomial Fit - XSH version - nb of frames: %d", nframes ) ;
#if 1
  /*
    For better efficiency, one could/should use the pointer to pixel data
    in the image instead of using cpl_image_get function.
    Use cpl_image_get_data (input frames are PRE format ==> Float data)
  */

  pixbuf = cpl_malloc( nframes * sizeof(double *) ) ;
  assure( pixbuf != NULL, cpl_error_get_code(),
	  "Cant allocate memory for pixel data" ) ;
  maskbuf = cpl_malloc( nframes * sizeof(cpl_binary *) ) ;
  assure( maskbuf != NULL, cpl_error_get_code(),
	  "Cant allocate memory for BPM data" ) ;
  /* Creating list of data and masks */
  for( ll = 0 ; ll < nframes ; ll++ ) {
    cpl_image *pcur = cpl_imagelist_get( medlist, ll ) ;

    *(pixbuf+ll) = cpl_image_get_data( pcur ) ;
    *(maskbuf+ll) = cpl_mask_get_data( cpl_image_get_bpm( pcur ) ) ;
  }

  /* Loop over pixels */
  for ( iy = 0 ; iy < ny ; iy++ )
    for( ix = 0 ; ix < nx ; ix++ ) {
      cpl_binary ** pmask = maskbuf ;
      float ** cur = pixbuf ;
      int npix = 0 ;

      pixvect = cpl_vector_new( nframes ) ;
      assure( pixvect != NULL, cpl_error_get_code(),
	      "Cant create PixVect Vector" ) ;
      temps = cpl_vector_new( nframes ) ;
      assure( temps != NULL, cpl_error_get_code(),
	      "Cant create Time Vector" ) ;
      xsh_msg_dbg_low( "++ Fit one pixel: %d,%d", ix, iy ) ;

      for( ll = 0 ; ll<nframes ; ll++ , pmask++, cur++ ) {
	double pixval ;
	cpl_binary rej ;
	float *pcur = *cur ;
	cpl_binary *mask = *pmask ;

	pixval = *(pcur + (iy*nx) + ix ) ;
	rej = *(mask + (iy*nx) + ix ) ;
	if ( rej == 0 ) {
	  cpl_vector_set( temps, ll, *(dit+ll) ) ;
	  cpl_vector_set( pixvect, ll, pixval ) ;
	  npix++ ;
	}
      }
      xsh_msg_dbg_low( "  Nb of pixels: %d", npix ) ;
      /* Adjust the size of the vector to npix */
      if ( npix != nframes ) {
	cpl_vector_set_size( temps, npix ) ;
	cpl_vector_set_size( pixvect, npix ) ;
      }
      /* Now fit */
      poly = xsh_polynomial_fit_1d_create( temps, pixvect,
					   degree, NULL ) ;
      xsh_free_vector( &temps ) ;
      xsh_free_vector( &pixvect ) ;
      assure( poly != NULL, CPL_ERROR_ILLEGAL_INPUT,
	      "Error calling xsh_polynomial_fit_1d_create" ) ;
      /* And fill the resulting coefficients frames */
      for( ll = 0 ; ll <= degree ; ll++ ) {
	cpl_image_set( cpl_imagelist_get( poly_list, ll ),
		       ix+1, iy+1, cpl_polynomial_get_coeff( poly, &ll ) ) ;
      }
      xsh_free_polynomial( &poly ) ;
    }
#else
  for ( iy = 1 ; iy <= ny ; iy++ )
    for( ix = 1 ; ix <= nx ; ix++ ) {
      int npix = 0 ;

      pixvect = cpl_vector_new( nframes ) ;
      assure( pixvect != NULL, cpl_error_get_code(),
	      "Cant create PixVect Vector" ) ;
      temps = cpl_vector_new( nframes ) ;
      assure( temps != NULL, cpl_error_get_code(),
	      "Cant create Time Vector" ) ;
      xsh_msg_dbg_low( "++ Fit one pixel: %d,%d", ix, iy ) ;
      for( ll = 0 ; ll < nframes ; ll++ ) {
	double pixval ;
	int rej ;
	cpl_image *cur = NULL ;
	cur = cpl_imagelist_get( medlist, ll ) ;
	assure( cur != NULL, cpl_error_get_code(),
		"Cant get image #%d from liste" ) ;
	/* Add pixel value to vector */
	pixval = cpl_image_get( cur, ix, iy, &rej ) ;
	xsh_msg_dbg_low( " Pixval: %lf", pixval ) ;
	if ( rej == 0 ) {
	  cpl_vector_set( temps, ll, *(dit+ll) ) ;
	  cpl_vector_set( pixvect, ll, pixval ) ;
	  npix++ ;
	}
      }
      xsh_msg_dbg_low( "  Nb of pixels: %d", npix ) ;
      /* Adjust the size of the vector to npix */
      if ( npix != nframes ) {
	cpl_vector_set_size( temps, npix ) ;
	cpl_vector_set_size( pixvect, npix ) ;
      }
      /* Now fit */
      poly = xsh_polynomial_fit_1d_create( temps, pixvect,
					   degree, NULL ) ;
      assure( poly != NULL, cpl_error_get_code(),
	      "Error calling xsh_polynomial_fit_1d_create" ) ;
      xsh_free_vector( &temps ) ;
      xsh_free_vector( &pixvect ) ;
      /* And fill the resulting coefficients frames */
      //xsh_msg( "  Pixel %d,%d:", ix, iy ) ;
      for( ll = 0 ; ll <= degree ; ll++ ) {
	cpl_image_set( cpl_imagelist_get( poly_list, ll ),
		       ix, iy, cpl_polynomial_get_coeff( poly, &ll ) ) ;
      }
      xsh_free_polynomial( &poly ) ;
    }
#endif

  final_list = cpl_imagelist_duplicate( poly_list ) ;

 cleanup:
  xsh_free_imagelist( &poly_list ) ;
  xsh_free_polynomial( &poly ) ;
  xsh_free_vector( &temps ) ;
  xsh_free_vector( &pixvect ) ;
  xsh_free_imagelist( &poly_list ) ;
  cpl_free( pixbuf ) ;
  cpl_free( maskbuf ) ;

  return final_list ;
}
#endif

/** 
 * Computes the linearity bad pixel mask from median frames with increasing
 * exposure times.
 * 
 * @param medSet The frameset of median frames (as returned by remove_crh)
 * @param instrument Pointer to the instrument description structure
 * @param lin_clipping Parameters of linearity sigma clipping
 * 
 * @return The linearity bad pixel mask
 */
cpl_frame *
xsh_compute_linearity (cpl_frameset * medSet, xsh_instrument *instrument,
		       xsh_clipping_param * lin_clipping,const int decode_bp)
{
  cpl_image * resBpmap = NULL ;	/**< The final result bad pixel map image  */
  int nframes = 0 ;
  cpl_frame * resFrame = NULL ;	/**< The final frame */
  int i = 0, ix = 0, iy = 0, nx = 0, ny = 0;
  int nbad, totbad = 0 ;
  cpl_imagelist *medList = NULL ; /**< Temporary image list (to be freed) */
  cpl_imagelist *bpmapList = NULL ; /**< Temporary image list (to be freed) */
  cpl_frame *current = NULL ;
  cpl_propertylist *bpmap_header = NULL ; /**< Temporary propertylist
					     (to be freed) */
  double *Dit = NULL;
#if defined(USE_CPL_POLYNOMIAL)
  cpl_vector * v_dit = NULL ;
  //  cpl_image * img_err = NULL ;
#endif
  cpl_imagelist * polyList = NULL ; /**< Temporary image list (to be freed) */
  

  xsh_msg( "==> xsh_compute_linearity" ) ;
  assure (medSet != NULL, CPL_ERROR_ILLEGAL_INPUT, "Frameset is NULL");
  nframes = cpl_frameset_get_size (medSet);


  /* Create an imagelist from the medSet frameset
     Create the imagelist of bad pixel maps (to build the general one)
  */
  medList = cpl_imagelist_new() ;
  assure( medList != NULL, cpl_error_get_code(),
	  "Cant create medList" ) ;
  bpmapList = cpl_imagelist_new() ; /**< List of Bad pixel maps */
  assure( bpmapList != NULL, cpl_error_get_code(),
	  "Cant create bpmapList" ) ;

  current = cpl_frameset_get_frame( medSet,0 ) ;
  assure( current != NULL, cpl_error_get_code(),
	  "Cant get current frame" ) ;

  /* get the Detector Integration Time */
  Dit = cpl_malloc( nframes * sizeof(double) ) ;
  assure( Dit != NULL, cpl_error_get_code(),
	  "Cant allocate Dit array" ) ;
#if defined(USE_CPL_POLYNOMIAL)
  v_dit = cpl_vector_wrap( nframes, Dit ) ;
  assure( v_dit != NULL, cpl_error_get_code(),
	  "Cant create v_dit vector" ) ;
#endif
  xsh_msg( "Loop over all frames" ) ;

  for( i = 0 ; i<nframes ; i++ ) {
    const char *filename ;
    cpl_propertylist *header = NULL ;
    cpl_image * curimage = NULL ;
    cpl_image *curbpmap = NULL ;    

    filename = cpl_frame_get_filename(current);
    assure( filename != NULL, cpl_error_get_code(),
	    "Cant get filename from frame %d", i ) ;

    curimage = cpl_image_load( filename, XSH_PRE_DATA_TYPE, 0, 0 ) ;
    assure( curimage != NULL, cpl_error_get_code(),
	    "Cant get curimage from frame %d", i ) ;

    cpl_imagelist_set( medList, curimage, i ) ;
    header = cpl_propertylist_load( filename, 0 ) ;
    assure( header != NULL, cpl_error_get_code(),
	    "Cant get header of frame %d", i ) ;

    xsh_msg_dbg_low( "  Getting EXPTIME/DIT[%d]", i ) ;
    /* Get EXPTIME (or DIT in case of NIR) and store */
    if ( instrument->arm == XSH_ARM_NIR )
      *(Dit+i) = xsh_pfits_get_dit( header ) ;
    else *(Dit+i) = xsh_pfits_get_exptime( header ) ;
    cpl_propertylist_delete( header ) ;

    if ( i == 0 ) {
      nx = cpl_image_get_size_x( curimage ) ;
      ny = cpl_image_get_size_y( curimage ) ;
    }

    /* Now get the bad pixel map from this frame (extension Nb 2) */
    xsh_msg_dbg_low( "  Loading BpMap[%d]", i ) ;

    curbpmap = cpl_image_load( filename, CPL_TYPE_INT, 0, 2 ) ;
    assure( curbpmap != NULL, cpl_error_get_code(),
	    "Cant load bpmap frame #%d", i ) ;
    cpl_imagelist_set( bpmapList, curbpmap, i ) ;
    if ( i == 0 ) {
      bpmap_header = cpl_propertylist_load( filename, 2 ) ;
      assure( bpmap_header != NULL, cpl_error_get_code(),
	      "Cant get bpmap header of frame %d", i ) ;
    }
    current = cpl_frameset_get_frame( medSet,i+1 ) ;
    xsh_msg_dbg_low( "-- Next Frame" ) ;
  }

  /* Collapse all bad pixel maps from the frameset into a single one */
  xsh_msg( "Collapsing BpMaps's" ) ;

  resBpmap = xsh_bpmap_collapse_bpmap_create ( bpmapList,decode_bp ) ;
  assure( resBpmap != NULL, cpl_error_get_code(),
	  "Cant create collapsed bpmap" ) ;
  /* memory cleanup */
  xsh_msg_dbg_low( "Freeing bpmapList" ) ;
  xsh_free_imagelist( &bpmapList ) ;

  /* Insert bad pixels into the images of the list before fitting */
  for( i = 0 ; i<nframes ; i++ ) {
    xsh_set_image_cpl_bpmap ( cpl_imagelist_get( medList, i ),
			      resBpmap, decode_bp ) ;
  }

#if defined(USE_CPL_POLYNOMIAL)
  /* CPL imagelist Polynomial fit */
  xsh_msg( "Now fit polynomial - CPL version" ) ;

  /* Use the function provided by J.Larsen (not the "real" CPL one) */
  polyList = xsh_fit_imagelist_polynomial( v_dit, medList, 0,
					   LINEAR_POLYNOMIAL_DEGREE,
					   CPL_FALSE, NULL ) ;
  //  polyList = cpl_imagelist_fit_polynomial( medList, Dit,
  //				   LINEAR_POLYNOMIAL_DEGREE ) ;
  assure( polyList != NULL, cpl_error_get_code(),
	  "Cant fit polynomial - CPL version" ) ;
  xsh_msg( "Polynomial Fit Finished - CPL Version" ) ;
#else
  /*
    The cpl function cpl_imagelist_fit_polynomial does not work
    as expected. Hence we have to create our own fit function,
    based on the CPL function xsh_polynomial_fit_1d_create
  */
  xsh_msg( "Now fit polynomial - XSH version" ) ;
  polyList = xsh_imagelist_fit_polynomial( medList, Dit, nframes, nx, ny,
					   LINEAR_POLYNOMIAL_DEGREE ) ;
  assure( polyList != NULL, cpl_error_get_code(),
	  "Cant fit polynomial - XSH version" ) ;
  xsh_msg( "Polynomial Fit Finished - XSH Version" ) ;
#endif
  /* The cpl bad pixels of polyList are not set according to the bad pixels
     of the input image list ==> do it now
  */
  for( i = 0 ; i<=LINEAR_POLYNOMIAL_DEGREE ; i++ ) {
    check(xsh_set_image_cpl_bpmap ( cpl_imagelist_get( polyList, i ),
				    resBpmap, decode_bp )) ;
  }

  /* Check linearity per pixel */
  xsh_msg_dbg_low( "Clipping sigma: %f", lin_clipping->sigma ) ;

  for( i=0; i <= LINEAR_POLYNOMIAL_DEGREE ; i++ ) {
    /* Check linearity for each polynome parameter */
    cpl_image *pplane = NULL ;
    int iter = 0 ;
    double mean = 0.0, stdev = 0.0, med = 0.0;

    pplane = cpl_imagelist_get( polyList, i ) ;
    assure( pplane != NULL, cpl_error_get_code(),
	    "Gant get plane Nb %d", i ) ;
    /* Iteration:
       Calculate average and sigma (taking into account bad pixels)
       mark bad pixels (bad pixel map is empty for the 1st iteration)
    */

    xsh_msg( "Sigma clipping on coefficient %d", i ) ;
    for( iter = 0 ; iter<lin_clipping->niter ; iter++ ) {
#if defined(OPTIMIZE_IMAGE_HANDLING)
      double *pdata = NULL ;
      cpl_binary *pbpm = NULL ;
#endif
      xsh_msg( "   Iteration Nb %d/%d", iter+1, lin_clipping->niter ) ;
      /* Calculate mean and stdev of current parameter image pplane */
      mean = cpl_image_get_mean( pplane ) ;
      stdev = cpl_image_get_stdev( pplane ) ;
      med = cpl_image_get_median( pplane ) ;
      xsh_msg( "      Mean: %lf, stdev: %lf, Median: %lf",
	       mean, stdev, med ) ;
      /* Loop over good pixels
	 compare current pplane value with sigma*lin_clipping->sigma
	 Mark bad pixels
      */
      nbad = 0 ;
#if defined(OPTIMIZE_IMAGE_HANDLING)
      pdata = cpl_image_get_data_double( pplane ) ;
      assure( pdata != NULL, cpl_error_get_code(),
	      "Cant get pplane[%d] data", i ) ;
      pbpm = cpl_mask_get_data( cpl_image_get_bpm( pplane ) ) ;
      assure( pdata != NULL, cpl_error_get_code(),
	      "Cant get pplane[%d] bpm", i ) ;

      for( iy = 0; iy < ny ; iy++ )
	for( ix = 0 ; ix < nx ; ix++ ) {
	  double coeff = *(pdata + ix + (iy*nx)) ;
	  cpl_binary rej = *(pbpm + ix + (iy*nx)) ;
	  if ( rej != 0 ) {
	    xsh_msg_dbg_low( "  **** rejected %d,%d", ix+1, iy+1 ) ;
	    continue ;
	  }
#else
      for( iy = 1 ; iy <= ny ; iy++ )
	for( ix = 1 ; ix<=nx ; ix++ ) {
	  int rej ;
	  double coeff = cpl_image_get( pplane, ix, iy, &rej ) ;
	  if ( rej != 0 ) {
	    xsh_msg_dbg_low( "  **** rejected %d,%d", ix, iy ) ;
	    continue ;
	  }
#endif
	  //xsh_msg( "   **** %d,%d -> %lf", ix, iy, coeff ) ;
	  if ( fabs( coeff - mean) > stdev*lin_clipping->sigma ) {
	    xsh_msg_dbg_low( "          BAD PIXEL at %d,%d - %lf vs %lf",
		     ix, iy, mean, coeff ) ;
	    /* add bad pixel to this pseudo image for next iteration*/
#if defined(OPTIMIZE_IMAGE_HANDLING)
	    cpl_image_reject( pplane, ix+1, iy+1 ) ;
	    /* add to the global bad pixel map */
	    xsh_bpmap_set_bad_pixel( resBpmap, ix+1, iy+1,
				     QFLAG_NON_LINEAR_PIXEL ) ;
#else
	    cpl_image_reject( pplane, ix, iy ) ;
	    /* add to the global bad pixel map */
	    xsh_bpmap_set_bad_pixel( resBpmap, ix, iy,
				     QFLAG_NON_LINEAR_PIXEL ) ;
#endif
	    nbad++ ;
	  }
	}
      xsh_msg( "      Nbad: %d", nbad ) ;
      if ( nbad == 0 ) break ; // stop iteration if no bad detected
      totbad += nbad ;
    }
    /* Now add mean, rms and med of each coefficient to bpmap header */
    xsh_pfits_set_qc_multi( bpmap_header, (void *)&mean,
			    XSH_QC_BP_MAP_LINi_MEAN, instrument, i ) ;
    xsh_msg_dbg_low( "%s (%d) = %lf", XSH_QC_BP_MAP_LINi_MEAN, i, mean ) ;
    xsh_pfits_set_qc_multi( bpmap_header, (void *)&med,
			    XSH_QC_BP_MAP_LINi_MED, instrument, i ) ;
    xsh_msg_dbg_low( "%s (%d) = %lf", XSH_QC_BP_MAP_LINi_MED, i, med ) ;
    xsh_pfits_set_qc_multi( bpmap_header, (void *)&stdev,
			    XSH_QC_BP_MAP_LINi_RMS, instrument, i ) ;
    xsh_msg_dbg_low( "%s (%d) = %lf", XSH_QC_BP_MAP_LINi_RMS, i, stdev ) ;
  }
  xsh_msg( "====> Total Non linear pixels: %d", totbad ) ;
  /* Add the total nb of bad pixels to the bpmap header */
  {
    int nbadpix = cpl_image_count_rejected( resBpmap ) ;
    xsh_pfits_set_qc( bpmap_header, (void *)&nbadpix,
		      XSH_QC_BP_MAP_NBADPIX, instrument ) ;
    xsh_msg_dbg_low( "%s = %d", XSH_QC_BP_MAP_NBADPIX, nbadpix ) ;
  }


  /* Finished */
  /* Save bpmap image file */
  {
    char bpmapFname[132] ;
    const char *sarm = xsh_instrument_arm_tostring( instrument ) ;

    sprintf( bpmapFname, "LINEARITY_BAD_PIXEL_MAP_%s.fits", sarm ) ;
    check_msg( cpl_image_save( resBpmap, bpmapFname, CPL_BPP_32_SIGNED,
			       bpmap_header, CPL_IO_DEFAULT ),
	       "Cant save resulting BpMap image" ) ;
    //xsh_add_temporary_file(bpmapFname);

    /* insert bpmap into resFrame */

    check(resFrame=xsh_frame_product(bpmapFname,
                                     XSH_GET_TAG_FROM_ARM(XSH_BP_MAP_LIN,
							  instrument),
				     CPL_FRAME_TYPE_IMAGE,
				     CPL_FRAME_GROUP_PRODUCT,
				     CPL_FRAME_LEVEL_FINAL));

  }

 cleanup:
  if ( Dit != NULL ) cpl_free( Dit ) ;
  xsh_free_imagelist( &polyList ) ;
  xsh_free_imagelist( &bpmapList ) ;
  xsh_free_imagelist( &medList ) ;
  xsh_free_propertylist( &bpmap_header ) ;
  xsh_free_image( &resBpmap ) ;

  return resFrame ;

}

/*----------------------------------------------------------------------------*/

/**@}*/
