/**
 * Copyright INRIA and ENS, contributors Peterlongo and Chikhi
 * pierre.peterlongo@inria.fr
 * rayan.chikhi@irisa.fr
 *
 * This software is a computer program whose purpose is to detect the
 * presence of a starter in a set of NGS reads, and to provide the neighbor
 * in case of success..
 *
 * This software is governed by the CeCILL license under French law and
 * abiding by the rules of distribution of free software.  You can  use,
 * modify and/ or redistribute the software under the terms of the CeCILL
 * license as circulated by CEA, CNRS and INRIA at the following URL
 * "http://www.cecill.info".
 
 * As a counterpart to the access to the source code and  rights to copy,
 * modify and redistribute granted by the license, users are provided only
 * with a limited warranty  and the software's author,  the holder of the
 * economic rights,  and the successive licensors  have only  limited
 * liability.
 
 * In this respect, the user's attention is drawn to the risks associated
 * with loading,  using,  modifying and/or developing or reproducing the
 * software by the user in light of its specific status of free software,
 * that may mean  that it is complicated to manipulate,  and  that  also
 * therefore means  that it is reserved for developers  and  experienced
 * professionals having in-depth computer knowledge. Users are therefore
 * encouraged to load and test the software's suitability as regards their
 * requirements in conditions enabling the security of their systems and/or
 * data to be ensured and,  more generally, to use and operate it in the
 * same conditions as regards security.
 *
 * The fact that you are presently reading this means that you have had
 * knowledge of the CeCILL license and that you accept its terms.
 */

/*
 * read_coherence.c
 *
 *  Created on: 22 oct. 2010
 *      Author: rayan chikhi
 */

#ifndef _GNU_SOURCE
#define _GNU_SOURCE // needed for strndup
#endif

#include "../commons.h"
#ifdef __cplusplus
extern "C"
{
#endif
#include "read_coherence.h"
    
    //#define MAX(X,Y) ((X) < (Y) ? (Y) : (X))
    
    //#define debug_groups
    //#define debug_votes
//#define DEBUG_READ_COHERENCE
    
#define SPACE (MAX_VOTES/2) // don't increase this too much, has to fit in MAX_VOTES
    // macro to initialize some variables useful for the hashmap iterator
#define MAPPING_POSITIONS_ITERATOR_DECLARATION()	\
char *key;						\
char *current_read=NULL;				\
int position_fragment;				\
list_ *mapping_positions;				\
int current_read_len=0;				\
hash_iter iterator;
    
    // Go through the hashmap Mapped and for each fragment
    // get the reads that have seeds that match somewhere on this fragment
    // returns:
    // - current_read
    // - current_read_len
    // - position_fragment
#define MAPPING_POSITIONS_ITERATOR_START(map)				\
iterator=hash_iterator_init(map); /* get the first element of the Mapped hashmap*/ \
if((long int)(iterator)!=-1){						\
/*printf("Fragment %s covered by \n", fragment);*/			\
while(!hash_iterator_is_end(map, iterator)){     /* go through all the mapped reads*/ \
/* get key(read) and corresponding mapping positions (pwi) */		\
hash_iterator_return_entry(map, iterator, &key, (void**)&mapping_positions); \
current_read_len=strlen(key); /* needed to compute votes*/		\
current_read=key;							\
if(mapping_positions){						\
cell_mapsembler *c=mapping_positions->first; /* == list_start_iterator(mapping_positions) */ \
while(c!=NULL){ /* == list_is_end_iterator(mapping_positions) */	\
p_couple value= (p_couple) c->val; /* == list_get_current_element(mapping_positions) */ \
position_fragment=value->a;
    
    // end iterator
#define MAPPING_POSITIONS_ITERATOR_END(map)		\
c=c->prox; /* == list_next(mapping_positions) */	\
}							\
}							\
iterator=hash_iterator_next(map, iterator);		\
}							\
}
    
#define SORTED_WEIGHTED_MAPPING_POSITIONS_ITERATOR_DECLARATION()	\
list_ *sorted_reads;							\
cell_mapsembler *read;						\
int current_read_weight;
    
    
    
    
#define SORTED_WEIGHTED_MAPPING_POSITIONS_ITERATOR_START(map)		\
sorted_reads = (list_ *)hash_list_sorted_by_positions(map);		\
read=sorted_reads->first;						\
while (read != NULL)							\
{									\
/* discard duplicated reads by testing if next read == current read*/ \
current_read_weight=1;						\
while (read->prox!=NULL && strcmp((char *)((p_charint_couple)(read->val))->a,(char *)((p_charint_couple)(read->prox->val))->a)==0) \
{									\
read = read->prox;						\
current_read_weight++;						\
}									\
char *current_read=(char *)((p_charint_couple)(read->val))->a;	\
assert( current_read != NULL);					\
int position_fragment=((p_charint_couple)(read->val))->b;		\
int current_read_len=((p_charint_couple)(read->val))->a_len;
    
    
#define SORTED_WEIGHTED_MAPPING_POSITIONS_ITERATOR_END()		\
read=read->prox;							\
}									\
list_specific_free(sorted_reads, free_charint_couple);
    
#define GROUP_ITERATOR_DECLARATION()		\
cell_mapsembler *read;
    
#define GROUP_ITERATOR_START(reads)					\
read=reads->first;							\
while (read != NULL)							\
{									\
char *current_read=(char *)((p_charint_couple)(read->val))->a;	\
assert( current_read != NULL);					\
int position_fragment=((p_charint_couple)(read->val))->b;		\
int current_read_len=((p_charint_couple)(read->val))->a_len;
    
#define GROUP_ITERATOR_END()			\
read=read->prox;				\
}
    
    // macro to print votes
#define PRINT_VOTES() {int vx,vy,start_vx;char ACTG[4]={'A','C','T','G'};printf("votes for %s\n",fragment);start_vx=0;while (votes[start_vx][1]==0 &&votes[start_vx][0]==0&&votes[start_vx][2]==0&&votes[start_vx][3]==0) start_vx++;for (vx=start_vx;vx<MAX_VOTES;vx++){int cont=0;for (vy=0;vy<4;vy++)	if (votes[vx][vy]!=0) cont=1;if (cont==0)break;printf(" vote[%d] ",vx);for (vy=0;vy<4;vy++)printf("%c:%d ",ACTG[vy],votes[vx][vy]);printf("\n");	}}
    
    // macro to print nucleotide-based consensus (certainly not read-coherent)
#define PRINT_CONSENSUS() {int vx,vy,start_vx;char ACTG[4]={'A','C','T','G'};printf("nucleotide consensus for\n%s\n",fragment);start_vx=0;while (votes[start_vx][1]==0 &&votes[start_vx][0]==0&&votes[start_vx][2]==0&&votes[start_vx][3]==0) start_vx++;int nb_subst=0;for (vx=start_vx;vx<MAX_VOTES;vx++){int cont=0;for (vy=0;vy<4;vy++)	if (votes[vx][vy]!=0) cont=1;if (cont==0)break;int max=0, max_vy=0; for (vy=0;vy<4;vy++)if (votes[vx][vy]>max){max=votes[vx][vy]; max_vy=vy;} printf("%c",ACTG[max_vy]); if (ACTG[max_vy]!=fragment[vx])nb_subst++;}printf(" (%d substitutions)\n",nb_subst);}
    
    int debug_read_coherence;
    /*
     * function called in read_coherent_homologs to free local variables
     */
    char ** stop_and_clear_everything(list_ *read_groups, list_ *sorted_reads) {
        /*
         * free sorted_reads
         */
        list_specific_free(sorted_reads, free_charint_couple); // need to free the charint couple, but doesnt free the char inside
        
        /*
         * free read_groups
         */
        
        free_read_groups(read_groups, 1);
        
        /*
         * return NULL, as it indicates that no consensus was found
         */
        return NULL;
    }
    
    int is_empty_read(char *current_read) {
        int i = 0;
        while (current_read[i] != '\0') {
            if (current_read[i] != 'U')
                return 0;
            i++;
        }
        return 1;
    }
    
    /*
     * before attempting actual fork, make sure we're not creating a consensus that's a substring of an existing group
     */
    int check_before_forking(list_ *read_groups, group_info* current_group_info,
                             int group_fork_up_to_position, char *current_read,
                             int position_fragment, const char *fragment, int len_fragment,
                             int max_substitutions_allowed, hash_t groups_consensuses) {
        int brand_new_consensus = 1;
        // construct potential consensus
        char *potential_consensus = strdup(current_group_info->consensus);
        int current_read_len = strlen(current_read);
        int l;
        for (l = MAX(0,position_fragment); l
             <MIN(len_fragment,position_fragment+current_read_len); l++) {
            potential_consensus[l] = current_read[l - position_fragment];
        }
        
        /*int *dummy; // WIP-HASH
         if (hash_entry_by_key(groups_consensuses,potential_consensus,&dummy)!=0)
         {*/
        // check that it's not already a substring of a previous consensus
        cell_mapsembler *other_group_cell = read_groups->first;
        while (other_group_cell != NULL) {
            group_info *other_read_group_info =
            (group_info *) ((p_pointers_couple) (other_group_cell->val))->b;
            
            if (strcmp(other_read_group_info->consensus, potential_consensus)==0) {
                brand_new_consensus = 0;
                break;
            }
            other_group_cell = other_group_cell->prox;
        }
        /*} // WIP-HASH
         if ((hash_entry_by_key(groups_consensuses,potential_consensus,&dummy)!=0) && brand_new_consensus==0)
         printf("confirmed by hash\n");
         if ((hash_entry_by_key(groups_consensuses,potential_consensus,&dummy)==0) && brand_new_consensus==1)
         printf("confirmed by hash\n");
         if ((hash_entry_by_key(groups_consensuses,potential_consensus,&dummy)!=0) && brand_new_consensus==1)
         printf("missed by hash\n");
         if ((hash_entry_by_key(groups_consensuses,potential_consensus,&dummy)==0) && brand_new_consensus==0)
         printf("missed by hash\n");*/
        
        
        // count substitutions with fragment
        int len_potential_consensus = strlen(potential_consensus);
        int number_substitutions = 0;
        for (l = 0; l < len_potential_consensus; l++)
            if (fragment[l] != potential_consensus[l])
                number_substitutions++;
        free(potential_consensus);
        return brand_new_consensus && (number_substitutions
                                       <= max_substitutions_allowed);
    }
    
    int check_fragment_coverage(group_t current_read_group, group_info *current_read_group_info, int min_coverage, int up_to)
    {
        int fragment_fully_covered = 1;
        int l;
        for (l = 0; l < up_to; l++)
        {
            if (current_read_group_info->fragment_coverage[l] < min_coverage) {
#ifdef DEBUG_READ_COHERENCE
                    printf("group %X pos %d below coverage (%d<%d)\n\n",
                           ( current_read_group), l, current_read_group_info->fragment_coverage[l] ,
                           min_coverage);
#endif
                
                fragment_fully_covered = 0;
                break;
            }
        }
        return fragment_fully_covered;
    }
    
    void find_farthest_perfect_overlapping_read(char *current_read, int current_read_len, int position_fragment, char*ri, int ri_length, int ri_position_fragment, char **farthest_read,int *farthest_position, int *farthest_read_position_fragment, int*farthest_read_len, int*so_far_good_overlaps)
    {
        // we only need to test those which have a chance to perfect_overlap
        // ----------------------------------------------------- fragment
        //                            ri ---------------
        //                 				             excluded ----------------
        //   excluded ----------------
        //        good current_read  -----------------
        if ((!(position_fragment
               > ri_position_fragment
               + ri_length))
            && (!(position_fragment
                  + current_read_len
                  < ri_position_fragment))) {
            // don't test reads before an already valid far overlap
            if (position_fragment
                + current_read_len
                > *farthest_position) {
                // finally, make sure there's a perfect overlap
                if (*so_far_good_overlaps
                    && overlap_perfect(
                                       current_read,
                                       position_fragment,
                                       current_read_len,
                                       ri,
                                       ri_position_fragment)) {
                        *farthest_read
                        = current_read;
                        *farthest_read_position_fragment
                        = position_fragment;
                        *farthest_read_len
                        = current_read_len;
                        *farthest_position
                        = position_fragment
                        + current_read_len;
                    } else {
                        // should have overlapped but did not: this read cannot fork that group
                        *so_far_good_overlaps = 0;
                    }
            }
        }
    }
    
    /**
     * given a fragment and the set of reads that map on it (cf "print_fragment_cover" in "fragment is covered.c" for an overview on how the map is used"). Basically for each fragment, the map gives its relative position with respect to the fragment (may be negative).
     * Note that each read is given in the correct order: no need to check reverse complements.
     * Since errors in reads are permitted, the fragment has to be covered at a depth == error_threshold
     *
     *  We have to find a subset of reads that
     *  a) fully cover the fragment with at most max_subsitutions_allowed substitutions
     *  b) are coherent together (no error between them)
     *
     */
    
    char ** read_coherent_homologs(const hash_t map, const char * fragment,
                                   const int max_substitutions_allowed, const int min_coverage,
                                   const int error_threshold, const int size_seed, int *nb_consensuses) {
        
        char ** substarters = (char **) malloc((MAX_GROUPS + 1) * sizeof(char *));
        test_alloc(substarters);
        int nb_groups = 0;
        int len_fragment;
        int votes[MAX_VOTES][4];
        MAPPING_POSITIONS_ITERATOR_DECLARATION()
        
        assert( map != NULL);
        assert( fragment != NULL);
        
        
        
        /*
         
         * dynamic tests are made during the construction of groups to make sure that each group indeed has a chance to be read-coherent
         * they are controlled with the following switches:
         */
        int group_coverage_aware = 1; // change this to include DYNAMIC TEST that groups are fully covering the fragment
        int group_substitutions_aware = 1; // change this to include DYNAMIC TEST that groups have less than max_substitutions_allowed with the fragment
        /*
         * but anyway, keep in mind we also do a final post-processing test at consensus construction..
         */
        
        /*
         * a group is a hash table that contains reads and positions.. quite similar to the "map" parameter really
         * here is the definition of the list of groups, each element of the list is a couple (group,group_info)
         */
        list_ *read_groups = list_create();
        
        *nb_consensuses = 0;
        
        // create a votes array that spans the whole multiple alignement (reads aligned to the fragment)
        init_votes(votes);
        
        len_fragment = strlen(fragment);
        
        /*
         * populate votes for all the reads that are aligned to this fragment
         */
        int nb_reads = 0;
        init_votes(votes);
        MAPPING_POSITIONS_ITERATOR_START(map)
        {
            int offset_read, offset_fragment;
            if (position_fragment < 0) {
                offset_read = abs(position_fragment);
                offset_fragment = 0;
            } else {
                offset_read = 0;
                offset_fragment = position_fragment;
            }
            populate_votes(current_read, current_read_len,
                           offset_read, offset_fragment, votes);
            nb_reads++;
            
        }
        MAPPING_POSITIONS_ITERATOR_END(map)
        
        /*
         * give up if too many reads were mapped
         */
        if (nb_reads > MAX_MAPPED_READS) {
            fprintf(
                    stderr,
                    "%d reads mapped to this starter, i'm not wasting time computing its homologs\n",
                    nb_reads);
            return NULL;
        }
        
        /*
         * only consider votes spanning the fragment itself
         */
        trim_votes(votes, len_fragment);
        
        /*
         * warn the user if error_threshold don't match votes
         */
        analyze_votes(votes, error_threshold);
        
#ifdef  debug_votes
        PRINT_VOTES()
        PRINT_CONSENSUS()
#endif
        
        
        /*
         * error-correct all these reads
         *
         */
#ifdef DEBUG_READ_COHERENCE
                 printf("correcting reads\n");
#endif
            MAPPING_POSITIONS_ITERATOR_START(map)
            //printf("correcting read:            %s, threshold %d\n",current_read,error_threshold);
            rudimentary_error_correction(current_read, votes,
                                         position_fragment, error_threshold,
                                         current_read);
        
        MAPPING_POSITIONS_ITERATOR_END(map)
        /*
         * at this point, reads in ((map)) can be altered (cutted, corrected, even empty)
         *
         TODO: maybe take into account U's inside reads in read_coherent_holomogs such that theyre not considered as substitutions
         */
        
        // (optimization) create a hash of all consensus for faster execution of check_before_forking
        // (it's not yet in production)
        hash_t groups_consensuses = hash_create(MAX_GROUPS); // WIP-HASH
        
        /*
         * perform the multiple grouping procedure
         * here, we loop over all reads mapped to the fragment, in sorted order
         * this creates groups.
         */
#ifdef DEBUG_READ_COHERENCE
            printf("performing multiple grouping procedure\n");
#endif
        //	int last_checked_coverage_position=-len_fragment;
        SORTED_WEIGHTED_MAPPING_POSITIONS_ITERATOR_DECLARATION()
        SORTED_WEIGHTED_MAPPING_POSITIONS_ITERATOR_START(map)
        {
            //	printf("grouping with read %s\n", read);
            int read_was_added_to_one_group = 0;
            int read_forked_one_group = 0;
            assert( current_read != NULL);
            
            // discard un-informative (UUUU) reads
            if (is_empty_read(current_read)) {
                read = read->prox;
                continue;
            }
            
            /*
             * FIXME: what to do with reads smaller than size_seed (and also long ones like UUUUU..UUUA) ?
             * right now im allowing them.. except the UUUUU one
             * and even modified overlap_perfect to consider them (since useful info is below size_seed), is that good?
             */
            
#ifdef debug_groups
            {
                printf("* considering read %s (pos=%d); printing current groups:\n",current_read,position_fragment);
                cell_mapsembler *group_cell=read_groups->first;
                while (group_cell != NULL)
                {
                    group_t current_read_group=(group_t)((p_pointers_couple)(group_cell->val))->a;
                    group_info *current_read_group_info = (group_info *) ((p_pointers_couple) (group_cell->val))->b;
                    printf("group %lX:\n",(unsigned long)current_read_group);
                    print_group_cover(current_read_group, current_read_group_info, (char *)fragment,-1);
                    group_cell=group_cell->prox;
                }
            }
#endif
            /*
             * loop over groups to test for perfect overlap and fork if necessary
             */
            list_ *forked_read_groups = list_create();
            int nb_forked_groups = 0;
            cell_mapsembler *group_cell = read_groups->first;
            
            while (group_cell != NULL) {
                group_t
                current_read_group =
                (group_t) ((p_pointers_couple) (group_cell->val))->a;
                group_info
                *current_read_group_info =
                (group_info *) ((p_pointers_couple) (group_cell->val))->b;
                assert( current_read_group != NULL);
                assert( current_read_group_info != NULL);
                
                /*  1. if overlap_perfect( farthest_read_in_the_group, current_read ) then add it to the group */
                
                /*
                 * 1.a finding farthest read from the current group
                 */
                int farthest_position = -len_fragment;
                int farthest_read_position_fragment = 0;
                int farthest_read_len = 0;
                char* farthest_read = NULL;
                
                {
                    GROUP_ITERATOR_DECLARATION()
                    GROUP_ITERATOR_START(current_read_group)
                    {
                        if (position_fragment + current_read_len
                            > farthest_position) {
                            farthest_read = current_read;
                            farthest_read_position_fragment
                            = position_fragment;
                            farthest_read_len = current_read_len;
                            farthest_position = position_fragment
                            + current_read_len;
                        }
                    }
                    GROUP_ITERATOR_END()
                }
                assert( farthest_read != NULL);
                
                /*
                 * 1.b.(i) if the farthest position is before the current read position, it means the group can't be
                 * 	   extended anymore, so lets remove that group
                 */
                
                if (farthest_position < position_fragment) {
#ifdef DEBUG_READ_COHERENCE
                        printf(
                               "that group (%X) cannot be extended anymore, removing it\n",
                               current_read_group);
#endif
                    cell_mapsembler *next = group_cell->prox;
                    remove_group_from_groups(&nb_groups, read_groups,
                                             group_cell);
                    group_cell = next;
                    continue;
                }
                
                /*
                 * 1.b.(ii) before the current position_fragment, test the group coverage and if it has a position below min_coverage, lets remove that group
                 */
                
                if (!check_fragment_coverage(current_read_group, current_read_group_info,  min_coverage, position_fragment)) {
#ifdef DEBUG_READ_COHERENCE
                        printf(
                               "that group (%X) has insufficient coverage so far (positions < %d)\n",
                               current_read_group, position_fragment );
#endif
                    cell_mapsembler *next = group_cell->prox;
                    remove_group_from_groups(&nb_groups, read_groups,
                                             group_cell);
                    group_cell = next;
                    continue;
                }
                
                /*
                 * 1.c testing for perfect overlap and insert if match
                 */
                // uncomment the next line for extreme group debugging
                //printf("group id %X, testing perfect_overlap(%s,\n                        %s) = %d\n",(int)current_read_group,farthest_read,current_read,overlap_perfect(farthest_read, farthest_read_position_fragment, farthest_read_len, current_read, position_fragment));
                if (overlap_perfect(farthest_read,
                                    farthest_read_position_fragment, farthest_read_len,
                                    current_read, position_fragment)) {
                    // reads which are strictly contained in the current group are uninformative, we insert them in a separate structure (used only for coverage info)
                    if (farthest_position<position_fragment+current_read_len)
                    {
                        if (append_read_to_group(current_read_group,
                                                 current_read_group_info, current_read, current_read_len,
                                                 position_fragment, fragment, len_fragment,
                                                 current_read_weight, group_coverage_aware,
                                                 group_substitutions_aware,
                                                 max_substitutions_allowed,groups_consensuses)) {
 #ifdef DEBUG_READ_COHERENCE
                                printf(
                                       "inserted read %s (pos=%d) into group %X\n",
                                       current_read, position_fragment,
                                       current_read_group);
#endif
                        }
                    }
                    else
                    {
                        append_contained_read_to_group(current_read_group,
                                                       current_read_group_info, current_read, current_read_len,
                                                       position_fragment, fragment, len_fragment, current_read_weight,
                                                       group_coverage_aware,
                                                       group_substitutions_aware,
                                                       max_substitutions_allowed,groups_consensuses);
                    }
                    read_was_added_to_one_group = 1;
                } else {
                    /* 2. if we could not perfect_overlap this read to the group,
                     * then fork it if it includes a read r such that o_p(r, current_read)*/
                    
                    group_t
                    current_read_group =
                    (group_t) ((p_pointers_couple) (group_cell->val))->a;
                    group_info
                    *current_read_group_info =
                    (group_info *) ((p_pointers_couple) (group_cell->val))->b;
                    assert( current_read_group != NULL);
                    assert( current_read_group_info != NULL);
                    
                    int ri_position_fragment = position_fragment;
                    int ri_length = current_read_len;
                    char *ri = current_read;
                    int ri_weight = current_read_weight;
                    int group_fork_up_to_position = 0;
                    int found_a_perfect_overlap = 0;
                    {
                        int farthest_position = -len_fragment;
                        int farthest_read_position_fragment = 0;
                        int farthest_read_len = 0;
                        char* farthest_read = NULL;
                        
                        /* we need to take the farthest such r to make sure the future read will be group-coherent*/
                        int so_far_good_overlaps = 1;
                        GROUP_ITERATOR_DECLARATION()
                        GROUP_ITERATOR_START(current_read_group)
                        {
                            find_farthest_perfect_overlapping_read(current_read,current_read_len,position_fragment, ri,ri_length,ri_position_fragment,&farthest_read,&farthest_position, &farthest_read_position_fragment, &farthest_read_len, &so_far_good_overlaps);
                        }
                        GROUP_ITERATOR_END()
                        
                        /*
                         * if the non-contained reads cannot fork, try with the contained reads
                         */
                        if (farthest_read == NULL) {
                            so_far_good_overlaps = 1;
                            GROUP_ITERATOR_DECLARATION()
                            GROUP_ITERATOR_START(current_read_group_info->contained_reads)
                            {
                                find_farthest_perfect_overlapping_read(current_read,current_read_len,position_fragment, ri,ri_length,ri_position_fragment,&farthest_read,&farthest_position, &farthest_read_position_fragment, &farthest_read_len, &so_far_good_overlaps);
                            }
                            GROUP_ITERATOR_END()
                        }
                        
                        // if it found a valid fork, record some infos. check_before_forking will be called later
                        if (farthest_read != NULL) {
                            found_a_perfect_overlap = 1;
                            group_fork_up_to_position
                            = farthest_position;
                        }
                        
                    }
                    
                    if (found_a_perfect_overlap && check_before_forking(
                                                                        read_groups, current_read_group_info,
                                                                        group_fork_up_to_position, ri,
                                                                        ri_position_fragment, fragment, len_fragment,
                                                                        max_substitutions_allowed,groups_consensuses)) {
                        
                        /* 2.b fork the group! */
                        group_t new_group;
                        group_info *new_group_info;
                        
                        create_new_group(&new_group, &new_group_info,
                                         fragment);
                        
#ifdef DEBUG_READ_COHERENCE
                            printf(
                                   "forking group id %X with read %s (pos=%d) (new group id: %X) because perfect overlap found up to pos=%d\n",
                                   current_read_group, current_read,
                                   position_fragment, new_group,
                                   group_fork_up_to_position);
#endif
                        
                        
                        /*
                         * cloning the group with every read that aligns before position group_fork_up_to_position
                         */
                        GROUP_ITERATOR_DECLARATION()
                        GROUP_ITERATOR_START(current_read_group)
                        {
                            if (position_fragment+current_read_len <= group_fork_up_to_position)
                                append_read_to_group(new_group,
                                                     new_group_info,
                                                     current_read, current_read_len,
                                                     position_fragment,
                                                     fragment, len_fragment, 1,
                                                     group_coverage_aware,
                                                     group_substitutions_aware,
                                                     max_substitutions_allowed,groups_consensuses);
                            
                        }
                        GROUP_ITERATOR_END()
                        
                        // add contained reads too
                        GROUP_ITERATOR_START(current_read_group_info->contained_reads)
                        {
                            if (position_fragment+current_read_len <= group_fork_up_to_position)
                                append_contained_read_to_group(new_group,
                                                               new_group_info,
                                                               current_read, current_read_len,
                                                               position_fragment,
                                                               fragment, len_fragment, 1,
                                                               group_coverage_aware,
                                                               group_substitutions_aware,
                                                               max_substitutions_allowed,groups_consensuses);
                        }
                        GROUP_ITERATOR_END()
                        
                        /*
                         * append the current read to the newly forked group
                         */
                        if (!append_read_to_group(new_group,
                                                  new_group_info, ri, ri_length,
                                                  ri_position_fragment, fragment,
                                                  len_fragment, ri_weight, group_coverage_aware,
                                                  group_substitutions_aware,
                                                  max_substitutions_allowed,groups_consensuses)) {
                            /*
                             * if it turns out that adding the read would make the group not read-coherent, remove it.. all that for nothing
                             */
                            free_group(new_group, new_group_info);
                            
                        } else {
                            /*
                             * else add the group to the list of newly forked read groups, as well as the list of all groups
                             */
                            read_forked_one_group = 1;
                            if (append_group_to_groups(&nb_forked_groups,
                                                       forked_read_groups, new_group,
                                                       new_group_info))
                                return stop_and_clear_everything(
                                                                 read_groups, sorted_reads);
                        }
                    }
                }
                group_cell = group_cell->prox;
            }
            // end of group loop for overlapping/forking
            
            /*
             * remove newly forked groups that are strictly included in others
             * add the others to the groups list
             */
            group_cell = forked_read_groups->first;
            list_ *removed_forked_groups = list_create();
            int nb_removed_forked_groups = 0;
            while (group_cell != NULL) {
                
                group_t
                current_read_group =
                (group_t) ((p_pointers_couple) (group_cell->val))->a;
                group_info
                *current_read_group_info =
                (group_info *) ((p_pointers_couple) (group_cell->val))->b;
                
                int is_not_included = 1;
                int brand_new_consensus = 1;
                
                // now compare that group with of newly forked groups
                cell_mapsembler *other_group_cell = forked_read_groups->first;
                while (other_group_cell != NULL) {
                    if (other_group_cell != group_cell) {
                        group_t
                        other_read_group =
                        (group_t) ((p_pointers_couple) (other_group_cell->val))->a;
                        group_info
                        *other_read_group_info =
                        (group_info *) ((p_pointers_couple) (other_group_cell->val))->b;
                        
                        // first filter: consensus should be a substring of other consensus
                        if (strstr(other_read_group_info->consensus,
                                   current_read_group_info->consensus)) {
                            
                            // don't compare to already erased groups
                            if (find_group(removed_forked_groups,
                                           other_read_group) == 0) {
                                
                                // actual comparison: group list of reads is included in other group list of reads
                                if (list_included(current_read_group,
                                                  other_read_group,
                                                  charint_couple_exact_compare)) {
                                    
#ifdef DEBUG_READ_COHERENCE
                                        printf(
                                               "removing newly forked group %X (%d reads) because it's strictly included in one of the newly forked groups %X (%d reads)\n",
                                               current_read_group,
                                               current_read_group_info->nb_reads,
                                               other_read_group,
                                               other_read_group_info->nb_reads);
#endif
                                    is_not_included = 0;
                                    
                                    break;
                                }
                                
                            }
                        }
                    }
                    other_group_cell = other_group_cell->prox;
                }
                
                if (is_not_included) {
                    // compare that group with of previous groups
                    other_group_cell = read_groups->first;
                    while (other_group_cell != NULL) {
                        if (other_group_cell != group_cell) {
                            group_t
                            other_read_group =
                            (group_t) ((p_pointers_couple) (other_group_cell->val))->a;
                            group_info
                            *other_read_group_info =
                            (group_info *) ((p_pointers_couple) (other_group_cell->val))->b;
                            
                            if (strstr(other_read_group_info->consensus,
                                       current_read_group_info->consensus)) {
                                brand_new_consensus = 0;
                                
                                if (list_included(current_read_group,
                                                  other_read_group,
                                                  charint_couple_exact_compare)) {
                                    
#ifdef DEBUG_READ_COHERENCE
                                        printf(
                                               "removing newly forked group %X (%d reads) because it's strictly included in one of the previous groups %X (%d reads) %s\n",
                                               current_read_group,
                                               current_read_group_info->nb_reads,
                                               other_read_group,
                                               other_read_group_info->nb_reads,
                                               brand_new_consensus ? "was new consensus"
                                               : "");
#endif
                                    is_not_included = 0;
                                    break;
                                }
                            }
                            
                        }
                        other_group_cell = other_group_cell->prox;
                    }
                }
                
                if (is_not_included) {
                    // we verified that the newly forked group not included in others, let's add it to the main groups list
#ifdef DEBUG_READ_COHERENCE
                        printf("valid fork (group %lX, %d reads) %s\n",
                               current_read_group,
                               current_read_group_info->nb_reads,
                               brand_new_consensus ? "brand new consensus"
                               : "");
#endif
                    if (append_group_to_groups(&nb_groups, read_groups,
                                               current_read_group, current_read_group_info))
                        return stop_and_clear_everything(read_groups,
                                                         sorted_reads);
                } else {
                    append_group_to_groups(&nb_removed_forked_groups,
                                           removed_forked_groups, current_read_group,
                                           current_read_group_info);
                }
                group_cell = group_cell->prox;
            }
            free_read_groups(removed_forked_groups, 1);
            free_read_groups(forked_read_groups, 0);
            
            /*
             * if the read was not added to a group, nor forked any group
             * create a new group with the read (also initializes the very first group)
             */
            if ((!read_was_added_to_one_group) && (!read_forked_one_group)) {
                if (group_coverage_aware && position_fragment > 0) {
#ifdef DEBUG_READ_COHERENCE
                        printf(
                               "this fragment %s is certainly not read-coherent as the first reads aligns to it at %d\n",
                               current_read, position_fragment);
#endif
                    if (nb_groups == 0) /* if it is the first group and the read is already not covering it,
                                         * then nothing else can be done in this function */
                    {
#ifdef DEBUG_READ_COHERENCE
                            printf(
                                   "Will not create any new fragment. We give up!\n");
#endif
                        return stop_and_clear_everything(read_groups,
                                                         sorted_reads);
                    }
                } else {
                    group_t new_group;
                    group_info *new_group_info;
                    create_new_group(&new_group, &new_group_info, fragment);
#ifdef DEBUG_READ_COHERENCE
                    printf(
                               "creating new group %X (group number %d) for read %s (pos=%d) because it could not contribute to other groups\n",
                               new_group, nb_groups + 1,
                               current_read, position_fragment);
#endif
                    if (!append_read_to_group(new_group, new_group_info,
                                              current_read, 0, position_fragment, fragment,
                                              len_fragment, current_read_weight, group_coverage_aware,
                                              group_substitutions_aware,
                                              max_substitutions_allowed,groups_consensuses)) {
                        /* can't create a new group with this read, oh well */
                        free_group(new_group, new_group_info);
                    } else {
                        if (append_group_to_groups(&nb_groups, read_groups,
                                                   new_group, new_group_info))
                        {
#ifdef DEBUG_READ_COHERENCE
                                printf("can't append group %x to groups\n",(unsigned long)new_group);
#endif
                            return stop_and_clear_everything(read_groups,
                                                             sorted_reads);
                        }
                    }
                }
            }
        } // end of the big read loop
        SORTED_WEIGHTED_MAPPING_POSITIONS_ITERATOR_END()
        
        hash_free(groups_consensuses);
        
#ifdef DEBUG_READ_COHERENCE
            printf(
                   "%d %s %s groups constructed from the reads for the fragment %s.\n",
                   nb_groups, group_coverage_aware ? "fragment-coverage coherent"
                   : "",
                   group_substitutions_aware ? "fragment-substitutions coherent"
                   : "", fragment);
#ifdef debug_groups
            cell_mapsembler *group_cell = read_groups->first;
            while (group_cell != NULL) {
                group_t current_read_group =
                (group_t) ((p_pointers_couple) (group_cell->val))->a;
                group_info *current_read_group_info =
                (group_info *) ((p_pointers_couple) (group_cell->val))->b;
                printf("group %X:\n",current_read_group);
                print_group_cover(current_read_group, current_read_group_info, (char *)fragment, -1);
                group_cell = group_cell->prox;
            }
#endif
            printf("\n");
#endif
        
        /*
         * construct consensuses from groups, removing those which lack coverage or have too many substitutions
         */
        cell_mapsembler *group_cell = read_groups->first;
        while (group_cell != NULL) {
            group_t current_read_group =
            (group_t) ((p_pointers_couple) (group_cell->val))->a;
            group_info *current_read_group_info =
            (group_info *) ((p_pointers_couple) (group_cell->val))->b;
            int last_written_position = 0;
            int l;
            int number_substitutions = 0;
            
            substarters[*nb_consensuses] = strdup(fragment);
            
#ifdef DEBUG_READ_COHERENCE
                printf(
                       "construct consensus from group %X\n (fragment=%s) (dynamic consensus=%s) with reads:\n",
                       current_read_group, fragment, current_read_group_info->consensus);
#endif
            /*
             * create the new consensus  also compute the number of substitutions
             * for coverage, we can only trust what we computed dynamically (because contained reads and reads multiplicities were discarded)
             */
            GROUP_ITERATOR_DECLARATION()
            GROUP_ITERATOR_START(current_read_group)
            {
#ifdef DEBUG_READ_COHERENCE
                    printf("\t %s pos %d\n",(char *)((charint_couple*)(read->val))->a, ((charint_couple*)(read->val))->b);
#endif
                for (l = MAX(last_written_position,position_fragment); l < MIN(position_fragment+current_read_len,len_fragment); l++) {
                    substarters[*nb_consensuses][l] = current_read[l
                                                                   - position_fragment];
                    last_written_position
                    =MAX(last_written_position+1,l+1);
                    if (fragment[l] != substarters[*nb_consensuses][l]) {
#ifdef DEBUG_READ_COHERENCE
                            printf(
                                   "subst %c %c between %s and %s position %d\n",
                                   fragment[l],
                                   substarters[*nb_consensuses][l],
                                   fragment, substarters[*nb_consensuses],
                                   l);
#endif
                        number_substitutions++;
                    }
                }
            }
            GROUP_ITERATOR_END()
            
            /*
             * redundant check for read coherence.. very helpful!
             */
            if (number_substitutions
                != current_read_group_info->number_substitutions) {
                fprintf(
                        stderr,
                        "error group %X: recomputed number of substitutions (%d) not matching dynamic number of substitutions (%d) (so i'm glad i coded this test)\n fragment: %s\nconsensus:%s\n",
                        current_read_group, number_substitutions,
                        current_read_group_info->number_substitutions, fragment,
                        substarters[*nb_consensuses]);
                exit(1);
            }
            
            /*
             * redundant check for consensus..
             */
            if (strcmp(substarters[*nb_consensuses],
                       current_read_group_info->consensus) != 0) {
                fprintf(
                        stderr,
                        "error group %X: recomputed consensus (%s) not matching dynamic consensus (%s) (so i'm glad i coded this test)\n fragment: %s\n",
                        current_read_group, substarters[*nb_consensuses],
                        current_read_group_info->consensus, fragment);
                exit(1);
            }
            
            /*
             * test if the fragment is fully covered by reads with per-position depth > min_coverage
             */
            int fragment_fully_covered = check_fragment_coverage(current_read_group, current_read_group_info,  min_coverage, len_fragment);
            
            
            /*
             * the new consensus is validated if the fragment is fully covered and with less than max_substitutions_allowed
             */
            if ((!fragment_fully_covered) || (number_substitutions
                                              > max_substitutions_allowed)) {
#ifdef DEBUG_READ_COHERENCE
                    printf(
                           "group %X was not consensus'ed because %s, nb_substitutions: %d/%d \nfragment: %s\nconsensus:%s\n",
                           current_read_group,
                           (!fragment_fully_covered) ? "not fully covered" : "",
                           number_substitutions, max_substitutions_allowed,
                           fragment, substarters[*nb_consensuses]);
#endif
                free(substarters[*nb_consensuses]);
            } else {
                if (*nb_consensuses < MAX_GROUPS_TO_RETURN)
                    (*nb_consensuses)++;
                else {
                    fprintf(
                            stderr,
                            " Warning, too many sub-groups created for fragment %s, we keep only %d of them\n",
                            fragment, *nb_consensuses);
                    break;
                }
            }
            
            group_cell = group_cell->prox;
        }
        
        /*
         * free the read groups
         */
        
        free_read_groups(read_groups, 1);
        
#ifdef DEBUG_READ_COHERENCE
            printf(
                   "%d consensuses constructed from read-coherent groups related to the fragment %s.\n",
                   *nb_consensuses, fragment);
            int ci;
            for (ci = 0; ci < *nb_consensuses; ci++) {
                printf("consensus %d: %s\n", ci, substarters[ci]);
            }
            printf("\n");
#endif
        
        // reallocating substarters because they were MAX_GROUPS+1 of them before
        substarters = (char **)realloc(substarters, (*nb_consensuses+1) * sizeof(char *));
        test_alloc(substarters);
        return substarters;
    }
    
#ifdef __cplusplus
}
#endif
