#!/usr/bin/env python

""" MultiQC submodule to parse output from Picard ValidateSamFile """

import logging
from collections import OrderedDict

from multiqc.plots import table

# Initialise the logger
log = logging.getLogger(__name__)

# Possible warnings and descriptions
WARNING_DESCRIPTIONS = {
    'ADJACENT_INDEL_IN_CIGAR': 'CIGAR string contains an insertion (I) followed by deletion (D), or vice versa',
    'BAM_FILE_MISSING_TERMINATOR_BLOCK': 'BAM appears to be healthy, but is an older file so doesn\'t have terminator block',
    'E2_BASE_EQUALS_PRIMARY_BASE': 'Secondary base calls should not be the same as primary, unless one or the other is N',
    'INVALID_DATE_STRING': 'Date string is not ISO-8601',
    'INVALID_QUALITY_FORMAT': 'Quality encodings out of range; appear to be Solexa or Illumina when Phred expected. Avoid exception being thrown as a result of no qualities being read.',
    'MISSING_TAG_NM': 'The NM tag (nucleotide differences) is missing',
    'PAIRED_READ_NOT_MARKED_AS_FIRST_OR_SECOND': 'Pair flag set but not marked as first or second of pair',
    'RECORD_MISSING_READ_GROUP': 'A SAMRecord is found with no read group id',
}

# All possible errors and descriptions
ERROR_DESCRIPTIONS = {
    'CIGAR_MAPS_OFF_REFERENCE': 'Bases corresponding to M operator in CIGAR extend beyond reference',
    'DUPLICATE_PROGRAM_GROUP_ID': 'Same program group id appears more than once',
    'DUPLICATE_READ_GROUP_ID': 'Same read group id appears more than once',
    'EMPTY_READ': 'Indicates that a read corresponding to the first strand has a length of zero and/or lacks flow signal intensities (FZ)',
    'HEADER_RECORD_MISSING_REQUIRED_TAG': 'Header tag missing in header line',
    'HEADER_TAG_MULTIPLY_DEFINED': 'Header tag appears more than once in header line with different value',
    'INVALID_ALIGNMENT_START': 'Alignment start position is incorrect',
    'INVALID_CIGAR': 'CIGAR string error for either read or mate',
    'INVALID_FLAG_FIRST_OF_PAIR': 'First of pair flag set for unpaired read',
    'INVALID_FLAG_MATE_NEG_STRAND': 'Mate negative strand flag set for unpaired read',
    'INVALID_FLAG_MATE_UNMAPPED': 'Mate unmapped flag is incorrectly set',
    'INVALID_FLAG_NOT_PRIM_ALIGNMENT': 'Not primary alignment flag set for unmapped read',
    'INVALID_FLAG_PROPER_PAIR': 'Proper pair flag set for unpaired read',
    'INVALID_FLAG_READ_UNMAPPED': 'Mapped read flat not set for mapped read',
    'INVALID_FLAG_SECOND_OF_PAIR': 'Second of pair flag set for unpaired read',
    'INVALID_FLAG_SUPPLEMENTARY_ALIGNMENT': 'Supplementary alignment flag set for unmapped read',
    'INVALID_INDEX_FILE_POINTER': 'Invalid virtualFilePointer in index',
    'INVALID_INDEXING_BIN': 'Indexing bin set on SAMRecord does not agree with computed value',
    'INVALID_INSERT_SIZE': 'Inferred insert size is out of range',
    'INVALID_MAPPING_QUALITY': 'Mapping quality set for unmapped read or is >= 256',
    'INVALID_MATE_REF_INDEX': 'Mate reference index (MRNM) set for unpaired read',
    'INVALID_PLATFORM_VALUE': 'The read group has an invalid value set for its PL field',
    'INVALID_PREDICTED_MEDIAN_INSERT_SIZE': 'PI tag value is not numeric',
    'INVALID_REFERENCE_INDEX': 'Reference index not found in sequence dictionary',
    'INVALID_TAG_NM': 'The NM tag (nucleotide differences) is incorrect',
    'INVALID_VERSION_NUMBER': 'Does not match any of the acceptable versions',
    'MATE_CIGAR_STRING_INVALID_PRESENCE': 'A cigar string for a read whose mate is NOT mapped',
    'MATE_FIELD_MISMATCH': 'Read alignment fields do not match its mate',
    'MATE_NOT_FOUND': 'Read is marked as paired, but its pair was not found',
    'MATES_ARE_SAME_END': 'Both mates of a pair are marked either as first or second mates',
    'MISMATCH_FLAG_MATE_NEG_STRAND': 'Mate negative strand flag does not match read strand flag',
    'MISMATCH_FLAG_MATE_UNMAPPED': 'Mate unmapped flag does not match read unmapped flag of mate',
    'MISMATCH_MATE_ALIGNMENT_START': 'Mate alignment does not match alignment start of mate',
    'MISMATCH_MATE_CIGAR_STRING': 'The mate cigar tag does not match its mate\'s cigar string',
    'MISMATCH_MATE_REF_INDEX': 'Mate reference index (MRNM) does not match reference index of mate',
    'MISMATCH_READ_LENGTH_AND_E2_LENGTH': 'Lengths of secondary base calls tag values and read should match',
    'MISMATCH_READ_LENGTH_AND_QUALS_LENGTH': 'Length of sequence string and length of base quality string do not match',
    'MISMATCH_READ_LENGTH_AND_U2_LENGTH': 'Secondary base quals tag values should match read length',
    'MISSING_HEADER': 'The SAM/BAM file is missing the header',
    'MISSING_PLATFORM_VALUE': 'The read group is missing its PL (platform unit) field',
    'MISSING_READ_GROUP': 'The header is missing read group information',
    'MISSING_SEQUENCE_DICTIONARY': 'There is no sequence dictionary in the header',
    'MISSING_VERSION_NUMBER': 'Header has no version number',
    'POORLY_FORMATTED_HEADER_TAG': 'Header tag does not have colon',
    'READ_GROUP_NOT_FOUND': 'A read group ID on a SAMRecord is not found in the header',
    'RECORD_OUT_OF_ORDER': 'The record is out of order',
    'TAG_VALUE_TOO_LARGE': 'Unsigned integer tag value is deprecated in BAM. Template length',
    'TRUNCATED_FILE': 'BAM file does not have terminator block',
    'UNRECOGNIZED_HEADER_TYPE': 'Header record is not one of the standard types',
}


def _default_data_entry():
    return {'WARNING_count': 0, 'ERROR_count': 0, 'file_validation_status': 'pass'}


def parse_reports(self):
    """
        Find Picard ValidateSamFile reports and parse their data based on wether we
        think it's a VERBOSE or SUMMARY report
    """

    # Get data
    data = _parse_reports_by_type(self)

    if data:
        #  Filter to strip out ignored sample names (REQUIRED)
        data = self.ignore_samples(data)

        # Populate the general stats table
        _add_data_to_general_stats(self, data)

        # Add any found data to the report
        _add_section_to_report(self, data)

        # Write parsed data to a file
        self.write_data_file(data, 'multiqc_picard_validatesamfile')

    self.picard_ValidateSamFile_data = data  # Seems like the right thing to do
    return len(data)


def _parse_reports_by_type(self):
    """ Returns a data dictionary

        Goes through logs and parses them based on 'No errors found', VERBOSE or SUMMARY type.
    """

    data = dict()

    for file_meta in self.find_log_files('picard/sam_file_validation', filehandles=True):
        sample = file_meta['s_name']

        if sample in data:
            log.debug("Duplicate sample name found! Overwriting: {}".format(sample))

        filehandle = file_meta['f']
        first_line = filehandle.readline().rstrip()
        filehandle.seek(0)  # Rewind reading of the file

        if 'No errors found' in first_line:
            sample_data = _parse_no_error_report()
        elif first_line.startswith('ERROR') or first_line.startswith('WARNING'):
            sample_data = _parse_verbose_report(filehandle)
        else:
            sample_data = _parse_summary_report(filehandle)

        data[sample] = sample_data

    return data


def _parse_no_error_report():
    return _default_data_entry()


def _parse_verbose_report(filehandle):
    sample_data = _default_data_entry()
    errors = warnings = 0
    for line in filehandle:
        if line.startswith('WARNING:'):
            warnings += 1
        elif line.startswith('ERROR:'):
            errors += 1

    sample_data['WARNING_count'] = warnings
    sample_data['ERROR_count'] = errors

    if errors:
        sample_data['file_validation_status'] = 'fail'
    elif warnings:
        sample_data['file_validation_status'] = 'warn'

    return sample_data


def _parse_summary_report(filehandle):
    sample_data = _default_data_entry()
    for problem_type, name, count in _histogram_data(filehandle):
        sample_data[name] = count
        sample_data[problem_type+'_count'] += count

    if sample_data['ERROR_count']:
        sample_data['file_validation_status'] = 'fail'
    elif sample_data['WARNING_count']:
        sample_data['file_validation_status'] = 'warn'

    return sample_data


def _histogram_data(iterator):
    """ Yields only the row contents that contain the histogram entries """
    histogram_started = False
    header_passed = False
    for l in iterator:
        if '## HISTOGRAM' in l:
            histogram_started = True
        elif histogram_started:
            if header_passed:
                values = l.rstrip().split("\t")
                problem_type, name = values[0].split(':')
                yield problem_type, name, int(values[1])
            elif l.startswith('Error Type'):
                header_passed = True


def _add_section_to_report(self, data):
    """
        Adds found data to the report via several HTML generators
    """

    # Count samples that have errors and/or warnings
    pass_count = error_count = only_warning_count = 0
    for sample_data in data.values():
        if sample_data['file_validation_status'] == 'fail':
            error_count += 1
        elif sample_data['file_validation_status'] == 'warn':
            only_warning_count += 1
        else:
            pass_count += 1

    # Add overview note
    plot_html = []
    note_html = _generate_overview_note(
                        pass_count=pass_count,
                        only_warning_count=only_warning_count,
                        error_count=error_count,
                        total_count=len(data)
                    )
    plot_html.append(note_html)

    # Add the detailed table, but only if we have something to show
    if error_count or only_warning_count:
        table_html = _generate_detailed_table(data)
        plot_html.append(table_html)

    # Finally, add the html to the report as a section
    self.add_section(
            name='SAM/BAM File Validation',
            anchor='picard_validatesamfile',
            description=('This tool reports on the validity of a SAM or BAM '
                         'file relative to the SAM-format specification.'),
            helptext='''
            A detailed table is only shown if errors or warnings are found. Details about the errors and warnings are only shown if a `SUMMARY` report was parsed.

            For more information on the warnings, errors and possible fixes please read [this broadinstitute article](https://software.broadinstitute.org/gatk/documentation/article.php?id=7571).''',
            plot="\n".join(plot_html),
    )


def _add_data_to_general_stats(self, data):
    """
        Add data for the general stats in a Picard-module specific manner
    """
    headers = _get_general_stats_headers()

    self.general_stats_headers.update(headers)

    header_names = ('ERROR_count', 'WARNING_count', 'file_validation_status')

    general_data = dict()
    for sample in data:
        general_data[sample] = {column: data[sample][column] for column in header_names}
        if sample not in self.general_stats_data:
            self.general_stats_data[sample] = dict()
        if data[sample]['file_validation_status'] != 'pass':
            headers['file_validation_status']['hidden'] = False
        self.general_stats_data[sample].update(general_data[sample])


def _get_general_stats_headers():
    """
        Returns a header dict for the general stats collumns
    """
    headers = OrderedDict()

    headers['file_validation_status'] = {
        'title': "Validation",
        'description': 'ValidateSamFile (File Validation)'
    }

    headers['WARNING_count'] = {
        'title': "# Warnings",
        'description': 'ValidateSamFile (number of warnings)',
        'scale': 'Oranges',
        'shared_key': 'ValidateSamEntries',
        'colour': '255,237,160',
        'format': '{:.0f}',
        'hidden': True,
    }

    headers['ERROR_count'] = {
        'title': "# Errors",
        'description': 'ValidateSamFile (number of errors)',
        'scale': 'Reds',
        'shared_key': 'ValidateSamEntries',
        'colour': '252,146,114',
        'format': '{:.0f}',
        'hidden': True,
    }

    return headers


def _generate_overview_note(pass_count, only_warning_count, error_count, total_count):
    """ Generates and returns the HTML note that provides a summary of validation status. """

    note_html = ['<div class="progress">']
    pbars = [
        [ float(error_count), 'danger', 'had errors' ],
        [ float(only_warning_count), 'warning', 'had warnings' ],
        [ float(pass_count), 'success', 'passed' ]
    ]
    for b in pbars:
        if b[0]:
            note_html.append(
                '<div class="progress-bar progress-bar-{pbcol}" style="width: {pct}%" data-toggle="tooltip" title="{count} {sample} {txt}">{count}</div>'. \
                format(
                    pbcol = b[1],
                    count = int(b[0]),
                    pct = (b[0]/float(total_count))*100.0,
                    txt = b[2],
                    sample = 'samples' if b[0] > 1 else 'sample'
                )
            )
    note_html.append('</div>')

    return "\n".join(note_html)


def _generate_detailed_table(data):
    """
        Generates and retuns the HTML table that overviews the details found.
    """
    headers = _get_general_stats_headers()

    # Only add headers for errors/warnings we have found
    for problems in data.values():
        for problem in problems:
            if problem not in headers and problem in WARNING_DESCRIPTIONS:
                headers['WARNING_count']['hidden'] = False
                headers[problem] = {
                    'description': WARNING_DESCRIPTIONS[problem],
                    'namespace': 'WARNING',
                    'scale': headers['WARNING_count']['scale'],
                    'format': '{:.0f}',
                    'shared_key': 'warnings',
                    'hidden': True,  # Hide by default; to unclutter things.
                }
            if problem not in headers and problem in ERROR_DESCRIPTIONS:
                headers['ERROR_count']['hidden'] = False
                headers[problem] = {
                    'description': ERROR_DESCRIPTIONS[problem],
                    'namespace': 'ERROR',
                    'scale': headers['ERROR_count']['scale'],
                    'format': '{:.0f}',
                    'shared_key': 'errors',
                    'hidden': True,  # Hide by default; to unclutter things.
                }

    table_config = {
        'table_title': 'Picard: SAM/BAM File Validation',
    }

    return table.plot(data=data, headers=headers, pconfig=table_config)
