/***************************** LICENSE START ***********************************

 Copyright 2012 ECMWF and INPE. This software is distributed under the terms
 of the Apache License version 2.0. In applying this license, ECMWF does not
 waive the privileges and immunities granted to it by virtue of its status as
 an Intergovernmental Organization or submit itself to any jurisdiction.

 ***************************** LICENSE END *************************************/

#include <stdio.h>

#include <iostream>
#include <sstream>
#include <fstream>
#include <cstdio>

#include "BufrFilterDef.h"
#include "BufrLocationCollector.h"
#include "BufrMetaData.h"
#include "LogHandler.h"
#include "MvKeyProfile.h"
#include "MvMiscelaneous.h"
#include "MvEccBufr.h"
#include "MvTmpFile.h"

#include "eccodes.h"

#define MV_CODES_CHECK(a,msg) MvEccBufr::codesCheck(#a,__FILE__,__LINE__,a,msg)

static std::string BUFR_DEFINITION_PATH_ENV;
static std::string BUFR_DUMP_EXE;
static std::string BUFR_FILTER_EXE;
static std::string BUFR_COPY_EXE;

#if 0
static bool shellCommand(const std::string &command, std::stringstream& out,std::stringstream& err,
                         std::stringstream& otherErr, const std::string &ftmp,int& exitCode)
{
    FILE *in;
    char cbuf[1024];

    exitCode=-1;

    std::string cmd;
    cmd+=command + " 2>" + ftmp;

    if (!(in = popen(cmd.c_str() ,"r")) )
    {
        otherErr << "Failed to execute command: " << command;
        return false;
    }

    while(fgets(cbuf, sizeof(cbuf), in) != NULL)
    {
        out << cbuf;
    }

    int ret=pclose(in);
    exitCode=WEXITSTATUS(ret);

    if (!(in = fopen(ftmp.c_str() ,"r")) )
    {
        otherErr << "Failed to read file " << ftmp << " containing STDERR of command";
        return false;
    }

    while(fgets(cbuf, sizeof(cbuf), in) != NULL)
    {
        err << cbuf;
    }

    fclose(in);

    return true;
}
#endif

static bool shellCommand(const std::string &command, std::stringstream& out,std::stringstream& err,
                         std::stringstream& otherErr, int& exitCode)
{
    FILE *in;
    char cbuf[1024];
    MvTmpFile tmpFile;

    exitCode=-1;

    std::string cmd;
    cmd+=command + " 2>" + tmpFile.path();

    if (!(in = popen(cmd.c_str() ,"r")) )
    {
        otherErr << "Failed to execute command: " << command;
        return false;
    }

    while(fgets(cbuf, sizeof(cbuf), in) != NULL)
    {
        out << cbuf;
    }

    int ret=pclose(in);
    exitCode=WEXITSTATUS(ret);

    if (!(in = fopen(tmpFile.path().c_str() ,"r")) )
    {
        otherErr << "Failed to read file " << tmpFile.path() << " containing STDERR of command";
        return false;
    }

    while(fgets(cbuf, sizeof(cbuf), in) != NULL)
    {
        err << cbuf;
    }

    fclose(in);

    return true;
}

//=====================================
//
// BufrDataDump
//
//=====================================

BufrDataDump::~BufrDataDump()
{
    clear();
}

void BufrDataDump::clear()
{
   text_.clear();
   debug_.clear();
}

bool BufrDataDump::filterMessage(const std::string& inFile, int msgCnt,const std::string& outFile, std::string& errOut)
{
    std::string cmd=BUFR_COPY_EXE + " -w count=" + metview::toString<int>(msgCnt) + " " + inFile + " " + outFile;

    GuiLog().task() << "Filtering out message: "  << msgCnt <<
                   GuiLog::commandKey() << cmd;

    //Run the command
    std::stringstream in, err;
    bool hasError=false;
    int exitCode;
    std::stringstream shellErr;
    bool ret=shellCommand(cmd,in,err,shellErr,exitCode);

    if(exitCode > 0)
    {
        hasError=true;
        GuiLog().error() << "Command exited with code: "  << exitCode;
        if(err.str().empty())
            errOut+="<b>Command</b>" + cmd + " exited with <b>code:</b> " + metview::toString<int>(exitCode) + " ";
    }

    if(!ret)
    {
        hasError=true;
        GuiLog().error() << shellErr.str();
        errOut+=shellErr.str();
    }

    //Get stderr of command
    if(!err.str().empty())
    {
        hasError=true;
        GuiLog().error() << err.str();
        errOut="<b>Command </b>" + cmd + " <b>failed.</b> <br>" + err.str();
    }

    return !hasError;
}

bool BufrDataDump::filterSubset(const std::string& inFile, int msgCnt,int subsetCnt,const std::string& outFile, std::string& errOut)
{   
    MvTmpFile rulesFile;

    std::string cmd=BUFR_FILTER_EXE + " -o " + outFile + " " + rulesFile.path() + " " + inFile;

    GuiLog().task() << "Filtering out subset: " << subsetCnt << " from message: " << msgCnt <<
                   GuiLog::commandKey() << cmd;

    //Create the filter
    std::ofstream out;
    out.open(rulesFile.path().c_str());
    if(!out.good())
    {
        GuiLog().error() << "Cannot create temporary rules file: " << rulesFile.path();
        errOut="Cannot create temporary rules file: " + rulesFile.path();
        return false;
    }

    out << "set unpack=1;" << std::endl;
    out << "set extractSubset=" << subsetCnt << ";" << std::endl;
    out << "set doExtractSubsets=1;" << std::endl;
    out << "if(count == " << msgCnt << " ) { write; } " << std::endl;
    out.close();

    //Run the filter
    std::stringstream in, err;
    bool hasError=false;
    int exitCode;
    std::stringstream shellErr;
    bool ret=shellCommand(cmd,in,err,shellErr,exitCode);

    if(exitCode > 0)
    {
        hasError=true;
        GuiLog().error() << "Command exited with code: "  << exitCode;
        if(err.str().empty())
            errOut+="<b>Command</b>" + cmd + " exited with <b>code:</b> " + metview::toString<int>(exitCode) + " ";
    }

    if(!ret)
    {
        hasError=true;
        GuiLog().error() << shellErr.str();
        errOut+=shellErr.str();
    }

    //Get stderr of command
    if(!err.str().empty())
    {
        hasError=true;
        GuiLog().error() << err.str();
        errOut="<b>Command </b>" + cmd + " <b>failed.</b> <br>" + err.str();
    }

    return !hasError;
}

//msgCnt starts from 1!!!
bool BufrDataDump::read(const std::string& sourceFile, int msgCnt,int subsetCnt,int subsetNum, bool compressed,std::string& errOut)
{
    std::stringstream in, err, cnt_s;
    std::string inFile;
    MvTmpFile subsetFile;

    //For non-compressed messages we need to filter out the required subset
    if(!compressed && subsetNum > 1)
    {
        MvTmpFile msgFile;

        if(!filterMessage(sourceFile,msgCnt,msgFile.path(),errOut))
            return false;
        if(!filterSubset(msgFile.path(),1,subsetCnt,subsetFile.path(),errOut))
            return false;

        inFile=subsetFile.path(); //safe to do

        cnt_s << 1;
    }
    else
    {
        inFile=sourceFile;
        cnt_s << msgCnt;
    }

    std::string cmd=BUFR_DUMP_EXE + " -ja  -w count=" + cnt_s.str() + " " + "\"" + inFile + "\"" ; //+ "  > " + tmpFile_;
    std::stringstream msg;
    if(!compressed && subsetNum > 1)
    {
        msg << "Generating json dump for message: " << msgCnt << " subset: " << subsetCnt;
    }
    else
    {
        msg << "Generating json dump for message: " << msgCnt;
    }

    GuiLog().task() << msg.str() << GuiLog::commandKey() << cmd;


    bool hasError=false;
    int exitCode;
    std::stringstream shellErr;
    bool ret=shellCommand(cmd,in,err,shellErr,exitCode);

    //Get stdout of command
    text_=in.str();

    //For uncompressed subsets we need to set the correct numberOfSubsets
    //in the json file. At the moment it contains 1, because to access the subset
    //first we filter it out as a separate message and then run the bufr dump on it to
    //generate json. Since we run the dump on a single subset the numberOfSubsets
    //in the resulting dump will always be 1.
    if(!compressed && subsetNum > 1)
    {
        size_t pos=text_.find("numberOfSubsets");
        if(pos != std::string::npos)
        {
            pos=text_.find("value\" : 1",pos);
            if(pos != std::string::npos)
            {
                std::stringstream nosTxt;
                nosTxt << "value\" : "  << subsetNum;
                text_.replace(pos,10,nosTxt.str());
            }
        }
    }

    if(exitCode > 0)
    {
        hasError=true;
        GuiLog().error() << "Command exited with code: "  << exitCode;
        if(err.str().empty())
            errOut+="<b>Command</b>" + cmd + " exited with <b>code:</b> " + metview::toString<int>(exitCode) + " ";
    }

    if(!ret)
    {
        hasError=true;
        GuiLog().error() << shellErr.str();
        errOut+=shellErr.str();
    }

    //Get stderr of command
    if(!err.str().empty())
    {
        hasError=true;
        GuiLog().error() << err.str();
        errOut="<b>Command </b>" + cmd + " <b>failed.</b> <br>" + err.str();
    }

    return !hasError;
}

bool BufrDataDump::debug(const std::string& sourceFile, int msgCnt,int /*subsetCnt*/,int subsetCnt, bool compressed,std::string& errOut)
{
    std::stringstream in, err, cnt_s;  
    std::string inFile;
    MvTmpFile msgFile;
    MvTmpFile subsetFile;

    //Filter out the given message
    if(!filterMessage(sourceFile,msgCnt,msgFile.path(),errOut))
        return false;

    //For non-compressed messages we need to filter out the required susbset
    if(!compressed && subsetCnt > 1)
    {
        if(!filterSubset(msgFile.path(),1,subsetCnt,subsetFile.path(),errOut))
            return false;

        inFile=subsetFile.path();
    }
    else
    {
        inFile=msgFile.path();
    }

    cnt_s << 1;

    //Build command
    std::string cmd="export ECCODES_DEBUG=1; " + BUFR_DUMP_EXE + " -ja  -w count=" + cnt_s.str() + " " + "\"" + inFile + "\"" ; //+ "  > " + tmpFile_;

    GuiLog().task() << "Generating debug information for message: " << msgCnt << GuiLog::commandKey() << cmd;

    //Run command
    int exitCode;
    std::stringstream shellErr;
    bool ret=shellCommand(cmd,in,err,shellErr,exitCode);

    //Debud is dumped to stderr
    debug_=err.str();

    //Remove unwanted strings from debug text
    std::string rs="ECCODES DEBUG   :  ";
    size_t start_pos = 0;
    while((start_pos = debug_.find(rs, start_pos)) != std::string::npos)
    {
        debug_.replace(start_pos, rs.length(),"");       
    }

    bool hasError=false;
    if(exitCode >0)
    {
        if(debug_.empty())
        {
            GuiLog().error() << "Command exited with code: "  << exitCode;
            errOut+="Command exited with code: " + metview::toString<int>(exitCode) + " ";
            hasError=true;
        }
    }
    if(!ret)
    {
        GuiLog().error() << shellErr.str();
        errOut+=shellErr.str();
        hasError=true;
    }

    return !hasError;
}

bool BufrDataDump::runMessageFilter(const std::string& filterStr,const std::string& inFile,
                                    const std::string& outFile)
{
    MvTmpFile rulesFile;

    std::string cmd=BUFR_FILTER_EXE + " -o " + outFile + " " + rulesFile.path() + " " + inFile;

    //GuiLog().task() << "Filtering out subset: " << subsetCnt << " from message: " << msgCnt <<
    //              GuiLog::commandKey() << cmd;

    //Create the filter
    std::string errOut;
    std::ofstream out;
    out.open(rulesFile.path().c_str());
    if(!out.good())
    {
        GuiLog().error() << "Cannot create temporary rules file: " << rulesFile.path();
        errOut="Cannot create temporary rules file: " + rulesFile.path();
        return false;
    }

    //out << "set unpack=1;" << std::endl;
    //out << "set extractSubset=" << subsetCnt << ";" << std::endl;
    //out << "set doExtractSubsets=1;" << std::endl;
    //out << "if(count == " << msgCnt << " ) { write; } " << std::endl;
    out << filterStr;
    out.close();

    //Run the filter
    std::stringstream in, err;
    bool hasError=false;
    int exitCode;
    std::stringstream shellErr;
    bool ret=shellCommand(cmd,in,err,shellErr,exitCode);

    if(exitCode > 0)
    {
        hasError=true;
        GuiLog().error() << "Command exited with code: "  << exitCode;
        if(err.str().empty())
            errOut+="<b>Command</b>" + cmd + " exited with <b>code:</b> " + metview::toString<int>(exitCode) + " ";
    }

    if(!ret)
    {
        hasError=true;
        GuiLog().error() << shellErr.str();
        errOut+=shellErr.str();
    }

    //Get stderr of command
    if(!err.str().empty())
    {
        hasError=true;
        GuiLog().error() << err.str();
        errOut="<b>Command </b>" + cmd + " <b>failed.</b> <br>" + err.str();
    }

    return !hasError;
}

//=====================================
//
// BufrMetaData
//
//=====================================

BufrMetaData::BufrMetaData()
{
#if 0
    string cmd;

    GRIB_DEFINITION_PATH_ENV="";
    const char* grb_def_path=getenv("GRIB_DEFINITION_PATH");
    if(grb_def_path)
    {
        GRIB_DEFINITION_PATH_ENV=string(grb_def_path);
    }
#endif

    //Define the commnad line tools
#ifdef ECCODES_UI
    char *bdexe=getenv("CODES_UI_BUFR_DUMP");
    char *bfexe=getenv("CODES_UI_BUFR_FILTER");
    char *bcexe=getenv("CODES_UI_BUFR_COPY");
#else
    char *bdexe=getenv("METVIEW_BUFR_DUMP");
    char *bfexe=getenv("METVIEW_BUFR_FILTER");
    char *bcexe=getenv("METVIEW_BUFR_COPY");
#endif

    if(bdexe)
    {
        BUFR_DUMP_EXE=bdexe;
    }
    else
    {
        BUFR_DUMP_EXE="bufr_dump";
    }

    if(bfexe)
    {
        BUFR_FILTER_EXE=bfexe;
    }
    else
    {
        BUFR_FILTER_EXE="bufr_filter";
    }

    if(bcexe)
    {
        BUFR_COPY_EXE=bcexe;
    }
    else
    {
        BUFR_COPY_EXE="bufr_copy";
    }
}

BufrMetaData::~BufrMetaData()
{
    clear();
}

void BufrMetaData::clear()
{
    for(std::vector<MvEccBufrMessage*>::iterator it=messages_.begin(); it != messages_.end(); ++it)
    {
        delete *it;
    }
    messages_.clear();
    firstTypicalDate_.clear();
    firstKeys_.clear();

    MvMessageMetaData::clear();
}

bool BufrMetaData::useMessageSizeForProgressIndicator()
{
    return false;
}

int BufrMetaData::computeTotalMessageNum()
{
    FILE* fp;
    fp = fopen(fileName_.c_str(),"r");
    if(!fp)
    {
        //log->error("GribMetaData::readMessages() ---> Cannot open grib file: \n        " + fileName_ + "\n");
        return 0;
    }

    int res=0;
    if(codes_count_in_file(0,fp,&res) != CODES_SUCCESS)
        res=0;

    fclose(fp);
    return res;
}

int BufrMetaData::getEstimatedMessageNum()
{    
    //return messageNum_;
    return totalMessageNum_;
}

void BufrMetaData::getKeyList(string section,MvKeyProfile* prof)
{
    for(vector<MvKey*>::iterator it=allKey_->begin(); it != allKey_->end(); it++)
    {
        if((*it)->metaData("section") == section)
        {
            prof->addKey((*it)->clone());
        }
    }
}

void BufrMetaData::setFileName(string fname)
{
    clear();

    fileName_=fname;
    totalMessageNum_=computeTotalMessageNum();
}

void BufrMetaData::loadKeyProfile(MvKeyProfile *prof)
{
    prof->clearKeyData();
    readMessages(prof);
}

void BufrMetaData::readMessages(MvKeyProfile *prof)
{
    FILE* fp;
    codes_handle* ch=NULL;
    int err=0;
    int bufr_count=0;
    std::string cbuff;

    assert(prof);

    //Set the proper keytypes in the profile
    for(unsigned int i=0; i< prof->size() ; i++ )
    {
        if(prof->at(i)->name() != "MV_Index")
        {
            prof->at(i)->setKeyType(MvKey::EccodesKey);
        }
    }

    GuiLog().task() << "Generating bufr key list for all the messages" <<
                GuiLog::methodKey() << "ecCodes C interface";

    messageNum_=0;

    //open bufr file for reading
    fp = fopen(fileName_.c_str(),"r");
    if(!fp)
    {
        GuiLog().error() << "BufrMetaData::readMessages() ---> Cannot open bufr file: \n        " <<
                      fileName_;
        return;
    }

    //For filtered datasets
    if(filterEnabled_==true)
    {

    }
    else
    {
        //Get messages form the file
        while((ch = codes_handle_new_from_file(NULL,fp,PRODUCT_BUFR,&err)) != NULL || err != CODES_SUCCESS)
        {
            if(!ch)
            {         
                GuiLog().error() <<  "BufrMetaData::readMessages() --->  Unable to create code handle for message count: " << bufr_count+1;
                //continue;
            }

            MvEccBufrMessage* m=0;
            if(firstScan_)
            {
                m=new MvEccBufrMessage(ch,bufr_count);
                messages_.push_back(m);
            }
            else
            {
                m=messages_[bufr_count];
            }
    
            //printf("message: %d\n",bufr_count);
            if(!m->isHeaderValid() || !m->isDataValid() )
            {
                prof->markErrorRow(bufr_count);
            }

            readMessage(prof,ch,bufr_count);
            bufr_count++;

            //printf(" <-- message\n");

            broadcast(&MvMessageMetaDataObserver::messageScanStepChanged,bufr_count);

            if(ch)
                codes_handle_delete(ch);
        }

        if(firstScan_)
        {
            totalMessageNum_=bufr_count;
        }
    }

    fclose(fp);

    messageNum_=bufr_count;
    firstScan_=false;

    //------------------------------
    // Format key "count"
    //------------------------------

    char c[20];
    int n,c_format_id;
    const char c_format[4][5]={"%02d","%03d","%04d","%05d"};

    int num=messageNum_;
    if(num < 100)
    {
        c_format_id=0;
    }
    else if(num < 1000)
    {
        c_format_id=1;
    }
    else if(num < 10000)
    {
        c_format_id=2;
    }
    else
    {
        c_format_id=3;
    }

    MvKey* key=prof->key("count");
    if(key)
    {
        for(unsigned int i=0; i< key->value().size(); i++)
        {
            string s=key->value()[i];
            if(s != "")
            {
                stringstream sst(s);
                sst >> n;
                sprintf(c,c_format[c_format_id],n);
                s=c;
                key->setValue(i,s);
            }
        }

    }

    key=prof->key("MV_Index");
    if(key)
    {
        for(unsigned int i=0; i< key->value().size(); i++)
        {
            sprintf(c,c_format[c_format_id],i+1);
            string s(c);
            key->setValue(i,s);
        }

    }
}

void BufrMetaData::readMessageFailed(MvKeyProfile* prof,int msgCnt)
{
    prof->markErrorRow(msgCnt);
    for(unsigned int i=0; i< prof->size() ; i++ )
    {
        prof->at(i)->addValue("N/A");
    }
}

//R
void BufrMetaData::readMessage(MvKeyProfile* prof,codes_handle *ch,int msgCnt)
{
    if(!ch)
    {
        readMessageFailed(prof,msgCnt);
        return;
    }

    const int MAX_VAL_LEN=1024;
    long longValue;
    char value[MAX_VAL_LEN];
    string svalue;
    size_t vlen=MAX_VAL_LEN;

    long centre;
    if(MV_CODES_CHECK(codes_get_long(ch,"bufrHeaderCentre",&centre),0) == false)
    {
        readMessageFailed(prof,msgCnt);
        return;
    }

    if(MV_CODES_CHECK(codes_get_long(ch,"section1Flags",&longValue),0) == false)
    {
        readMessageFailed(prof,msgCnt);
        return;
    }

    if(firstTypicalDate_.empty())
    {
        long lDateValue;
        if(MV_CODES_CHECK(codes_get_long(ch,"typicalDate",&lDateValue),0))
        {
            firstTypicalDate_ = metview::toString<long>(lDateValue);
        }
        else
        {
            firstTypicalDate_="N/A";
        }
    }

    bool hasLocal2=(centre == 98 && longValue != 0);

    for(unsigned int i=0; i< prof->size() ; i++ )
    {
        size_t len;
        int keyType;
        const char* name=prof->at(i)->name().c_str();
        std::string strName=prof->at(i)->name();
        bool foundValue=false;

        std::size_t pos;
        if((pos = strName.find(":98")) == strName.size()-3)
        {
            if(hasLocal2)
            {
                strName=strName.substr(0, strName.size()-3);
                name=strName.c_str();
            }
            else
            {
                prof->at(i)->addValue("N/A");
                continue;
            }
        }

        if(grib_get_native_type(ch,name,&keyType) == CODES_SUCCESS &&
           grib_get_size(ch,name,&len) == CODES_SUCCESS &&
           (keyType == CODES_TYPE_STRING || keyType == CODES_TYPE_LONG ||
            keyType == CODES_TYPE_DOUBLE))
        {
            if(keyType == CODES_TYPE_STRING || len == 1)
            {
                if((prof->at(i)->readIntAsString() == false) &&
                   keyType ==CODES_TYPE_LONG)
                {
                    if(codes_get_long(ch,name,&longValue) == CODES_SUCCESS)
                    {                      
                        stringstream s;
                        s << longValue;
                        prof->at(i)->addValue(s.str().c_str());

                        foundValue=true;
                    }
                }
                else
                {
                    vlen=MAX_VAL_LEN;
                    if(codes_get_string(ch,name,value,&vlen) == CODES_SUCCESS)
                    {
                        if(prof->at(i)->name() == "section1Flags")
                        {
                             prof->at(i)->addValue((strcmp(value,"0") == 0)?"0":"1");
                        }
                        else
                        {
                            prof->at(i)->addValue(value);
                        }
                        foundValue=true;
                    }
                }
            }
#if 0
            else if((strcmp(name,"paramId") == 0 || strcmp(name,"parameter.paramId") == 0) &&
                    keyType == GRIB_TYPE_LONG)
            {
                if(prof->at(i)->readIntAsString() == false)
                {
                    if(grib_get_long(gh,name,&longValue) == GRIB_SUCCESS)
                    {
                        stringstream s;
                        s << longValue;
                        prof->at(i)->addValue(s.str().c_str());
                        foundValue=true;
                    }
                }
                else
                {
                    vlen=MAX_VAL_LEN;
                    if(grib_get_string(gh,name,value,&vlen) == GRIB_SUCCESS)
                    {
                        prof->at(i)->addValue(value);
                        foundValue=true;
                    }
                }
            }
#endif
            else
            {
                stringstream s;
                s << len;
                svalue="Array (" + s.str() + ")";
                prof->at(i)->addValue(svalue);
                foundValue=true;
            }
        }

        if(!foundValue)
        {
            prof->at(i)->addValue("N/A");
        }
    }
}

MvEccBufrMessage* BufrMetaData::message(int index) const
{
    assert(index >=0 && index < static_cast<int>(messages_.size()));
    return messages_.at(index);
}

const std::set<std::string>& BufrMetaData::firstMessageKeys()
{
    readFirstMessageKeys();
    return firstKeys_;
}

void BufrMetaData::readFirstMessageKeys()
{
    if(!firstKeys_.empty())
      return;

    FILE* fp;
    codes_handle* ch=NULL;
    int err=0;
    int bufr_count=0;

    std::string cbuff;

    GuiLog().task() << "Collecting bufr keys from first message" <<
            GuiLog::methodKey() << "ecCodes C interface";

    //open bufr file for reading
    fp = fopen(fileName_.c_str(),"r");
    if(!fp)
    {
        GuiLog().error() << "BufrMetaData::readFirstMessageKeys() ---> Cannot open bufr file: \n        " <<
                  fileName_;
        return;
    }

    //Get messages form the file
    while((ch = codes_handle_new_from_file(NULL,fp,PRODUCT_BUFR,&err)) != NULL || err != CODES_SUCCESS)
    {
        if(!ch)
        {
            GuiLog().error() <<  "BufrMetaData::readFirstMessageKeys() --->  Unable to create code handle for message count: " << bufr_count+1;
            continue;
        }
        else
        {
            bufr_keys_iterator* kiter=NULL;
            char* name=0;

            codes_set_long(ch,"unpack",1);

            kiter=codes_bufr_data_section_keys_iterator_new(ch);
            if (!kiter)
            {
                codes_handle_delete(ch);
                continue;
            }

            while(codes_bufr_keys_iterator_next(kiter))
            {
                if((name = codes_bufr_keys_iterator_get_name(kiter)))
                {
                    std::string nameStr(name);

                    if(!nameStr.empty() && nameStr.find("->") == std::string::npos)
                    {
                        if(nameStr[0] == '#')
                        {
                            std::size_t ipos = nameStr.find("#",1);
                            if (ipos != std::string::npos)
                                nameStr=nameStr.substr(ipos+1);
                        }

                        firstKeys_.insert(nameStr);
                    }
                 }
             }

             codes_handle_delete(ch);
             break;
        }

    }

    fclose(fp);
}

void BufrMetaData::readCompressedData(MvKeyProfile* prof,int index,int subsetNum)
{
    FILE* fp;
    codes_handle* ch=NULL;
    int err=0;
    int bufr_count=0;

    assert(prof);
    assert(prof->size() == 0);

    std::string cbuff;

    GuiLog().task() << "Generating bufr key list for all the messages" <<
            GuiLog::methodKey() << "ecCodes C interface";

    //open bufr file for reading
    fp = fopen(fileName_.c_str(),"r");
    if(!fp)
    {
        GuiLog().error() << "BufrMetaData::readMessages() ---> Cannot open bufr file: \n        " <<
                  fileName_;
        return;
    }

    //For filtered datasets
    if(filterEnabled_==true)
    {

    }
    else
    {
        //The first key will be the subset count
        MvKey *subsetKey=new MvKey("subset","subset");
        subsetKey->setValueType(MvKey::IntType);
        subsetKey->setConstant(true); //!!!!
        prof->addKey(subsetKey);

        //Get messages form the file
        while((ch = codes_handle_new_from_file(NULL,fp,PRODUCT_BUFR,&err)) != NULL || err != CODES_SUCCESS)
        {
            if(!ch)
            {
                GuiLog().error() <<  "BufrMetaData::readMessages() --->  Unable to create code handle for message count: " << bufr_count+1;
                //continue;
            }

            if(bufr_count == index)
            {
                readCompressedData(prof,ch);
                int n=prof->valueNum();
                if(n > 1 && n==subsetNum)
                    subsetKey->setIntRange(1,subsetNum);
                else
                    prof->clear();

                if(ch) codes_handle_delete(ch);
                break;

            }

            bufr_count++;
            if(ch) codes_handle_delete(ch);
        }
    }

    fclose(fp);
}


void BufrMetaData::readCompressedData(MvKeyProfile* prof,codes_handle* ch)
{
    //int codes_bufr_copy_data(grib_handle* hin, grib_handle* hout)
    //{
    bufr_keys_iterator* kiter=NULL;
    char* name=0;
    int nkeys=0;
    int keyType = 0;
    long longValue =0;
    long* longArray=0;
    size_t longArrayLen=0;
    double doubleValue=0.;
    double* doubleArray=0;
    size_t doubleArrayLen=0;   
    char** strArray=0;
    size_t strArrayLen=0;
    size_t len;
    size_t maxStrLen=128;

    if (ch==NULL)
    {
        return;
    }

    codes_set_long(ch,"unpack",1);

    kiter=codes_bufr_data_section_keys_iterator_new(ch);
    if (!kiter) return;

    while(codes_bufr_keys_iterator_next(kiter))
    {
        if((name = codes_bufr_keys_iterator_get_name(kiter)))
        {
            std::string nameStr(name);
            MvKey *mvk=new MvKey(nameStr,nameStr) ;           
            prof->addKey(mvk);

            if(codes_get_native_type(ch,name,&keyType) == CODES_SUCCESS &&
               codes_get_size(ch,name,&len) == CODES_SUCCESS)
            {
                if(keyType == CODES_TYPE_STRING)
                {
                    mvk->setValueType(MvKey::StringType);
                    if(len == 1)
                    {
                        char buf[maxStrLen];
                        mvk->setConstant(true);
                        codes_get_string(ch,name,buf,&maxStrLen);
                        mvk->addStringValue(std::string(buf));
                    }
                    else if(len > 1)
                    {
                        mvk->setConstant(false);
                        if(len != strArrayLen)
                        {
                           if(strArray)
                           {
                               for(size_t i = 0; i < strArrayLen; ++i)
                                   free(strArray[i]);

                               free(strArray);
                           }
                           strArrayLen=len;
                           strArray = (char**)malloc(len*sizeof(char*));
                           for(size_t i = 0; i < len; ++i)
                              strArray[i] = new char[maxStrLen];
                        }

                        //Get all values
                        size_t itotal = maxStrLen*len;
                        codes_get_string_array(ch,name,strArray,&itotal);

                        for(size_t i = 0; i < len; ++i)
                            mvk->addStringValue(std::string(strArray[i]));
                    }
                }
                else if(keyType == CODES_TYPE_LONG)
                {
                    mvk->setValueType(MvKey::IntType);
                    mvk->setIntMissingValue(CODES_MISSING_LONG);
                    if(len == 1)
                    {
                        mvk->setConstant(true);
                        if(codes_get_long(ch,name,&longValue) == CODES_SUCCESS)
                        {
                            mvk->addIntValue(longValue);
                        }
                    }
                    else if(len > 1)
                    {
                        mvk->setConstant(false);
                        if(len != longArrayLen)
                        {
                           if(longArray)
                           {
                               free(longArray);
                           }
                           longArrayLen=len;
                           longArray = (long*)malloc(len*sizeof(long));

                        }
                        assert(len > 1);
                        assert(longArray);
                        size_t itotal=len;
                        CODES_CHECK(codes_get_long_array(ch,name,longArray,&itotal),0);
                        for(size_t i=0; i < len; i++)
                        {
                            int iv=longArray[i];
                            mvk->addIntValue(iv);
                        }
                    }
                }
                else if(keyType == CODES_TYPE_DOUBLE)
                {
                    mvk->setValueType(MvKey::DoubleType);
                    mvk->setDoubleMissingValue(CODES_MISSING_DOUBLE);
                    if(len == 1)
                    {
                        mvk->setConstant(true);
                        if(codes_get_double(ch,name,&doubleValue) == CODES_SUCCESS)
                        {
                            mvk->addDoubleValue(doubleValue);
                        }
                    }
                    else if(len > 1)
                    {
                        mvk->setConstant(false);
                        if(len != doubleArrayLen)
                        {
                           if(doubleArray)
                           {
                               free(doubleArray);
                           }
                           doubleArrayLen=len;
                           doubleArray = (double*)malloc(len*sizeof(double));
                        }
                        assert(len > 1);
                        assert(doubleArray);
                        size_t itotal=len;
                        CODES_CHECK(codes_get_double_array(ch,name,doubleArray,&itotal),0);
                        for(size_t i=0; i < len; i++)
                           mvk->addDoubleValue(doubleArray[i]);
                    }
                }
            }

        /* if the copy fails we want to keep copying without any error messages.
    This is because the copy can be between structures that are not
    identical and we want to copy what can be copied and skip what
    cannot be copied because is not in the output handle
    */
        //err=codes_copy_key(hin, hout, name, 0);
        //if (err==0) nkeys++;
        //grib_context_free(ch->context,name);
            //free(name);
            //name=0;
        }
    }

    if (nkeys > 0)
    {
        /* Do the pack if something was copied */
        //err=grib_set_long(hout, "pack", 1);
    }

    codes_bufr_keys_iterator_delete(kiter);

    if(longArray)
        free(longArray);
    if(doubleArray)
        free(doubleArray);

    if(strArray)
    {
        for(size_t i = 0; i < strArrayLen; ++i)
            free(strArray[i]);
    }

}

string BufrMetaData::formatDate(string y, string m, string d)
{
    string res=y;
    res.append((m.size() == 1)?"0" + m : m);
    res.append((d.size() == 1)?"0" + d : d);

    if(res.find("N/A") != string::npos)
        res="N/A";

    return res;
}

string BufrMetaData::formatTime(string h, string m, string s)
{
    string res=(h.size() == 1)?"0" + h : h;
    res.append((m.size() == 1)?"0" + m : m);
    res.append((s.size() == 1)?"0" + s : s);

    if(res.find("N/A") != string::npos)
        res="N/A";

    return res;
}

//Read all locations in the BUFR file
bool BufrMetaData::readLocations(BufrLocationCollector* collector,std::string& errOut)
{
    FILE* fp;
    codes_handle* ch=NULL;
    int err=0;
    int bufr_count=1;

    //assert(prof);
    //assert(prof->size() == 0);

    std::string cbuff;

    GuiLog().task() << "Scanning for locations" <<
            GuiLog::methodKey() << "ecCodes C interface";

    //open bufr file for reading
    fp = fopen(fileName_.c_str(),"r");
    if(!fp)
    {
        GuiLog().error() << "BufrMetaData::readLocations() ---> Cannot open bufr file: \n        " <<
                  fileName_;
        errOut+="Cannot open bufr file: " + fileName_;
        return false;
    }

    //For filtered datasets
    if(filterEnabled_==true)
    {

    }
    else
    {        
        while((ch = codes_handle_new_from_file(NULL,fp,PRODUCT_BUFR,&err)) != NULL || err != CODES_SUCCESS)
        {
            if(!ch)
            {
                GuiLog().error() <<  "BufrMetaData::readLocations() --->  Unable to create code handle for message count: " << bufr_count+1;
                //continue;
            }

            readMessageLocations(ch,bufr_count,collector);

            broadcast(&MvMessageMetaDataObserver::locationScanStepChanged,bufr_count);

            bufr_count++;
            if(ch) codes_handle_delete(ch);
        }
    }

    fclose(fp);

    return true;
}

//Reads the locations from a given message amd collect them into an MvKeyProfile
void BufrMetaData::readMessageLocations(codes_handle *ch,int msgCnt,BufrLocationCollector *collector)
{
    if(!ch)
    {
        //readMessageFailed(prof,msgCnt);
        return;
    }

    long longValue;

    //Header keys
    long centre=0;
    if(MV_CODES_CHECK(codes_get_long(ch,"bufrHeaderCentre",&centre),0) == false)
    {
        //readMessageFailed(prof,msgCnt);
        return;
    }

    if(MV_CODES_CHECK(codes_get_long(ch,"section1Flags",&longValue),0) == false)
    {
        //readMessageFailed(prof,msgCnt);
        return;
    }

    long subsetNum=0;
    if(MV_CODES_CHECK(codes_get_long(ch,"numberOfSubsets",&subsetNum),0) == false)
    {
        //readMessageFailed(prof,msgCnt);
        return;
    }

    long compressed=0;
    if(MV_CODES_CHECK(codes_get_long(ch,"compressedData",&compressed),0) == false)
    {
        //readMessageFailed(prof,msgCnt);
        return;
    }

    bool hasLocal2=(centre == 98 && longValue != 0);

    //For ECMWF bufrs with section 2 we read the lat/lon from the
    //header when there are no subsets
    if(hasLocal2 && subsetNum == 1)
    {
        double latVal,lonVal;
        bool hasLocalPos=true;
        std::string latName="localLatitude";
        std::string lonName="localLongitude";
        if(MV_CODES_CHECK(codes_get_double(ch,latName.c_str(),&latVal),0)== false)
        {
            hasLocalPos=false;
        }
        if(MV_CODES_CHECK(codes_get_double(ch,lonName.c_str(),&lonVal),0)== false)
        {
            hasLocalPos=false;
        }

        if(hasLocalPos)
        {
            collector->add(msgCnt,1,-1,latVal,lonVal);
            return;
        }
    }

    //Read the position from the data
    size_t latAllocLen=0, lonAllocLen=0;
    size_t latLen=0, lonLen=0;
    double *lonArray=0;
    double *latArray=0;

    //expand the message
    codes_set_long(ch,"unpack",1);

    //compressed data

    if(compressed)
    {        
        int maxRank=1000000;
        int ir=1;
        while(ir < maxRank)
        {
            double latVal,lonVal;
            latLen=0;
            lonLen=0;

            std::string latName="#" + metview::toString(ir) + "#latitude";
            std::string lonName="#" + metview::toString(ir) + "#longitude";

            //MV_CODES_CHECK(codes_get_size(ch, latName.c_str(), &latLen),0);
            //MV_CODES_CHECK(codes_get_size(ch, lonName.c_str(), &lonLen),0);

            codes_get_size(ch, latName.c_str(), &latLen);
            codes_get_size(ch, lonName.c_str(), &lonLen);

            if(latLen == 0 || lonLen == 0)
                break;

            //Single value
            if(latLen == 1)
            {
                MV_CODES_CHECK(codes_get_double(ch,latName.c_str(),&latVal),0);
            }
            //Array
            else
            {
                readDoubleArray(ch,latName,latLen,latAllocLen,&latArray);
            }

            //Single value
            if(lonLen == 1)
            {
                MV_CODES_CHECK(codes_get_double(ch,lonName.c_str(),&lonVal),0);
            }
            //Array
            else
            {
                readDoubleArray(ch,lonName,lonLen,lonAllocLen,&lonArray);
            }

            if(latLen != 1 && lonLen != 1 &&
              (latLen != lonLen || latLen != static_cast<size_t>(subsetNum)))
                break;

            for(int i=0; i < subsetNum; i++)
            {
                collector->add(msgCnt,i+1,ir,
                               (latLen==1)?latVal:latArray[i],
                               (lonLen==1)?lonVal:lonArray[i]);
            }
            ir++;
       }
    }

    //uncompressed
    else
    {
        //loop for the subsets
        for(int i=0; i < subsetNum; i++)
        {
            latLen=0;
            lonLen=0;

            std::string latName="/subsetNumber="+ metview::toString<int>(i+1) + "/latitude";
            std::string lonName="/subsetNumber="+ metview::toString<int>(i+1) + "/longitude";

            //MV_CODES_CHECK(codes_get_size(ch, latName.c_str(), &latLen),0);
            //MV_CODES_CHECK(codes_get_size(ch, lonName.c_str(), &lonLen),0);

            codes_get_size(ch, latName.c_str(), &latLen);
            codes_get_size(ch, lonName.c_str(), &lonLen);

            if(latLen == lonLen && latLen > 0)
            {
                //Single value
                if(latLen == 1)
                {
                    double latVal,lonVal;
                    MV_CODES_CHECK(codes_get_double(ch,latName.c_str(),&latVal),0);
                    MV_CODES_CHECK(codes_get_double(ch,lonName.c_str(),&lonVal),0);

                    collector->add(msgCnt,i+1,1,latVal,lonVal);
                }
                //array
                else
                {
                    readDoubleArray(ch,latName,latLen,latAllocLen,&latArray);
                    readDoubleArray(ch,lonName,latLen,lonAllocLen,&lonArray);

                    for(size_t j=0; j < latLen; j++)
                    {                       
                        collector->add(msgCnt,i+1,j+1,latArray[j],lonArray[j]);
                    }
                }
            }
        }
    }

    if(latAllocLen > 0)
    {      
        free(latArray);
    }
    if(lonAllocLen > 0)
    {
        free(lonArray);
    }
}


void BufrMetaData::readDoubleArray(codes_handle* ch,const std::string& key,std::size_t len,std::size_t& allocLen,double **data)
{
    if(len > allocLen)
    {
        if(*data)
        {
            free(*data);
        }
        allocLen=len;
        *data = (double*)malloc(len*sizeof(double));
    }

    MV_CODES_CHECK(codes_get_double_array(ch,key.c_str(),*data,&len),0);
}
