/***************************** LICENSE START ***********************************

 Copyright 2016 ECMWF and INPE. This software is distributed under the terms
 of the Apache License version 2.0. In applying this license, ECMWF does not
 waive the privileges and immunities granted to it by virtue of its status as
 an Intergovernmental Organization or submit itself to any jurisdiction.

 ***************************** LICENSE END *************************************/

#include "BufrFilterEngine.h"

#include "MvException.h"
#include "MvMiscelaneous.h"
#include "Tokenizer.h"

#include "BufrFilterDef.h"
#include "MvKeyProfile.h"

#include <assert.h>
#include <iostream>
#include <fstream>
#include <exception>

long MvBufrValueGroup::longMissingValue_=kBufrMissingIntValue;
double MvBufrValueGroup::doubleMissingValue_=kBufrMissingValue;

MvBufrValue::MvBufrValue(const MvKeyValue& value,bool valueTypeKnown) :
    value_(value), cond_(0), rankCond_(0), valueTypeKnown_(valueTypeKnown), hasRank_(false) { init(); }

MvBufrValue::MvBufrValue(const MvKeyValue& value,const MvKeyConditionDefinition& condDef,bool valueTypeKnown) :
    value_(value), cond_(0), condDef_(condDef), rankCond_(0), valueTypeKnown_(valueTypeKnown), hasRank_(false) {init();}

//MvBufrValue::MvBufrValue(const MvKeyValue& value,MvKeyCondition* cond, bool valueTypeKnown) :
//    value_(value), cond_(cond), rankCond_(0), valueTypeKnown_(valueTypeKnown), hasRank_(false) {init();}

void MvBufrValue::init()
{
    keyNameWithoutRank_=value_.key();
    if(value_.key().find("#") == 0)
    {
        hasRank_=true;
        keyNameWithoutRank_=MvObs::keyWithoutOccurrenceTag(keyNameWithoutRank_);
    }
}

void MvBufrValue::setRankCondition(MvKeyCondition *rankCond)
{
    rankCond_=rankCond;

    assert(rankCond_ && rankCond_->data() == "rank");
    assert(rankCond_ && hasRank_ == false);

    if(rankCond_)
    {
        if(rankCond_->data() != "rank")
            throw MvException("Invalid rank condition is defined for parameter " + keyNameWithoutRank_);

        if(hasRank_)
            throw MvException("Key name cannot contain rank! Keyname=" + value_.key());
    }

}

bool MvBufrValue::isSameKey(const std::string& name) const
{
    return (hasRank_)?(value_.key() == name):
            (keyNameWithoutRank_ == MvObs::keyWithoutOccurrenceTag(name));
}

//======================================================================
//
// MvBufrValueGroup
//
//======================================================================

MvBufrValueGroup::~MvBufrValueGroup()
{
    for(std::size_t i=0; i < items_.size(); i++)
        if(items_[i].cond_)
            delete items_[i].cond_;
}

void MvBufrValueGroup::add(const MvBufrValue& val)
{
    items_.push_back(val);
}

void MvBufrValueGroup::reset()
{
    for(std::size_t i=0; i < items_.size(); i++)
    {
        items_[i].value_.resetValue();
        if(items_[i].cond_)
           items_[i].cond_->reset();
    }
}

bool MvBufrValueGroup::isAllValueSet() const
{
    for(std::size_t i=0; i < items_.size(); i++)
    {
        if(!items_[i].value_.isSet())
            return false;
    }
    return true;
}

bool MvBufrValueGroup::isNoValueSet() const
{
    for(std::size_t i=0; i < items_.size(); i++)
    {
        if(items_[i].value_.isSet())
            return false;
    }
    return true;
}

bool MvBufrValueGroup::isMissingValue(double val)
{
    return fabs(val - doubleMissingValue_) < 10.0E-4;
}

bool MvBufrValueGroup::isMissingValue(long val)
{
    return val == longMissingValue_;
}

const MvKeyValue& MvBufrValueGroup::value(const std::string& keyName) const
{
    for(std::size_t i=0; i < items_.size(); i++)
    {
       if(items_[i].value_.key() == keyName)
           return items_[i].value_;
    }
    static MvKeyValue emptyVal;
    return emptyVal;
}

void MvBufrValueGroup::adjustType(int idx,int eccType)
{
    assert(idx >=0 && idx < static_cast<int>(items_.size()));

    if(items_[idx].valueTypeKnown_ == false)
    {
        std::string key=items_[idx].value_.key();
        MvVariant::Type type=items_[idx].value_.type();
        bool validType=false;

        if(eccType == CODES_TYPE_LONG)
        {
            validType=true;
            if(type != MvVariant::LongType)
            {
                items_[idx].value_=MvKeyValue(key,MvVariant::LongType);
            }
        }
        else if(eccType == CODES_TYPE_DOUBLE)
        {
            validType=true;
            if(type != MvVariant::DoubleType)
            {
                items_[idx].value_=MvKeyValue(key,MvVariant::DoubleType);
            }
        }
        else if(eccType == CODES_TYPE_STRING)
        {
            validType=true;
            if(type != MvVariant::StringType)
            {
                items_[idx].value_=MvKeyValue(key,MvVariant::StringType);
            }
        }

        if(validType)
        {
            //Build the condition if there is a definition for it
            if(items_[idx].cond_ == 0 && !items_[idx].condDef_.isEmpty())
            {
                items_[idx].cond_=MvKeyCondition::make(items_[idx].condDef_,
                                                       items_[idx].value_.type());
            }

            items_[idx].valueTypeKnown_=true;

            //CODES_TYPE_LONG, CODES_TYPE_DOUBLE, CODES_TYPE_STRING,
            //                  CODES_TYPE_MISSING, CODES_TYPE_UNDEFINED
        }
    }
}


bool MvBufrValueGroup::checkCurrentKey(MvObs* obs,int idx)
{
    assert(obs);

    //obs->currentKey() always contains the rank. If the rank is set on the item its
    //keyname contains the rank. The condition itself can never contain a rank for an item
    //in a valuegroup!!

    if(items_[idx].isSameKey(obs->currentKey()))
    {
        if(items_[idx].valueTypeKnown_ == false)
            adjustType(idx,obs->elementValueType());

        std::string sVal;
        long lVal=0;
        double dVal=0;
        bool hasValue=false;

        //Check rank condition
        if(items_[idx].rankCond_)
        {
            MvKeyCondition* rankCond=items_[idx].rankCond_;
            assert(rankCond->data() == "rank");
            int rank=MvObs::occurenceFromKey(obs->currentKey());
            rankCond->eval(rank);

            if(!rankCond->match())
            {
                items_[idx].value_.resetValue();
                if(MvKeyCondition *cond=items_[idx].cond_)
                {
                    cond->reset();
                }
                return true;
            }
        }

        //Check the condition
        if(MvKeyCondition *cond=items_[idx].cond_)
        {
            switch(cond->type())
            {
                case MvVariant::DoubleType:
                    dVal=obs->currentValue();
                    if(!isMissingValue(dVal))
                        cond->eval(dVal);
                    hasValue=true;
                    break;
                case MvVariant::LongType:
                    lVal=obs->currentIntValue();
                    if(!isMissingValue(lVal))
                        cond->eval(lVal);
                    hasValue=true;
                    break;
                case MvVariant::StringType:
                    sVal=obs->stringValue();
                    cond->eval(sVal);
                    hasValue=true;
                    break;
                default:
                    break;
            }

            //if the condition does not match we reset the collected value
            if(!cond->match())
            {
                items_[idx].value_.resetValue();
                    return true;
            }
        }

        //Collect the item value
        switch(items_[idx].value_.type())
        {
        case MvVariant::DoubleType:
        {
            if(!hasValue)
                dVal=obs->currentValue();

            if(includeMissingValue_ || !isMissingValue(dVal))
               items_[idx].value_.setValue(dVal);
            else
               items_[idx].value_.resetValue();

            break;
        }
        case MvVariant::LongType:
        {
            if(!hasValue)
                lVal=obs->currentIntValue();

            if(includeMissingValue_ || !isMissingValue(lVal))
               items_[idx].value_.setValue(lVal);
            else
               items_[idx].value_.resetValue();

            break;
        }
        case MvVariant::StringType:
            if(!hasValue)
                sVal=obs->stringValue();

            items_[idx].value_.setValue(sVal);
            break;
        default:
            break;
        }

        return true;
    }

    return false;
}

void MvBufrValueGroup::checkCurrentKey(MvObs* obs)
{
    assert(obs);

    for(std::size_t i=0; i < items_.size(); i++)
    {        
        if(checkCurrentKey(obs,i))
        {
            //if a given item is not collected (set) we return. A given
            //item can only be collected if all the previous ones were already collected (set)
            if(!items_[i].value_.isSet())
                return;
        }
    }
}

//======================================================================
//
// MvBufrConditionGroup
//
//======================================================================                     

void MvBufrConditionGroup::add(const MvBufrValue& val)
{
    MvBufrValueGroup::add(val);
    updateMatchStatus();
}

void MvBufrConditionGroup::reset()
{
    MvBufrValueGroup::reset();
    updateMatchStatus();
}

void MvBufrConditionGroup::checkCurrentKey(MvObs* obs)
{
    assert(obs);

    for(std::size_t i=0; i < items_.size(); i++)
    {
        if(MvBufrValueGroup::checkCurrentKey(obs,i))
        {
            updateMatchStatus();
            return;
        }
    }
}

void MvBufrConditionGroup::checkConditions(MvObs* obs)
{
    assert(obs);

    for(std::size_t i=0; i < items_.size(); i++)
    {
        std::string keyName=items_[i].value_.key();

        if(items_[i].valueTypeKnown_ == false)
            adjustType(i,obs->elementValueType(keyName));

        MvKeyCondition *cond=items_[i].cond_;
        if(cond)
        {
            switch(cond->type())
            {
                case MvVariant::DoubleType:                
                    {
                        std::vector<double> dVal;
                        obs->allValues(cond->key(),dVal);
                        for(std::size_t iv=0; iv < dVal.size(); iv++)
                        {
                            if(!isMissingValue(dVal[iv]))
                            {
                                cond->eval(dVal[iv]);
                                if(cond->match())
                                    break;
                            }
                        }
                    }
                    break;
                case MvVariant::LongType:
                    {
                        std::vector<long> lVal;
                        obs->allIntValues(cond->key(),lVal);
                        for(std::size_t iv=0; iv < lVal.size(); iv++)
                        {
                            if(!isMissingValue(lVal[iv]))
                            {
                                cond->eval(lVal[iv]);
                                if(cond->match())
                                    break;
                            }
                        }
                    }
                    break;
                case MvVariant::StringType:
                    {
                        std::vector<std::string> sVal;
                        obs->allStringValues(cond->key(),sVal);
                        for(std::size_t iv=0; iv < sVal.size(); iv++)
                        {
                            cond->eval(metview::simplified(sVal[iv]));
                            if(cond->match())
                                break;
                        }
                    }
                    break;
                default:
                    break;
             }

             //all conditions must match!!!
             if(!cond->match())
                return;
         }

        updateMatchStatus();                   
    }    
}

void MvBufrConditionGroup::updateMatchStatus()
{
    bool all=true;
    for(std::size_t i=0; i < items_.size(); i++)
    {
        if(!items_[i].cond_)
        {
            if(!items_[i].condDef_.isEmpty() || !items_[i].value_.isSet())
            {
                all=false;
                break;
            }
        }
        else if(!items_[i].cond_->match())
        {
            all=false;
            break;
        }
    }
    allMatch_=all;
}


//======================================================================
//
// MvBufrStandardGroup
//
//======================================================================

void MvBufrStandardGroup::checkCurrentKey(MvObs* obs)
{
    assert(obs);

    for(std::size_t i=0; i < items_.size(); i++)
    {
        if(MvBufrValueGroup::checkCurrentKey(obs,i))
        {
            return;
        }
    }
}

//======================================================================
//
// BufrFilter
//
//======================================================================

BufrFilterEngine::BufrFilterEngine(const std::string& inFileName,FilterMode filterMode,MvObsSetIteratorObserver* obsSetObserver) :
    filterMode_(filterMode),
    outBufr_(0),
    inFileName_(inFileName),
    msgCnt_(0),
    obsOrMsg_(NR_returnMsg),
    outType_(NoOutput),
    extractedHasRank_(false),
    result_(0),
    collectMode_(CollectFirst),
    includeMissingValue_(true)
{
    inObs_= new MvObsSet(inFileName_.c_str());
    obsIter_=new MvObsSetIterator(*inObs_);
    if(obsSetObserver)
        obsIter_->setObserver(obsSetObserver);
}

BufrFilterEngine::~BufrFilterEngine()
{
    delete obsIter_;
    delete inObs_;
    if(outBufr_)
        delete outBufr_;
}

bool BufrFilterEngine::isExtractedDefined() const
{
    return extracted_.size() > 0;
}

void BufrFilterEngine::close()
{
  //if( outType_ == eBUFR )
    if(outBufr_)
        outBufr_->close();
  //else
  //   _outFile.close();
}

void BufrFilterEngine::run(const BufrFilterDef& def,const std::string& resFileName,MvKeyProfile* resProf)
{
    result_=resProf;

    def_=def;

    getOutputOptions();
    getIndexOptions();
    getEditionOptions();
    getTypeOptions();
    getIdentOptions();
    getTimeOptions();
    getAreaOptions();
    getCustomOptions();

    if(outType_ != NoOutput)
        outBufr_=new MvObsSet(resFileName.c_str(),"w");

    try
    {
        while((currentObs_ = (*obsIter_)(obsOrMsg_) ))
        {           
            filterOne();
        }
        close();
    }
    catch(std::exception& e )
    {
      //marslog( LOG_WARN, "ObsFilter: %s", e.what() );
      //if( failOnError_ )
          //throw MvException( "ObsFilter failed" );
    }
}

void BufrFilterEngine::filterOne()
{
    if(outType_ == NoOutput)
        return;

    bool didExtract=false;
    int collectCnt=0;
    extracted_.reset();
    location_.reset();
    coordCond_.reset();
    customCond_.reset();

    MvObs* obs=&currentObs_;
    MvObs subsetObs;

    //Subsets
    if(currentObs_.msgSubsetCount() > 1)
    {
        //Uncompressed data
        if(!currentObs_.compressData() )
        {
            int currentSubset=currentObs_.subsetNumber();
            subsetObs=currentObs_.cloneSubset(currentSubset);
            obs=&subsetObs;
        }
        //Compressed data
        {
            /*bool flag = true;
            while(flag)
            {
               anObs;
               flag = anObs.Advance();   //next subset
            }
            return;*/
        }
    }

    //Check custom condition - it is only a message/subset filter condition,
    //but it is not yet implemented in the bufr itartor so we need to
    //do it here
    if(!customCond_.isEmpty())
    {
         //We have to make sure the message is upacked
         obs->setFirstDescriptor();
         customCond_.checkConditions(obs);
         if(!customCond_.match())
         {
             currentObs_.clearIterator();
             return;
         }
    }

    //we need to extract values
    if(outType_ != BufrOutput)
    {
        assert(!extracted_.isEmpty());

        //this force data to be expanded
        bool cont=obs->setFirstDescriptor();

        //We iterate through the keys
        while(cont)
        {         
            //------------------------------------------------------------------------------
            //We only filter by extracted conditions. No coordinate conditions are defined.
            //------------------------------------------------------------------------------
            if(coordCond_.isEmpty())
            {
                extracted_.checkCurrentKey(obs);
                location_.checkCurrentKey(obs);

                //if all the values are collected and all the conditions match
                if(extracted_.isAllValueSet())
                {
                    addToResult(obs);
                    extracted_.reset();
                    didExtract=true;
                    collectCnt++;
                }
            }

            //------------------------------------------------------------------------------
            //Has coordinate conditions
            //------------------------------------------------------------------------------
            else if(!coordCond_.isEmpty())
            {
                coordCond_.checkCurrentKey(obs);
                location_.checkCurrentKey(obs);
                if(coordCond_.match())
                {
                    if(extracted_.isAllValueSet())
                    {
                        addToResult(obs);
                        extracted_.reset();                        
                        didExtract=true;
                        collectCnt++;
                    }
                    extracted_.checkCurrentKey(obs);
                }
#if 0
                else
                {                    
                    //When the coodrinate conditions do not fully match
                    //and we have some extracted data (may not be full).
                    //In this case extracted was only partially collected.
                    if(!extracted_.isNoValueSet())
                    {                        
                        addToResult(obs);
                        didExtract=true;
                    }
                }
#endif
            }

            if(collectMode_ == CollectFirst)
                if(collectCnt == 1)
                    break;

            cont = obs->setNextDescriptor();
        }
    }

    //If we do not do it we will have a crash
    currentObs_.clearIterator();

    //We always create a new bufr file containing only the messages/subsets matching the filter
    if(outType_ == BufrOutput || didExtract)
    {
        assert(outBufr_);
        //first time
        if(msgCnt_ == 0)
        {
            //-- changed 2011-12-30/vk:
            //--  o target BUFR file is set to contain single subsets, but this is in effect
            //--     only when filtering using data values from BUFR Section 4
            //--  o this setting has no effect when filtering only with metadata values in
            //--     BUFR Section 1; in such cases multisubset messages are copied as they
            //--     are (metadata is common to all subsets in a multisubset msg)
            outBufr_->setSubsetMax(1); //-- obs_.msgSubsetCount() );
        }

        //If filtering requires values from data section then multisubset
        //messages have to be filtered on observation (subset) level!

        //filtering with header only i.e. write the whole message!
        if(obsOrMsg_ == NR_returnMsg)
        {
            outBufr_->write(currentObs_);
            //msgCnt_ += currentObs_.msgSubsetCount();
            msgCnt_++;
        }
        //filtering also with data values, write current subset only!
        else
        {
            if(obs->msgSubsetCount() > 1 && obs->compressData())
            {
                outBufr_->writeCompressed(obs);
            }
            else
            {
                outBufr_->add(*obs);
            }
            msgCnt_++;
        }
    }
}

void BufrFilterEngine::addToResult(MvObs* obs)
{
    assert(result_);
    //MvLocation locData=obs->location();
    TStaticTime timeData=obs->obsTime();

    for(std::size_t i=0; i < result_->size(); i++)
    {
        MvKey* rk=result_->at(i);
        assert(rk);
        std::string name=rk->name();
        if(rk->role() == MvKey::IndexRole)
        {
            int idx=rk->valueNum();
            rk->addIntValue(idx+1);
        }
        //The message index (starts from 1) in the filtered messages
        else if(rk->role() == MvKey::MessageIndexRole)
        {
            //rk->addIntValue(obsIter_->currentMessageNumber());
            rk->addIntValue(msgCnt_+1);
        }
        else if(rk->role() == MvKey::SubsetIndexRole)
        {
            rk->addIntValue(1);
        }
        else if(rk->role() == MvKey::DateRole)
        {
            rk->addIntValue(timeData.dateAsInt());
        }
        else if(rk->role() == MvKey::TimeRole)
        {
            rk->addStringValue(timeData.timeAsString());
        }
        else if(rk->role() == MvKey::LatitudeRole)
        {
            rk->addValue(location_.value("latitude").value());
            //rk->addDoubleValue(locData.latitude());
        }
        else if(rk->role() == MvKey::LongitudeRole)
        {
            rk->addValue(location_.value("longitude").value());
            //rk->addDoubleValue(locData.longitude());
        }
        else
        {
            MvVariant::Type type=MvVariant::NoType;

            //Coordinates
            if(rk->role() == MvKey::LevelRole)
                type=coordCond_.value(name).type();
            //extracted values
            else
                type=extracted_.value(name).type();

            if(type == MvVariant::LongType)
            {
                if(rk->valueType() != MvKey::LongType)
                {
                    rk->setValueType(MvKey::LongType);
                }
            }
            else if(type == MvVariant::DoubleType)
            {
                if(rk->valueType() != MvKey::DoubleType)
                {
                    rk->setValueType(MvKey::DoubleType);
                }
            }
            else if(type == MvVariant::StringType)
            {
                if(rk->valueType() != MvKey::StringType)
                {
                    rk->setValueType(MvKey::StringType);
                }
            }

            if(rk->role() == MvKey::LevelRole)
                rk->addValue(coordCond_.value(name).value());
            else
                rk->addValue(extracted_.value(name).value());
        }
    }

    extracted_.reset();
}

void BufrFilterEngine::toGeopoints(const std::string& fName)
{
    std::ofstream out;
    out.open(fName.c_str());
    if(!out.is_open())
    {
        throw MvException("Cannot open output file: " + fName);
    }

    const char mySeparator = '\t';
    out << "#GEO" << std::endl;
    out << "#lat" << mySeparator << "long" << mySeparator
             << "level" << mySeparator << "date" << mySeparator
             << "time" << mySeparator << std::endl << "#DATA" << std::endl;

    assert(result_);
    std::vector<MvKey*> keys(6,0);
    keys[0]=result_->key(MvKey::LatitudeRole);
    keys[1]=result_->key(MvKey::LatitudeRole);
    keys[2]=result_->key(MvKey::LevelRole);
    keys[5]=result_->key(MvKey::NoRole);

    int cnt=result_->valueNum();
    for(int i=0; i < cnt; i++)
    {
        //Using this style caused a crash:
        //out << ((keys[2])?keys[2]->valueAsString(i):"0")

        if(keys[0]) out << keys[0]->doubleValue()[i] << " ";
        else out << "-1 ";

        if(keys[1]) out << keys[1]->doubleValue()[i] << " ";
        else out << "-1 ";

        if(keys[2]) out << keys[2]->valueAsString(i) << " ";
        else out << "0 ";

        out << "0 0 ";

        if(keys[5])  out << keys[5]->valueAsString(i) << " ";
        else out << " ";

        out << std::endl;
    }
}

//--------------------------------------------
// Get filter options
//--------------------------------------------

void BufrFilterEngine::buildConditionDef(const std::string& key,const std::string& oper,
                                const std::string& valueKey,MvKeyConditionDefinition& condDef)
{
    if(!oper.empty())
    {
        //At this point we do not know the type of the parameter, so we
        //assume it is floating point value. We will determine the
        //real type from the data for the first time we have a
        //value to read and readjust the type of the param and that of the
        //conditions.
        std::vector<std::string> vals;
        getStringValues(valueKey,valueKey,vals);
        if(!vals.empty())
            condDef=MvKeyConditionDefinition(key,oper,vals);
    }
}

void BufrFilterEngine::getOutputOptions()
{
    //Get the output type
    std::string val=value("EXTRACT");
    if(val.empty())
    {
        throw MvException("No output type is specified");
    }

    //No extraction
    if(val == "OFF")
    {
        outType_=BufrOutput;
        return;
    }
    else
    {
        outType_=CsvOutput;
    }

    //Get the mumber of parameters
    int parCnt=0;
    intValue("PARAMETER_COUNT","parameterCount",1,parCnt);

    //Missing data handling
    std::string missing=value("MISSING_DATA");
    if(missing ==  "IGNORE")
        includeMissingValue_=false;
    else
        includeMissingValue_=true;

    extracted_.setIncludeMissingValue(includeMissingValue_);

    assert(coordCond_.isEmpty());
    assert(extracted_.isEmpty());

    //The paremeters to extract (with their conditions)
    for(int i=0; i < parCnt; i++)
    {
        MvKeyConditionDefinition condDef;

        std::string parKey=value("PARAMETER_" + metview::toString(i+1));
        std::string parOper=value("PARAMETER_OPERATOR_" + metview::toString(i+1));
        std::string parValueKey="PARAMETER_VALUE_"+ metview::toString(i+1);

        std::string parRank;
        if(getRank("PARAMETER_RANK_" + metview::toString(i+1),parRank))
        {
            extractedHasRank_=true;
            parKey="#" + parRank + "#" + parKey;
        }

        buildConditionDef(parKey,parOper,parValueKey,condDef);

        //We do not know the type of the parameter at this point, we set it to string
        MvKeyValue kVal(parKey,MvVariant::StringType);

        //A condition is defined but we do not know its type (we only have
        //the definition)
        if(!condDef.isEmpty())
        {
            extracted_.add(MvBufrValue(kVal,condDef,false));
        }
        //There is no condition
        else
        {
            extracted_.add(MvBufrValue(kVal,false));
        }
    }

    //The coordinate conditions
    int coordCnt=0;
    intValue("COORDINATE_COUNT","coordKeyCount",0,coordCnt);
    for(int i=0; i < coordCnt; i++)
    {
        std::string coordKey=value("COORDINATE_" + metview::toString(i+1));
        std::string coordOper=value("COORDINATE_OPERATOR_" + metview::toString(i+1));
        std::string coordValueKey="COORDINATE_VALUE_"+ metview::toString(i+1);

        //The rank for a coordinate can be a list!!!
        std::vector<int> coordRanks;
        MvKeyCondition* rankCond=0;
        if(getRanks("COORDINATE_RANK_" + metview::toString(i+1),coordRanks))
        {
            if(coordRanks.size() == 1)
            {
                coordKey="#" + metview::toString(coordRanks[0]) + "#" + coordKey;
            }
            else
            {
                assert(coordRanks.size() > 1);
                std::vector<MvVariant> vv=MvVariant::makeVector(coordRanks);
                rankCond=MvKeyCondition::make("RANK",coordKey,vv);
                if(rankCond)
                    rankCond->setData("rank");
            }
        }

        //We do not know the type of the parameter at this point, we set it to string
        MvKeyValue kVal(coordKey,MvVariant::StringType);

        //The condition
        //MvKeyCondition *cond=0;
        MvKeyConditionDefinition condDef;
        buildConditionDef(coordKey,coordOper,coordValueKey,condDef);

        //A condition is defined but we do not know its type (we only have
        //the definition)
        if(!condDef.isEmpty())
        {
            MvBufrValue bval(kVal,condDef,false);
            if(rankCond) bval.setRankCondition(rankCond);
            coordCond_.add(bval);
        }
        //There is no condition. We just need the existence of the coordinate.
        else
        {
            MvBufrValue bval(kVal,false);
            if(rankCond) bval.setRankCondition(rankCond);
            coordCond_.add(bval);
        }
    }

    if(!extracted_.isEmpty() || !coordCond_.isEmpty())
    {
        std::string collect=value("EXTRACT_MODE");
        if(collect == "FIRST")
            collectMode_=CollectFirst;       
        else if(collect == "ALL")
            collectMode_=CollectAll;
        else
            throw MvException("Invalid value specified for parameter EXTRACT_MODE: " + collect);

        assert(result_);

        MvKeyValue latVal("latitude",MvVariant::DoubleType);
        MvKeyValue lonVal("longitude",MvVariant::DoubleType);
        location_.add(MvBufrValue(latVal,false));
        location_.add(MvBufrValue(lonVal,false));

        MvKey* mvk=0;

#if 0
        mvk=new MvKey("index","#");
        mvk->setRole(MvKey::IndexRole);
        mvk->setValueType(MvKey::IntType);
        result_->addKey(mvk);
#endif

#if 0
        for(std::size_t i=0; i < standard_.size(); i++)
        {
            mvk=new MvKey(standard_.value(i).key(),standard_.value(i).key());
            mvk->setValueType(MvKey::DoubleType);
            result_->addKey(mvk);
        }
#endif

        mvk=new MvKey("message","message");
        mvk->setRole(MvKey::MessageIndexRole);
        mvk->setValueType(MvKey::IntType);
        result_->addKey(mvk);

        mvk=new MvKey("latitude","latitude");
        mvk->setRole(MvKey::LatitudeRole);
        mvk->setValueType(MvKey::DoubleType);
        mvk->setValuePrecision(2);
        result_->addKey(mvk);

        mvk=new MvKey("longitude","longitude");
        mvk->setRole(MvKey::LongitudeRole);
        mvk->setValueType(MvKey::DoubleType);
        mvk->setValuePrecision(2);
        result_->addKey(mvk);

        //Add coordinates to result
        if(value("EXTRACT_COORDINATE") == "ON")
        {
            //At this point we do not know the the type of these params. We set them to
            //double and will adjust them on first read
            for(std::size_t i=0; i < coordCond_.size(); i++)
            {
                mvk=new MvKey(coordCond_.value(i).key(),coordCond_.value(i).key());
                mvk->setRole(MvKey::LevelRole);
                mvk->setValueType(MvKey::DoubleType);
                mvk->setKeyType(MvKey::EccodesKey);
                result_->addKey(mvk);
            }
        }

        //Add date/time to result
        if(value("EXTRACT_DATE") == "ON")
        {
            mvk=new MvKey("date","date");
            mvk->setRole(MvKey::DateRole);
            mvk->setValueType(MvKey::IntType);
            result_->addKey(mvk);

            mvk=new MvKey("time","time");
            mvk->setRole(MvKey::TimeRole);
            mvk->setValueType(MvKey::StringType);
            result_->addKey(mvk);
        }

        //At this point we do not know the the type of these params. We set them to
        //double and will adjust them on first read
        for(std::size_t i=0; i < extracted_.size(); i++)
        {
                mvk=new MvKey(extracted_.value(i).key(),extracted_.value(i).key());
                mvk->setValueType(MvKey::DoubleType);
                mvk->setKeyType(MvKey::EccodesKey);
                result_->addKey(mvk);
        }

        //We  need data from the data section
        obsOrMsg_ = NR_returnObs;
    }
}


void BufrFilterEngine::getIndexOptions()
{
    std::vector<int> vals;
    getIntValues("MESSAGE_INDEX","Message index",1,vals);
    for(std::size_t i=0; i < vals.size(); i++)
    {
        obsIter_->setMessageNumber(vals[i]);
    }
}

void BufrFilterEngine::getEditionOptions()
{
    std::vector<int> vals;

    //Edition
    getIntValues("EDITION","edition",1,vals);
    for(std::size_t i=0; i < vals.size(); i++)
    {
        obsIter_->setEditionNumber(vals[i]);
    }

    //Centre - can be int or string
    vals.clear();
    if(isKeyValueNumber("CENTRE"))
    {
        getIntValues("CENTRE","Centre",0,vals);
        for(std::size_t i=0; i < vals.size(); i++)
        {
            obsIter_->setOriginatingCentre(vals[i]);
        }
    }
    else
    {
        std::vector<std::string> strVals;
        getStringValues("CENTRE","Centre",strVals);
        for(std::size_t i=0; i < strVals.size(); i++)
        {
            obsIter_->setOriginatingCentreAsStr(strVals[i]);
        }
    }

    //SubCentre
    vals.clear();
    getIntValues("SUBCENTRE","subCentre",0,vals);
    for(std::size_t i=0; i < vals.size(); i++)
    {
        obsIter_->setOriginatingSubCentre(vals[i]);
    }

    //Master table version
    vals.clear();
    getIntValues("MASTERTABLE","masterTable",0,vals);
    for(std::size_t i=0; i < vals.size(); i++)
    {
        obsIter_->setMasterTableVersion(vals[i]);
    }

    //Local table version
    vals.clear();
    getIntValues("LOCALTABLE","localTable",0,vals);
    for(std::size_t i=0; i < vals.size(); i++)
    {
        obsIter_->setLocalTableVersion(vals[i]);
    }
}

void BufrFilterEngine::getTypeOptions()
{
    std::vector<int> vals;

    //Types
    getIntValues("DATA_TYPE","dataCategory",1,vals);
    for(std::size_t i=0; i < vals.size(); i++)
    {
        obsIter_->setMessageType(vals[i]);
    }

    //SubTypes
    vals.clear();
    getIntValues("DATA_SUBTYPE","dataSubCategory",1,vals);
    for(std::size_t i=0; i < vals.size(); i++)
    {
        obsIter_->setMessageSubtype(vals[i]);
    }
}

void BufrFilterEngine::getIdentOptions()
{
    std::string keyName=value("IDENT_KEY");

    std::vector<std::string> vals;
    values("IDENT_VALUE",vals);

    if(vals.size() == 1 &&
       (vals[0].empty() || vals[0] == "ANY"))
        return;


    if(keyName == "ident")
    {
        for(std::size_t i=0; i < vals.size(); i++)
        {
            obsIter_->setHeaderIdent(vals[i]);
        }
    }
    else if(keyName == "wmo_station_id")
    {
        for(std::size_t i=0; i < vals.size(); i++)
        {
            //1000*blockNumber+stationNumber;
            obsIter_->setWmoStation(atoi(vals[i].c_str()));
        }

        //It is taken from the data section
        obsOrMsg_ = NR_returnObs;
    }
    else
    {
        obsIter_->setIdentKey(keyName);
        for(std::size_t i=0; i < vals.size(); i++)
        {
            obsIter_->setIdentValue(vals[i]);
        }

        //It is taken from the data section
        obsOrMsg_ = NR_returnObs;
    }
}

void BufrFilterEngine::getTimeOptions()
{
    std::string mode=value("DATEMODE");    
    std::string err;

    //Time window
    if(mode == "WINDOW")
    {
        int year=0, month=0, day=0, hour=0, minute=0, second=0;
        std::string dateVal=value("DATE");
        if(dateVal == "ANY") dateVal.clear();
        if(parseDate(dateVal, year, month, day, err) == false)
        {
            throw MvException("Paramater <b>Date</b>:<br>" + err);
        }

        std::string timeVal=value("TIME");
        if(timeVal == "ANY") timeVal.clear();
        if(parseTime(timeVal,hour,minute,second,err) == false)
        {
            throw MvException("Paramater <b>Time</b>:<br>" + err);
        }

        std::string winVal=value("WINDOW_IN_MINUTES");
        int winSec=-1;
        if(parseTimeWindow(winVal,winSec,err) == false)
        {
            throw MvException("Paramater <b>Window</b>:<br>" + err);
        }

        bool hasDate=(!dateVal.empty());
        bool hasTime=(!timeVal.empty());

        //Has date and time
        if(hasDate && hasTime)
        {
            TDynamicTime dt(year,month,day,hour,minute,second);

            //TODO: make it work for seconds!!! Currently only takes minutes
            obsIter_->setTimeRange(dt, winSec/60 );
         }

         //Only has time
         else if (!hasDate && hasTime)
         {            
            int t=hour*3600+minute*60+second;
            obsIter_->setTimeRangeInSecWithoutDate(t-winSec,t+winSec);
         }
         else if(hasDate && !hasTime)
         {
            throw MvException("Parameter <b>Time</b> is not speficied properly.");

#if 0
            //This should also be supported!!!!!
            if(winSec == 0)
            {
                if(hasDate && !hasTime)
                {
                }
            }
#endif
         }
    } //end window mode

    //Period mode
    else
    {
        int year1=0, month1=0, day1=0, hour1=0, minute1=0, second1=0;
        int year2=0, month2=0, day2=0, hour2=0, minute2=0, second2=0;

        std::string dateVal1=value("DATE_1");
        if(dateVal1 == "ANY") dateVal1.clear();
        if(parseDate(dateVal1,year1,month1,day1,err) == false)
        {
            throw MvException("Paramater <b>Date1</b>:<br>" + err);
        }

        std::string timeVal1=value("TIME_1");
        if(timeVal1 == "ANY") timeVal1.clear();
        if(parseTime(timeVal1,hour1,minute1,second1,err) == false)
        {
            throw MvException("Paramater <b>Time1</b>:<br>" + err);
        }

        std::string dateVal2=value("DATE_2");
        if(dateVal2 == "ANY") dateVal2.clear();
        if(parseDate(dateVal2,year2,month2,day2,err) == false)
        {
            throw MvException("Paramater <b>Date1</b>:<br>" + err);
        }

        std::string timeVal2=value("TIME_2");
        if(timeVal2 == "ANY") timeVal2.clear();
        if(parseTime(timeVal2,hour2,minute2,second2,err) == false)
        {
            throw MvException("Paramater <b>Time2</b>:<br>" + err);
        }

        bool hasDate1=(!dateVal1.empty());
        bool hasTime1=(!timeVal1.empty());

        bool hasDate2=(!dateVal2.empty());
        bool hasTime2=(!timeVal2.empty());

        //Has date and time
        if(hasDate1 && hasDate2 && hasTime1 && hasTime2)
        {          
            TDynamicTime dt1(year1,month1,day1,hour1,minute1,second1);
            TDynamicTime dt2(year2,month2,day2,hour2,minute2,second2);

            obsIter_->setTimeRange(dt1,dt2);
        }
        //only dates, no times
        else if(hasDate1 && hasDate2 && !hasTime1 && !hasTime2)
        {            
            TDynamicTime dt1(year1,month1,day1);
            TDynamicTime dt2(year2,month2,day2,23,59,59);

            obsIter_->setTimeRange(dt1,dt2);
        }
        //only times, no dates
        else if(!hasDate1 && !hasDate2 && hasTime1 && hasTime2)
        {
            //TODO should be also supported!!!
            throw MvException("Period is not speficied properly.");
        }
        else if(!hasDate1 && !hasDate2 && !hasTime1 && !hasTime2)
        {
            return;
        }
        else
        {
             throw MvException("Period is not speficied properly.");
        }
    }

    obsIter_->useObsTime(true); //always use data
    obsOrMsg_ = NR_returnObs;
}

void BufrFilterEngine::getAreaOptions()
{
    std::vector<std::string> vals;
    values("AREA",vals);

    if(vals.empty())
       return;

    if(vals.size() == 4)
    {
        if(vals[0]=="-180" && vals[1]=="90" &&
           vals[2]=="180" && vals[3]=="-90")
            return;

        float w=metview::fromString<float>(vals[0]);
        float n=metview::fromString<float>(vals[1]);
        float e=metview::fromString<float>(vals[2]);
        float s=metview::fromString<float>(vals[3]);

        std::string err;
        if(checkLon(w,e,err) == false)
        {
            throw MvException("Parameter <b>Area</b>:<br>" + err);
        }

        if(checkLat(n,s,err) == false)
        {
            throw MvException("Parameter <b>Area</b>:<br>" + err);
        }

        MvLocation c1(n,w);
        MvLocation c2(s,e);
        obsIter_->setArea(c1,c2);

        //It is taken from the data section
        obsOrMsg_ = NR_returnObs;
    }
}

void BufrFilterEngine::getCustomOptions()
{
    assert(customCond_.isEmpty());

    int customCnt=0;
    intValue("CUSTOM_COUNT","customKeyCount",0,customCnt);

    for(int i=0; i < customCnt; i++)
    {
        std::string customKey=value("CUSTOM_KEY_" + metview::toString(i+1));
        std::string customOper=value("CUSTOM_OPERATOR_" + metview::toString(i+1));
        std::string customValueKey="CUSTOM_VALUE_"+ metview::toString(i+1);
        std::string customValue=value(customValueKey);

        if(customOper.empty())
            throw MvException("Invalid operator is specified for custom condition " +  customKey);

        if(customValue.empty() || customValue == "ANY")
            throw MvException("Invalid value is specified for custom condition " +  customKey);

        std::string customRank;
        if(getRank("CUSTOM_RANK_" + metview::toString(i+1),customRank))
        {
            customKey="#" + customRank + "#" + customKey;
        }

        //We do not know the type of the parameter at this point, we set it to string
        MvKeyValue kVal(customKey,MvVariant::StringType);

        MvKeyConditionDefinition condDef;
        buildConditionDef(customKey,customOper,customValueKey,condDef);

        //A condition is defined but we do not know its type (we only have
        //the definition)
        if(!condDef.isEmpty())
        {
            customCond_.add(MvBufrValue(kVal,condDef,false));
        }
        //There is no condition
        else
        {
            throw MvException("Invalid custom condition for customKey_" +
                    metview::toString(i+1));
        }
    }

    //We need data from the data section
    if(customCnt > 0)
        obsOrMsg_ = NR_returnObs;
}

bool BufrFilterEngine::getRank(const std::string& rankKey,std::string& rankValue) const
{
    rankValue=value(rankKey);
    if(rankValue.empty() || rankValue == "ANY")
        return false;

    if(!isKeyValueNumber(rankKey))
    {
        throw MvException("Invalid value <b>" + rankValue +
                          "</b> specified for parameter <b>" +  rankKey + "</b>");

        return false;
    }

    int rank=-1;
    intValue(rankKey,rankKey,1,rank);
    return rank >= 1;
}

bool BufrFilterEngine::getRanks(const std::string& rankKey,std::vector<int>& rankValue) const
{
    rankValue.clear();

    std::vector<std::string> rankStrVals;
    values(rankKey,rankStrVals);
    if(rankStrVals.empty() || rankStrVals[0] == "ANY")
        return false;

    for(std::size_t i=0; i < rankStrVals.size(); i++)
    {
        if(!isNumber(rankStrVals[i]))
        {
            throw MvException("Invalid value <b>" + rankStrVals[i] +
                          "</b> specified for parameter <b>" +  rankKey + "</b>");

            return false;
        }
        if(metview::fromString<int>(rankStrVals[i]) < 1)
        {
            throw MvException("Invalid value <b>" + rankStrVals[i] +
                          "</b> specified for parameter <b>" +  rankKey + "</b>");

            return false;
        }
    }

    for(std::size_t i=0; i < rankStrVals.size(); i++)
        rankValue.push_back(metview::fromString<int>(rankStrVals[i]));

    return true;
}

void BufrFilterEngine::add(const std::string& key,const std::string& value)
{
    def_.vals_[key]=value;
}

//========================================================================
// Getting values
//========================================================================

bool BufrFilterEngine::isNumber(const std::string& val) const
{
    if(val.empty() || val == "ANY")
        return false;

    return metview::isNumber(val);
}

bool BufrFilterEngine::isKeyValueNumber(const std::string& key) const
{
    std::vector<std::string> v;
    values(key,v);

    if(v.size() > 0)
    {
        return isNumber(v[0]);
    }

    return false;
}

const std::string& BufrFilterEngine::value(const std::string& key,bool mustExist) const
{
    std::map<std::string,std::string>::const_iterator it=def_.vals_.find(key);
    if(it !=def_.vals_.end())
        return it->second;

    if(mustExist)
    {
        throw MvException("No parameter <b>" + key + "</b> specified!");
    }

    static std::string emptyStr;
    return emptyStr;
}

void BufrFilterEngine::intValue(const std::string& key,
                                 const std::string& param,
                                 int minVal,int& outVal) const
{
    std::string val=value(key);

    if(val.empty() || val == "ANY")
        return;

    outVal=atoi(val.c_str());
    if(outVal < minVal)
    {
        throw MvException("Invalid value <b>" + val + "</b> specified for parameter <b>" + param + "</b>");
    }
}

void BufrFilterEngine::values(const std::string& key,std::vector<std::string>& valueVec) const
{
    valueVec.clear();

    std::map<std::string,std::string>::const_iterator it=def_.vals_.find(key);
    if(it !=def_.vals_.end())
    {
        std::string v=it->second;
        if(v.find("/") == std::string::npos)
        {
            valueVec.push_back(v);
            return;
        }
        else
        {
            std::vector<std::string> tmpVec;
            Tokenizer parseV("/");
            parseV(v,tmpVec);
            for(std::size_t i=0; i < tmpVec.size(); i++)
            {
                valueVec.push_back(metview::simplified(tmpVec[i]));
            }
        }
    }
}

void BufrFilterEngine::getIntValues(const std::string& key,
                                 const std::string& param,
                                 int minVal,std::vector<int>& outVec) const
{
    std::vector<std::string> vals;
    values(key,vals);

    if(vals.size() == 1 &&
       (vals[0].empty() || vals[0] == "ANY"))
        return;

    for(std::size_t i=0; i < vals.size(); i++)
    {
        if(vals[i].empty())
        {
            throw MvException("No value specified for parameter <b>" + param + "</b>");
        }

        int v=atoi(vals[i].c_str());
        if(v < minVal)
        {
            throw MvException("Invalid value <b>" + vals[i] + "</b> specified for parameter <b>" + param + "</b>");
        }

        outVec.push_back(v);
    }
}

void BufrFilterEngine::getDoubleValues(const std::string& key,
                                 const std::string& param,
                                 double minVal,std::vector<double>& outVec) const
{
    std::vector<std::string> vals;
    values(key,vals);

    if(vals.size() == 1 &&
       (vals[0].empty() || vals[0] == "ANY"))
        return;

    for(std::size_t i=0; i < vals.size(); i++)
    {
        if(vals[i].empty())
        {
            throw MvException("No value specified for parameter <b>" + param + "</b>");
        }

        double v=atof(vals[i].c_str());
        if(v < minVal)
        {
            throw MvException("Invalid value <b>" + vals[i] + "</b> specified for parameter <b>" + param + "</b>");
        }

        outVec.push_back(v);
    }
}

void BufrFilterEngine::getStringValues(const std::string& key,
                                 const std::string& param,
                                 std::vector<std::string>& outVec) const
{
    std::vector<std::string> vals;
    values(key,vals);

    if(vals.size() == 1 &&
       (vals[0].empty() || vals[0] == "ANY"))
        return;

    outVec=vals;
}

//===================================================
// Check parameters
//===================================================

bool BufrFilterEngine::parseDate(const std::string& val,int& year, int &month, int& day,std::string& err) const
{
    if(val.empty())
        return true;

    if(val.size()==8)
    {
        year=metview::fromString<int>(val.substr(0,4));
        month=metview::fromString<int>(val.substr(4,2));
        day=metview::fromString<int>(val.substr(6,2));

        if(year < 1000 || year > 9999)
        {
            err="Invalid year specified: <b>" + metview::toString(year) + "</b>";
            return false;
        }
        if(month < 1 || month > 12)
        {
            err="Invalid month specified: <b>" + metview::toString(month) + "</b>";;
            return false;
        }
        if(day < 1 || day > 31)
        {
            err="Invalid day specified: <b>" + metview::toString(day) + "</b>";;
            return false;
        }
        return true;
    }
    else
    {
        err="Invalid date: " + metview::toBold(val);
        return false;
    }

    return false;
}

bool BufrFilterEngine::parseTime(const std::string& inVal,int& hour,int& minute,int& second,std::string& err) const
{
    if(inVal.empty())
        return true;

    hour=-1;
    minute=-1;
    second=-1;

    std::string val=inVal;

    //hmmss
    if(val.size() == 5 )
        val="0" + val;
    //hhmm
    else if(val.size() == 4)
        val=val + "00";
    //hmm
    else if(val.size() == 3)
        val="0" + val + "00";
    //hh
    else if(val.size() == 2)
        val=val + "0000";
    //h
    else if(val.size() == 1)
        val="0" + val + "0000";
    //invalid len
    else if(val.size() != 6)
    {
        err="Invalid time: " + metview::toBold(val) + " Time can only contain up to 6 digits!";
        return false;
    }

    hour=metview::fromString<int>(val.substr(0,2));
    minute=metview::fromString<int>(val.substr(2,2));
    second=metview::fromString<int>(val.substr(4,2));
    if(!checkHour(hour,err)) return false;
    if(!checkMinute(minute,err)) return false;
    if(!checkSecond(second,err)) return false;

    return true;
}

bool BufrFilterEngine::checkHour(int h,std::string& err) const
{
    if(h < 0 || h > 24)
    {
        err="Invalid hour: " + metview::toBold(h);
        return false;
    }
    return true;
}

bool BufrFilterEngine::checkMinute(int m,std::string& err) const
{
    if(m <0  || m > 59)
    {
        err="Invalid minute: " + metview::toBold(m);
        return false;
    }
    return true;
}

bool BufrFilterEngine::checkSecond(int s,std::string& err) const
{
    if(s <0  || s > 59)
    {
        err="Invalid second: " + metview::toBold(s);
        return false;
    }
    return true;
}

bool BufrFilterEngine::parseTimeWindow(std::string& winVal,int& winSec,std::string& err) const
{
    winSec=0;
    if(winVal.empty())
        return true;

    winSec=metview::fromString<int>(winVal);
    if(winSec < 0)
    {
        err="Invalid time window: " + metview::toBold(winVal);
        return false;
    }
    winSec*=60;
    return true;
}

bool BufrFilterEngine::checkLon(float lon1,float lon2,std::string& err) const
{
    if(lon1 < -180)
    {
        err="Invalid longitude: " + metview::toBold(lon1);
        return false;
    }
    if(lon2 > 180)
    {
        err="Invalid longitude: " + metview::toBold(lon2);
        return false;
    }
    return true;
}

bool BufrFilterEngine::checkLat(float lat1,float lat2,std::string& err) const
{
    if(lat1 > 90)
    {
        err="Invalid latitude: " + metview::toBold(lat1);
        return false;
    }
    if(lat2 < -90)
    {
        err="Invalid latitude: " + metview::toBold(lat2);
        return false;
    }
    return true;
}

