/***************************** LICENSE START ***********************************

 Copyright 2012 ECMWF and INPE. This software is distributed under the terms
 of the Apache License version 2.0. In applying this license, ECMWF does not
 waive the privileges and immunities granted to it by virtue of its status as
 an Intergovernmental Organization or submit itself to any jurisdiction.

 ***************************** LICENSE END *************************************/

// ObsFilter.cc, vk & br oct-94, rev vk 970729
//               qa-c++ vk 960725


#include "inc_iostream.h"
#include <time.h>
#include <vector>

#include "Metview.h"
#include "MvObs.h"
#include "MvException.h"

const char cTab   = 9;  // ascii code for 'tab'
const char cComma = ',';

const char cVectorPolar[] = "POLAR_VECTOR";
const char cVectorUV[]    = "XY_VECTOR";

const std::string TEMP_2M = "airTemperatureAt2M";  //12004;

enum eLevel
{
    LVL_Surface,
    LVL_SingleLevel,
    LVL_ThicknessLayer,
    LVL_Occurrence,
    LVL_CoordinateValue,
    LVL_CoordinateRange
};

enum eOutput
{
    eBUFR,
    eGEOPOINTS,
    ePOLAR_VECTOR,
    eXY_VECTOR,
    eCSV,
    eNCOLS
};
string outTypeStr[] = {"BUFR", "GEOPOINTS", "POLAR_VECTOR", "XY_VECTOR", "CSV", "NCOLS"};

//__________________________________________________________ ObsFilter

class ObsFilter : public MvService
{
private:
    ObsFilter(const ObsFilter& anOther);
    void operator=(const ObsFilter& anOther);

public:
    ObsFilter();
    ~ObsFilter();

    void serve(MvRequest&, MvRequest&);

protected:
    void getParams(void);
    void getData(void);
    void getParameterSpecifiers(void);
    void getTypes(void);
    void getTimes(void);
    void getWmoIds(void);
    void getLocation(void);
    void getCustom(void);
    MvRequest createRequest();
    void add();
    void close(void);
    float surfaceValue(long parInd);
    float pressureLevelValue(float level, long parInd);
    float layerValue(float level1, float level2, long parInd);
    float occurranceValue(int occurranceInd, long parIndex);
    float coordinateLevelValue(const std::string&, float, int);
    float coordinateRangeValue(const std::string&, float, float, int);

private:
    long _counter;
    MvRequest _in;
    MvRequest _data;
    MvObs obs_;
    MvObsSet* _inSet;
    MvObsSetIterator* _iter;
    eOutput outType_;
    MvObsSet* _outBufr;
    ofstream _outFile;
    eLevel _level;
    float _firstLevel;
    float _secondLevel;
    long _occurrenceIndex;
    vector<std::string> param_;
    ENextReturn _obsOrMsg;  //-- for multisubset msgs (filtering also with subset data?)
    boolean _includeMissingValues;
    fortfloat _missingValue;
    bool failOnError_;
    bool failOnEmptyOutput_;
    std::string _coordLevelDescriptor;
    bool hasAttributeCondition_;
};


//_________________________________________________________

ObsFilter::ObsFilter() :
    MvService("OBSFILTER"),
    _counter(0),
    _inSet(0),
    _iter(0),
    _outBufr(0),
    _level(LVL_Surface),
    _firstLevel(0.),
    _secondLevel(0.),
    _occurrenceIndex(0),
    _obsOrMsg(NR_returnMsg),  //-- as a default do not study subsets (obs's)
    _includeMissingValues(false),
    _missingValue(kBufrMissingValue),
    failOnError_(true),  //-- this breaks backwards compatibility
    failOnEmptyOutput_(false),
    hasAttributeCondition_(false)
{
}
//_________________________________________________________

ObsFilter::~ObsFilter()
{
    delete _iter;
    delete _inSet;
    delete _outBufr;
    obs_.clear();
}
//_________________________________________________________
void ObsFilter ::getData(void)
{
    //-- D A T A --
    _in.getValue(_data, "DATA");

    _inSet = new MvObsSet(_data);
    _iter  = new MvObsSetIterator(*_inSet);

    //-- O U T P U T --
    outType_ = (eOutput)((int)_in("OUTPUT"));

    if (strcmp(_in("FAIL_ON_ERROR"), "NO") == 0)
        failOnError_ = false;
    else
        failOnError_ = true;  // default value

    if (strcmp(_in("FAIL_ON_EMPTY_OUTPUT"), "YES") == 0)
        failOnEmptyOutput_ = true;
    else
        failOnEmptyOutput_ = false;  // default value
}
//_________________________________________________________
void ObsFilter ::getParameterSpecifiers(void)
{
    //-- P A R A M E T E R --
    int nrParams = _in.countValues("PARAMETER");

    param_.clear();
    if (nrParams == 0)  //-- first descriptor (parameter)
        param_.push_back(TEMP_2M);
    else
        param_.push_back((const char*)_in("PARAMETER"));

    if (nrParams > 1)  //-- if more descriptors (parameters)
    {
        for (int n = 1; n < nrParams; ++n)
            param_.push_back((const char*)_in("PARAMETER", n));
    }
    if ((outType_ == ePOLAR_VECTOR || outType_ == eXY_VECTOR) && nrParams < 2)
        throw MvException("ObsFilter-> Too few parameters defined. Wind data takes two parameters!!");

    const char* myMissingData = _in("MISSING_DATA");
    if (myMissingData && strcmp(myMissingData, "INCLUDE") == 0) {
        _includeMissingValues = true;
        _missingValue         = (double)_in("MISSING_DATA_VALUE");
    }
    else {
        _includeMissingValues = false;  // default values
        _missingValue         = kBufrMissingValue;
    }

    //Check if there are attributes in params
    for (size_t i = 0; i < param_.size(); i++) {
        if (param_[i].find("->") != std::string::npos) {
            hasAttributeCondition_ = true;
            break;
        }
    }

    //-- L E V E L --
    const char* levelStr = _in("LEVEL");
    if (levelStr && strcmp(levelStr, "SINGLE") == 0)
        _level = LVL_SingleLevel;
    else if (levelStr && strcmp(levelStr, "THICKNESS") == 0)
        _level = LVL_ThicknessLayer;
    else if (levelStr && strcmp(levelStr, "OCCURRENCE") == 0)
        _level = LVL_Occurrence;
    else if (levelStr && strcmp(levelStr, "DESCRIPTOR_VALUE") == 0)
        _level = LVL_CoordinateValue;
    else if (levelStr && strcmp(levelStr, "DESCRIPTOR_RANGE") == 0)
        _level = LVL_CoordinateRange;
    else
        _level = LVL_Surface;  // default value

    if (_in.countValues("FIRST_LEVEL") > 0)
        _firstLevel = (double)_in("FIRST_LEVEL");
    else
        _firstLevel = 0.;  //default value

    if (_in.countValues("SECOND_LEVEL") > 0)
        _secondLevel = (double)_in("SECOND_LEVEL");
    else
        _secondLevel = 0.;  //default value

    if (_in.countValues("OCCURRENCE_INDEX") > 0)
        _occurrenceIndex = (long)_in("OCCURRENCE_INDEX");
    else
        _occurrenceIndex = 0L;  //default value

    if (_in.countValues("LEVEL_DESCRIPTOR") > 0)
        _coordLevelDescriptor = (const char*)_in("LEVEL_DESCRIPTOR");
    else
        _coordLevelDescriptor = "";  //0L;  //default value

    //Check if there are attributes in _coordLevelDescriptor

    if (_coordLevelDescriptor.find("->") != std::string::npos) {
        hasAttributeCondition_ = true;
    }
}
//_________________________________________________________
void ObsFilter ::getTypes(void)
{
    int i;

    // It requires that this parameter returns number(s)
    const char* obsTypeStr = (const char*)_in("OBSERVATION_TYPES");
    if (obsTypeStr) {
        long myType = -999;
        for (i = 0; i < _in.countValues("OBSERVATION_TYPES"); i++) {
            myType = -999;
            if ((myType = (long)_in("OBSERVATION_TYPES", i)) >= 0)
                _iter->setMessageType((int)myType);
        }
    }

    // It requires that this parameter returns number(s)
    const char* obsSubTypeStr = (const char*)_in("OBSERVATION_SUBTYPES");
    if (obsSubTypeStr) {
        long mySubType = -999;
        for (i = 0; i < _in.countValues("OBSERVATION_SUBTYPES"); i++) {
            mySubType = -999;
            if ((mySubType = (long)_in("OBSERVATION_SUBTYPES", i)) >= 0)
                _iter->setMessageSubtype((int)mySubType);
        }
    }
}
//_________________________________________________________
void ObsFilter ::getTimes(void)
{
    const char* obsTimeStr = _in("DATE_AND_TIME_FROM");
    if (obsTimeStr && strcmp(obsTimeStr, "DATA") == 0)
        _iter->useObsTime(true);

    const char* dateStr = _in("DATE");
    const char* timeStr = _in("TIME");
    if (dateStr && strcmp(dateStr, "ANY") == 0 &&
        timeStr && strcmp(timeStr, "ANY") == 0) {
        // both DATE and TIME are ANY
    }  // TIME==ANY && DATE==value is illegal
    else {
        int myTime  = (int)_in("TIME");
        int myDelta = (int)_in("RESOLUTION_IN_MINS");

        if (dateStr && strcmp(dateStr, "ANY") == 0)  // only TIME is given
        {
            myDelta = 100 * (myDelta / 60) + myDelta % 60;  // minutes into HHMM
            if ((myDelta % 100) != 0)
                _iter->setTimeRangeWithoutDate(myTime - myDelta - 40  // 40=100-60!!!
                                               ,
                                               myTime + myDelta);
            else
                _iter->setTimeRangeWithoutDate(myTime - myDelta, myTime + myDelta);
        }
        else  // both TIME and DATE given
        {
            long myDate = (int)_in("DATE");
            TDynamicTime myObsTime(myDate, myTime);
            _iter->setTimeRange(myObsTime, myDelta);
        }
    }
}
//_________________________________________________________
void ObsFilter ::getWmoIds(void)
{
    int i;

    const char* blockStr = _in("WMO_BLOCKS");
    if (blockStr && strcmp(blockStr, "ANY") == 0) {
    }
    else {
        long myWmoBlock;
        for (i = 0; i < _in.countValues("WMO_BLOCKS"); i++) {
            if ((myWmoBlock = (long)_in("WMO_BLOCKS", i)) > 0)
                _iter->setWmoBlock((int)myWmoBlock);
        }
        _obsOrMsg = NR_returnObs;  //-- have to study each subset (obs)
    }

    const char* stationStr = _in("WMO_STATIONS");
    if (stationStr && strcmp(stationStr, "ANY") == 0) {
    }
    else {
        long myWmoStation;
        for (i = 0; i < _in.countValues("WMO_STATIONS"); i++)  // int i; ????
        {
            if ((myWmoStation = (long)_in("WMO_STATIONS", i)) > 0)
                _iter->setWmoStation(myWmoStation);
        }
        _obsOrMsg = NR_returnObs;  //-- have to study each subset (obs)
    }
}
//_________________________________________________________
void ObsFilter ::getLocation(void)
{
    const char* locFilterStr = _in("LOCATION_FILTER");
    if (locFilterStr && strcmp(locFilterStr, "AREA") == 0) {
        const char* areaStr = _in("AREA");
        if (areaStr) {
            if (_in.countValues("AREA") == 4) {
                float y1 = (double)_in("AREA", 0);
                float x1 = (double)_in("AREA", 1);
                float y2 = (double)_in("AREA", 2);
                float x2 = (double)_in("AREA", 3);
                MvLocation loc1(y1, x1);
                MvLocation loc2(y2, x2);
                _iter->setArea(loc1, loc2);
                _obsOrMsg = NR_returnObs;  //-- have to study each subset (obs)
            }
            else
                sendProgress("ObsFilter: Errors in AREA-parameter, Area filter not set!");
        }
    }


    if (locFilterStr && strcmp(locFilterStr, "LINE") == 0) {
        const char* lineStr = _in("LINE");
        if (lineStr) {
            if (_in.countValues("LINE") == 4) {
                float myDeltaInM = (double)_in("DELTA_IN_KM") * 1000;
                float y1         = (double)_in("LINE", 0);
                float x1         = (double)_in("LINE", 1);
                float y2         = (double)_in("LINE", 2);
                float x2         = (double)_in("LINE", 3);
                MvLocation loc1(y1, x1);
                MvLocation loc2(y2, x2);
                _iter->setXSectionLine(loc1, loc2, myDeltaInM);
                _obsOrMsg = NR_returnObs;  //-- have to study each subset (obs)
            }
            else
                sendProgress("ObsFilter: Errors in LINE-parameter, Line filter not set!");
        }
    }
}
//_________________________________________________________
void ObsFilter ::getCustom(void)
{
    const char* customFilterStr  = _in("CUSTOM_FILTER");
    std::string customDescriptor = "";
    if ((const char*)_in("CUSTOM_PARAMETER"))
        customDescriptor = (const char*)_in("CUSTOM_PARAMETER");
    float customFirstValue  = 0;
    float customSecondValue = 0;
    int mySelectValueCount  = _in.countValues("CUSTOM_VALUES");
    if (mySelectValueCount > 0)
        customFirstValue = (double)_in("CUSTOM_VALUES", 0);
    if (mySelectValueCount > 1)
        customSecondValue = (double)_in("CUSTOM_VALUES", 1);

    if (customFilterStr && strcmp(customFilterStr, "VALUE") == 0) {
        for (int i = 0; i < mySelectValueCount; i++) {
            _iter->select(customDescriptor, (double)_in("CUSTOM_VALUES", i));
        }
        _obsOrMsg = NR_returnObs;  //-- have to study each subset (obs)
    }

    if (customFilterStr && strcmp(customFilterStr, "RANGE") == 0) {
        if (mySelectValueCount == 2) {
            _iter->selectRange(customDescriptor, customFirstValue, customSecondValue);
            _obsOrMsg = NR_returnObs;  //-- have to study each subset (obs)
        }
        else
            throw MvException("ObsFilter-> Custom Filter/Range: number of CUSTOM_VALUES != 2 !!!");
    }

    if (customFilterStr && strcmp(customFilterStr, "EXCLUDE") == 0) {
        if (mySelectValueCount == 2) {
            _iter->excludeRange(customDescriptor, customFirstValue, customSecondValue);
            _obsOrMsg = NR_returnObs;  //-- have to study each subset (obs)
        }
        else
            throw MvException("ObsFilter-> Custom Filter/ExcludeRange: number of CUSTOM_VALUES != 2");
    }

    //Check if there are attributes in custom param
    if (customDescriptor.find("->") != std::string::npos) {
        hasAttributeCondition_ = true;
    }
}
//_________________________________________________________
void ObsFilter ::getParams(void)
{
    _obsOrMsg = NR_returnMsg;  //-- set msg level as default

    //-- these have no effect on _obsOrMsg --
    getData();
    getParameterSpecifiers();
    getTypes();

    //-- these may have an effect on _obsOrMsg --
    getTimes();
    getWmoIds();
    getLocation();
    getCustom();

    if (outType_ == eBUFR) {
        if (_obsOrMsg == NR_returnObs)
            sendProgress("ObsFilter-> Filtering with data values requested, have to study each subset (obs)");
        else
            sendProgress("ObsFilter-> Filtering on msg level (only section header info used)");
    }
    else
        _obsOrMsg = NR_returnObs;  //-- for geopoints always on obs level!!!
}
//_________________________________________________________
MvRequest
ObsFilter ::createRequest()
{
    switch (outType_) {
        case eBUFR:  //-- BUFR
        {
            MvRequest x("BUFR");  //-- BUFR return request
            x("TEMPORARY") = 1;
            x("PATH")      = marstmp();
            _outBufr       = new MvObsSet(x, "w");

            return x;
        }

        case eCSV:  //-- CSV
        {
            char* fileName = marstmp();

            MvRequest x("TABLE");  //-- CSV return request (TABLE)
            x("TEMPORARY") = 1;
            x("PATH")      = fileName;

            _outFile.open(fileName, ios::out);
            //-- construct CSV header line
            for (unsigned int p = 0; p < param_.size(); ++p) {
                if (p > 0)
                    _outFile << cComma;
                _outFile << "'" << param_[p] << "'";
            }
            _outFile << endl;

            return x;
        }

        case eGEOPOINTS:
        case ePOLAR_VECTOR:
        case eXY_VECTOR: {
            char* fileName = marstmp();

            MvRequest x("GEOPOINTS");  //-- GEOPOINTS return request
            x("TEMPORARY") = 1;
            x("PATH")      = fileName;

            _outFile.open(fileName, ios::out);

            _outFile << "#GEO" << endl;  //-- construct GEOPOINTS header

            if (outType_ == ePOLAR_VECTOR || outType_ == eXY_VECTOR)
                _outFile << "#FORMAT " << outTypeStr[outType_] << endl;

            if (outType_ == eGEOPOINTS)
                _outFile << "PARAMETER = " << param_[0] << endl;
            else
                _outFile << "PARAMETER = " << param_[0] << cTab << param_[1] << endl;

            _outFile << "#lat" << cTab << "long" << cTab
                     << "level" << cTab << "date" << cTab
                     << "time" << cTab;
            if (outType_ == eGEOPOINTS)
                _outFile << "value";
            else
                _outFile << "speed|u" << cTab << "direction|v";
            _outFile << endl;

            _outFile << "#DATA" << endl;

            return x;
        }

        case eNCOLS: {
            char* fileName = marstmp();

            MvRequest x("GEOPOINTS");
            x("TEMPORARY") = 1;
            x("PATH")      = fileName;

            _outFile.open(fileName, ios::out);

            _outFile << "#GEO" << endl;
            _outFile << "#FORMAT NCOLS" << endl;

            _outFile << "#COLUMNS " << endl
                     << "stnid" << cTab << "latitude" << cTab << "longitude" << cTab
                     << "level" << cTab << "date" << cTab
                     << "time" << cTab;

            for (size_t p = 0; p < param_.size(); p++) {
                _outFile << param_[p] << cTab;
            }

            _outFile << endl
                     << "#DATA" << endl;

            return x;
        }
    }

    MvRequest x("ERROR");  //-- internal error if here
    return x;
}
//_________________________________________________________
float ObsFilter::surfaceValue(long parInd)
{
    return obs_.valueC(param_[parInd]);
}
//_________________________________________________________
float ObsFilter::pressureLevelValue(float level, long parInd)
{
    return obs_.valueByPressureLevelC(level, param_[parInd]);
}
//_________________________________________________________
float ObsFilter::layerValue(float level1, float level2, long parInd)
{
    return obs_.valueByLayerC(level1, level2, param_[parInd]);
}
//_________________________________________________________
float ObsFilter::occurranceValue(int occurranceInd, long parIndex)
{
    return obs_.valueByOccurrenceC(occurranceInd, param_[parIndex]);
}
//_________________________________________________________
float ObsFilter::coordinateLevelValue(const std::string& coordDesc, float level, int parInd)
{
    return obs_.valueByLevelC(coordDesc, level, param_[parInd]);
}
//_________________________________________________________
float ObsFilter::coordinateRangeValue(const string& coordDesc, float level1, float level2, int parInd)
{
    return obs_.valueByLevelRangeC(coordDesc, level1, level2, param_[parInd]);
}

//_________________________________________________________
void ObsFilter ::add()
{
    // Expanded message if it is needed
    if (outType_ != eBUFR)
        obs_.expand();

    if (outType_ == eBUFR)  //-- BUFR
    {
        if (_counter == 0)  //-- BUFR: first time
        {
            //-- changed 2011-12-30/vk:
            //--  o target BUFR file is set to contain single subsets, but this is in effect
            //--     only when filtering using data values from BUFR Section 4
            //--  o this setting has no effect when filtering only with metadata values in
            //--     BUFR Section 1; in such cases multisubset messages are copied as they
            //--     are (metadata is common to all subsets in a multisubset msg)
            _outBufr->setSubsetMax(1);  //-- obs_.msgSubsetCount() );
        }

        //-- if filtering requires values from observation then multisubset
        //-- messages have to be filtered on observation (subset) level !
        if (_obsOrMsg == NR_returnMsg) {
            _outBufr->write(obs_);              //-- BUFR: filtering with "meta data" only
            _counter += obs_.msgSubsetCount();  //-- BUFR: i.e. write the whole message!
        }
        else {
            _outBufr->add(obs_);  //-- BUFR: filtering also with data values
            _counter++;           //-- BUFR: write current subset only!
        }
    }
    else if (outType_ == eCSV)  //-- CSV
    {
        ostringstream os;
        bool abortOnMissing = false;
        for (unsigned int p = 0; p < param_.size(); ++p) {
            if (p > 0)
                os << cComma;

            float val;
            switch (_level) {
                case LVL_Surface:
                    val = surfaceValue(p);
                    break;
                case LVL_SingleLevel:
                    val = pressureLevelValue(_firstLevel, p);
                    break;
                case LVL_ThicknessLayer:
                    val = layerValue(_firstLevel, _secondLevel, p);
                    break;
                case LVL_Occurrence:  //-- CSV: this does not make much sence here !!!
                    val = occurranceValue(_occurrenceIndex, p);
                    break;
                case LVL_CoordinateValue:
                    val = coordinateLevelValue(_coordLevelDescriptor, _firstLevel, p);
                    break;
                case LVL_CoordinateRange:
                    val = coordinateRangeValue(_coordLevelDescriptor, _firstLevel, _secondLevel, p);
                    break;
                default:
                    break;
            }

            if (obs_.elementValueType(param_[0]) == CODES_TYPE_STRING) {
                //-- string value has only basic retrieve...
                os << obs_.stringValue(param_[0]);
            }
            else {
                if (val == kBufrMissingValue)  //-- CSV: missing value?
                {
                    if (_includeMissingValues)
                        val = _missingValue;  //-- CSV: chage to user defined missing value
                    else {
                        abortOnMissing = true;  //-- CSV: force ignoring the whole obs
                        break;
                    }
                }

                os << val;  //-- CSV: write value or user defined missing value
            }
        }

        if (!abortOnMissing) {
            _outFile << os.str() << endl;
            _counter++;
        }
    }

    //traditional geopoints
    else if (outType_ != eNCOLS) {
        float myValue = 0;
        float myDirec = 0;  //-- for geovectors

        switch (_level) {
            case LVL_Surface:
                myValue = surfaceValue(0);
                if (outType_ > eGEOPOINTS)
                    myDirec = surfaceValue(1);
                break;

            case LVL_SingleLevel:
                myValue = pressureLevelValue(_firstLevel, 0);
                if (outType_ > eGEOPOINTS)
                    myDirec = pressureLevelValue(_firstLevel, 1);
                break;

            case LVL_ThicknessLayer:
                myValue = layerValue(_firstLevel, _secondLevel, 0);
                if (outType_ > eGEOPOINTS)
                    myDirec = layerValue(_firstLevel, _secondLevel, 1);
                break;

            case LVL_Occurrence:
                myValue = occurranceValue(_occurrenceIndex, 0);
                if (outType_ > eGEOPOINTS)
                    myDirec = occurranceValue(_occurrenceIndex, 1);
                break;

            case LVL_CoordinateValue:
                myValue = coordinateLevelValue(_coordLevelDescriptor, _firstLevel, 0);
                if (outType_ > eGEOPOINTS)
                    myDirec = coordinateLevelValue(_coordLevelDescriptor, _firstLevel, 1);
                break;

            case LVL_CoordinateRange:
                myValue = coordinateRangeValue(_coordLevelDescriptor, _firstLevel, _secondLevel, 0);
                if (outType_ > eGEOPOINTS)
                    myDirec = coordinateRangeValue(_coordLevelDescriptor, _firstLevel, _secondLevel, 1);
                break;

            default:
                break;
        }

        if (myValue != kBufrMissingValue || _includeMissingValues) {
            MvLocation myLoc   = obs_.location();
            TStaticTime myTime = obs_.obsTime();

            _outFile << myLoc.latitude() << cTab;
            _outFile << myLoc.longitude() << cTab;

            _outFile << _firstLevel << cTab;

            _outFile << myTime.GetYear();
            _outFile.width(2);
            _outFile.fill('0');
            _outFile << myTime.GetMonth();
            _outFile.width(2);
            _outFile.fill('0');
            _outFile << myTime.GetDay() << cTab;
            _outFile.width(2);
            _outFile.fill('0');
            _outFile << myTime.GetHour();
            _outFile.width(2);
            _outFile.fill('0');
            _outFile << myTime.GetMin() << cTab;

            if (obs_.elementValueType(param_[0]) == CODES_TYPE_STRING) {
                //-- string value has only basic retrieve...
                _outFile << obs_.stringValue(param_[0]);
            }
            else {
                if (_includeMissingValues && myValue == kBufrMissingValue)
                    myValue = _missingValue;

                _outFile << myValue;

                if (outType_ > eGEOPOINTS)
                    _outFile << cTab << myDirec;  //-- should we check against missing value?
            }

            _outFile << endl;

            _counter++;
        }
    }

    // Multi column geopoints
    else {
        std::vector<float> vals(param_.size(), 0);

        for (size_t p = 0; p < param_.size(); p++) {
            float val = kBufrMissingValue;
            switch (_level) {
                case LVL_Surface:
                    val = surfaceValue(p);
                    break;
                case LVL_SingleLevel:
                    val = pressureLevelValue(_firstLevel, p);
                    break;
                case LVL_ThicknessLayer:
                    val = layerValue(_firstLevel, _secondLevel, p);
                    break;
                case LVL_Occurrence:
                    val = occurranceValue(_occurrenceIndex, p);
                    break;
                case LVL_CoordinateValue:
                    val = coordinateLevelValue(_coordLevelDescriptor, _firstLevel, p);
                    break;
                case LVL_CoordinateRange:
                    val = coordinateRangeValue(_coordLevelDescriptor, _firstLevel, _secondLevel, p);
                    break;
                default:
                    break;
            }

            vals[p] = val;

            if (val == kBufrMissingValue && !_includeMissingValues)
                return;
        }

        MvLocation myLoc   = obs_.location();
        TStaticTime myTime = obs_.obsTime();
        std::string ident =  obs_.headerIdent();
        if (ident.empty())
            ident = "?";

        _outFile << ident << cTab;
        _outFile << myLoc.latitude() << cTab;
        _outFile << myLoc.longitude() << cTab;

        _outFile << _firstLevel << cTab;

        _outFile << myTime.GetYear();
        _outFile.width(2);
        _outFile.fill('0');
        _outFile << myTime.GetMonth();
        _outFile.width(2);
        _outFile.fill('0');
        _outFile << myTime.GetDay() << cTab;
        _outFile.width(2);
        _outFile.fill('0');
        _outFile << myTime.GetHour();
        _outFile.width(2);
        _outFile.fill('0');
        _outFile << myTime.GetMin() << cTab;


        for (size_t p = 0; p < param_.size(); p++) {
            if (obs_.elementValueType(param_[p]) == CODES_TYPE_STRING) {
                //-- string value has only basic retrieve...
                _outFile << obs_.stringValue(param_[p]);
            }
            else {
                if (_includeMissingValues && vals[p] == kBufrMissingValue)
                    vals[p] = _missingValue;

                _outFile << vals[p];
            }

            if (p < param_.size() - 1)
                _outFile << cTab;
        }

        _outFile << endl;
        _counter++;
    }

    obs_.clearIterator();
}

//_________________________________________________________
void ObsFilter ::close(void)
{
    if (outType_ == eBUFR)
        _outBufr->close();
    else
        _outFile.close();
}

//_________________________________________________________
void ObsFilter ::serve(MvRequest& in, MvRequest& out)
{
    cout << "ObsFilter::serve() in" << endl;
    in.print();

    time_t myStartTime;
    time(&myStartTime);
    clock_t myStartCpu = clock();

    // The input request needs to be expanded with flag EXPAND_LAST_NAME.
    // This is required by functions getTypes and getCustom.
    _in = in.ExpandRequest("ObsFilterDef", "ObsFilterRules", EXPAND_LAST_NAME);

    getParams();  //-- new _inSet & _iter

    //enables/disables eccodes optimisation
    _inSet->setUseSkipExtraAttributes(!hasAttributeCondition_);

    //we turn off caching compressed data - having it on does not improve performance for
    //obsfilter
    _inSet->setCacheCompressedData(false);

    MvRequest x = createRequest();
    _counter    = 0;  //-- m a i n   l o o p
    try {
        while ((obs_ = (*_iter)(_obsOrMsg))) {
            add();
        }
        close();
        obs_.clear();  //-- initialise MvObs (serve function may be call iteractively
    }
    catch (MvException& e) {
        string err = "ObsFilter-> Failed: ";
        err += e.what();
        if (failOnError_)
            throw MvException(err.c_str());
        else
            setError(0, err.c_str());
    }

    time_t myEndTime;
    time(&myEndTime);
    double wallClockTime = difftime(myEndTime, myStartTime);
    clock_t myEndCpu     = clock();
    double cpuTime       = ((double)(myEndCpu - myStartCpu)) / (double)CLOCKS_PER_SEC;  // CLK_TCK;

    sendProgress("ObsFilter-> Filtered %d observations (out of %d) into a %s file in %d secs (cpu %d secs)", _counter, _inSet->obsCount(), outTypeStr[(int)outType_].c_str(), (int)wallClockTime, (int)cpuTime);

    delete _iter;
    _iter = 0;

    delete _inSet;
    _inSet = 0;

    if (_counter == 0 && failOnEmptyOutput_)
        throw MvException("ObsFilter-> Abort: Fail-On-Empty-Output requested!");

    out = x;

    cout << "ObsFilter::serve() out" << endl;
    out.print();
}
//_________________________________________________________

int main(int argc, char** argv)
{
    MvApplication theApp(argc, argv);
    ObsFilter filter;

    theApp.run();
}
