/***************************** LICENSE START ***********************************

 Copyright 2012 ECMWF and INPE. This software is distributed under the terms
 of the Apache License version 2.0. In applying this license, ECMWF does not
 waive the privileges and immunities granted to it by virtue of its status as
 an Intergovernmental Organization or submit itself to any jurisdiction.

 ***************************** LICENSE END *************************************/

// BufrPicker.cc, Jul2011/vk, based on ObsFilter by vk & br oct-94, rev vk 970729
//
//  Picks single or vector values from BUFR messages into a geopoints file.
//  'BufrPicker' is targeted for complicated BUFR messages where the requested
//  value(s) occur several times, with different coordinate values.

//#include "inc_iostream.h"
//#include <time.h>

//#include "mars.h"
#include "Metview.h"
#include "MvObs.h"
#include "MvException.h"

const char mySeparator = '\t';             //-- 'tab'

//const unsigned long BRIGHT_TEMP = 12163;   //-- default value to be picked
const std::string BRIGHT_TEMP = "brightnessTemperature";

//__________________________________________________________ BufrPicker

class BufrPicker : public MvService
{
 private:
   BufrPicker( const BufrPicker& );
   void operator= ( const BufrPicker& );

 public:
   BufrPicker();
   ~BufrPicker();

   void serve( MvRequest&, MvRequest& );

 protected:
   void getParams();
   void getData();
   void getParameterSpecifiers();
   void pickValues( MvObs& );
   void pickValues_compressedData( MvObs& );
   void pickValues_uncompressedData( MvObs& );
   void add( MvObs&, double );
   void close();
   void clear();
   void convertDescriptorsToKeys(MvObs&);
   MvRequest createRequest();

 private:
   long                     counter_;
   MvRequest                in_;                   //-- input request
   MvRequest                data_;                 //-- sub-request of input request
   MvObsSet*                inSet_;                //-- input observation set
   MvObsSetIterator*        iter_;                 //-- input iterator (no filtering)
   ofstream                 outfile_;              //-- output geopoints file
   std::string              param_;                //-- value to be picked from requested "coordinates"
   std::string              param2_;               //-- second value, in case of geovector output
   std::vector<std::string> coordDescr_;           //-- coordinate descriptors
   std::vector<double>      coordValue_;           //-- values for the given coordinates
   int                      coordDescrCount_;      //-- number of coord descrs
   int                      geoVectorType_;        //-- 0:no vector, 1:polar_vector, 2:xy_vector
   ENextReturn              obsOrMsg_;             //-- for multisubset msgs (pick only the first subset)
   boolean                  includeMissingValues_;
   fortfloat                missingValue_;
   bool                     failOnError_;
   double                   pickedValue1_;
   double                   pickedValue2_;
   bool                     pickAll_;              //-- pick all values related to the innermost coordinate descriptor
   std::vector<std::string> geoVectorName_;        //-- [0]:empty, [1]:"POLAR_VECTOR", [2]:"XY_VECTOR"
};


//_________________________________________________________Constructor
BufrPicker::BufrPicker() : MvService( "BUFRPICKER" ),
   counter_( 0 ),
   inSet_( 0 ),
   iter_( 0 ),
   param_( BRIGHT_TEMP ),
   param2_( "" ),
   coordDescrCount_( 0 ),
   geoVectorType_( 0 ),
   obsOrMsg_( NR_returnMsg ),   //-- as a default pick the first subset from the next BUFR message //all subsets (obs's)
   includeMissingValues_( false ),
   missingValue_( kBufrMissingValue ),
   failOnError_(true),         //-- this breaks backwards compatibility
   pickedValue1_(kBufrMissingValue),
   pickedValue2_(kBufrMissingValue),
   pickAll_(true)
{
   geoVectorName_.push_back("");
   geoVectorName_.push_back("POLAR_VECTOR");
   geoVectorName_.push_back("XY_VECTOR");
}

//_________________________________________________________Destructor
BufrPicker::~BufrPicker()
{
   clear();
}

//_________________________________________________________clear
void
BufrPicker::clear()
{
   if ( iter_ )
   {
      delete iter_;
      iter_ = 0;
   }
   if ( inSet_ )
   {
      delete inSet_;
      inSet_ = 0;
   }

   coordDescr_.clear();
   coordValue_.clear();
   geoVectorName_.clear();
}

//_________________________________________________________getData
void
BufrPicker::getData()
{
   //-- D A T A --
   in_.getValue( data_, "DATA" );
   //data_.print();

   inSet_ = new MvObsSet( data_ );
   iter_  = new MvObsSetIterator( *inSet_ );

   //-- O U T P U T --
   const char* outStr = in_( "OUTPUT" );
   if ( !outStr )
      geoVectorType_ = 0;
   else if( std::string(outStr) == geoVectorName_[1] )
      geoVectorType_ = 1;
   else if( std::string(outStr) == geoVectorName_[2] )
      geoVectorType_ = 2;

  const char* fail = in_("FAIL_ON_ERROR");
  if( fail && strcmp( fail, "NO" ) == 0 )
     failOnError_ = false;
}

//_________________________________________________________getParameterSpecifiers
void
BufrPicker::getParameterSpecifiers()
{
  //-- P A R A M E T E R --

  if( in_.countValues( "PARAMETER" ) > 0 )
    param_ = (const char*)in_( "PARAMETER" );

  if( geoVectorType_ > 0 && in_.countValues( "PARAMETER" ) > 1 )
    param2_ = (const char*)in_( "PARAMETER", 1 );

  const char* myMissingData = in_( "MISSING_DATA" );
  if( myMissingData && strcmp( myMissingData, "INCLUDE" ) == 0  )
  {
      includeMissingValues_ = true;
      missingValue_ = (double)in_( "MISSING_DATA_VALUE" );
  }

  //-- Check coordinate descriptors and coordinate values

  coordDescrCount_ = in_.countValues( "COORDINATE_DESCRIPTORS" );
  if( coordDescrCount_ < 1 )
     throw MvException( "No coordinate descriptors given" );

  if( in_.countValues( "COORDINATE_VALUES" ) != coordDescrCount_ )
     throw MvException( "Different number of coordinate descriptors and values" );

  //-- OK, now get coordinate descriptors and coordinate values

  pickAll_ = false;
  int innermostCoordIndex = coordDescrCount_ - 1;
  for( int i=0; i < coordDescrCount_; ++i )
  {
     coordDescr_.push_back( (const char*)in_( "COORDINATE_DESCRIPTORS", i ) );

     string coordValStr = (const char*)in_( "COORDINATE_VALUES", i );  //-- to check for "ALL"

     if( coordValStr == "ALL" && i < innermostCoordIndex )       //-- 'ALL' not allowed for outer coords
     {
        marslog( LOG_EROR, "Value ALL allowed _only_ for the innermost coordinate descriptor" );
     }
     else if( coordValStr == "ALL" && i == innermostCoordIndex ) //-- 'ALL' allowed for the innermost
     {
        pickAll_ = true;
        coordValue_.push_back( -9999.99 );                       //-- dummy value
     }
     else                                                        //-- OK for rest of the cases
        coordValue_.push_back( (double)in_( "COORDINATE_VALUES", i ) );
  }
}

//_________________________________________________________getParams
void
BufrPicker::getParams()
{
  getData();
  getParameterSpecifiers();
}

//_________________________________________________________createRequest
MvRequest
BufrPicker::createRequest()
{
   char* fileName = marstmp();
   MvRequest x( "GEOPOINTS" );
   x( "TEMPORARY" ) = 1;
   x( "PATH" ) = fileName;

   outfile_.open( fileName, ios::out );

   outfile_ << "#GEO" << endl;

   if( geoVectorType_ > 0 )
      outfile_ << "#FORMAT " << geoVectorName_[geoVectorType_] << endl;

   if ( geoVectorType_ > 0 )
      outfile_ << "PARAMETER = " << param_ << mySeparator << param2_ << endl;
   else
      outfile_ << "PARAMETER = " << param_ << endl;

   outfile_ << "#lat" << mySeparator << "long" << mySeparator
            << "level" << mySeparator << "date" << mySeparator
            << "time" << mySeparator;

   if( geoVectorType_ > 0 )
      outfile_ << "speed|u" << mySeparator << "direction|v";
   else
      outfile_ << "value";

   outfile_ << endl;

   outfile_ << "#DATA" << endl;

   return x;
}

//_________________________________________________________pickValue
//
// Note: 'void' function - failure is indicated by missing value(s)
// because user may have requested to return missing values as well.
//
void
BufrPicker::pickValues( MvObs& anObs )
{
   // Pick all values related to a message, including all subsets

   // Compressed data
   if ( anObs.compressData() )
   {
      // FAMI20171102: VERY IMPORTANT
      // This current algorithm is too slow. It is based on BUFRDC concept, which
      // uses command anObs.Advance() to access the next subset in order to retrieve
      // information related to a given key. ecCodes does not need this command.
      // Given a key, it retrieves data related to all subsets in one goal.
      // At the moment, the data for a given key is retrieved "number of subset" times,
      // instead of only once.
      bool flag = true;
      while (flag)
      {
         pickValues_compressedData(anObs);
         flag = anObs.Advance();   //next subset
      }
      return;
   }

   // Uncompressed data
   int number_of_subsets = anObs.msgSubsetCount();
   if ( number_of_subsets == 1 )
      pickValues_uncompressedData(anObs);
   else
   {
      for ( int i = 1; i <= number_of_subsets; i++ )
      {
         MvObs obs = anObs.cloneSubset((long)i);
         pickValues_uncompressedData(obs);
      }
   }

   return;
}

//_________________________________________________________pickValue_compressedData

void
BufrPicker::pickValues_compressedData( MvObs& anObs )
{
   // "tighten in" 'startIndex' with outer coordinates
   int startIndex = 0;         // coordinate interval start index in the msg
   for( int i = 0; i < (coordDescrCount_ - 1); ++i )
   {
      // Method 'specifierIndex':
      // look for a coordinate descriptor with the given value,
      // starting from the given index, returning the index of the coordinate point

      // FAMI20171011 This code is not working properly even in previous versions.
      std::cout << "BufrPicker::pickValue() -> More than one Coordinate Descriptors not working" << std::endl;
#if 0  //FAMI20171011
      startIndex = anObs.specifierIndex( coordDescr_[ i ]
                                        , coordValue_[ i ]
                                        , startIndex );
#else
       startIndex = -1;
#endif

      if( startIndex < 0 )
      {
         cerr << "BufrPicker: No match for coordinate " << coordDescr_[ i ]
              << " with value " << coordValue_[ i ] << endl;
         // marslog( LOG_WARN, "BufrPicker: No match for coordinate %d", coordDescr_[ i ]);
         pickedValue1_ = kBufrMissingValue;
         pickedValue2_ = kBufrMissingValue;
         add( anObs, coordValue_[ coordDescrCount_ - 1 ]  );
         return;  // no match for this coordinate => give up
      }
   }

   // We have reached the innnermost coordinate interval, lets pick the value(s)

   if( !pickAll_ )   // just pick the value for the single given innermost coordinate
   {
      // Method 'valueBySpecifier':
      //  o  first looks for a coordinate descriptor with the given value
      //  o     starts looking from 'startIndex'
      //  o  if found then looks for descriptor 'param_'
      //  o     starts looking from where the given coordinate descriptor was found
      //  o  returns the value of descr 'param_' (or a missing value)
      //
      pickedValue1_ = anObs.valueBySpecifier( coordDescr_[ coordDescrCount_ - 1 ], coordValue_[ coordDescrCount_ - 1 ], param_ );

      if( !param2_.empty() )       // geovector ?
      {
         pickedValue2_ = anObs.valueBySpecifier( coordDescr_[ coordDescrCount_ - 1 ], coordValue_[ coordDescrCount_ - 1 ], param2_ );
      }
      else
         pickedValue2_ = 0;                 // a 'non-missing' value

      add( anObs, coordValue_[ coordDescrCount_ - 1 ] );
      return;
   }

   // Pick all values for the innermost coordinate
   // Use dummy if only one coordinate level
   std::string coordDescr = coordDescr_[ coordDescrCount_ - 1 ];
   std::string prevCoordDescr = coordDescrCount_ > 1 ? coordDescr_[ coordDescrCount_ - 2 ] : "";

   if ( !anObs.setFirstDescriptor() ) // (implicitly) force descriptors to be expanded
      return;

   //e int currentIndex = startIndex + 1;  // set start position after the previous coordinate location
   std::string currentDescr;
   bool has_written = false;
   bool cont = true;
   while( cont )
   {
      currentDescr = anObs.currentKeyWithoutRank();
      if( currentDescr == prevCoordDescr )
         break;   // outer coordinate value changed => break

      if( currentDescr == coordDescr )
      {                                  // found new inner coordinate value
         double currentCoordValue = anObs.currentValue();

         anObs.setNextDescriptor();      // advance to the next data
         //e ++currentIndex;

         pickedValue1_ = kBufrMissingValue;
         pickedValue2_ = kBufrMissingValue;

         // loop until next innermost coordinate value
         while( anObs.currentKeyWithoutRank() != coordDescr )
         {
            currentDescr = anObs.currentKeyWithoutRank();
            if( currentDescr == param_ )
               pickedValue1_ = anObs.currentValue(); // main data found

            if( currentDescr == param2_ )
               pickedValue2_ = anObs.currentValue(); // secondary data found (only if vector)

            cont = anObs.setNextDescriptor();
            //e ++currentIndex;
            if( ! cont )
            {                                    // no more data available
               add( anObs, currentCoordValue );  // add last data value(s) for current innermost coordinate
               has_written = true;
               break;                            // end-of-msg reached => break
            }
         }

         if ( has_written )
            has_written = false;
         else
            add( anObs, currentCoordValue );  // add data value(s) for current innermost coordinate

         // Remove this command later and also variable currentIndex: only valid for BUFRDC
         //e anObs[ currentIndex + 1 ];        // revert internal pointer
      }
      else
      {
         cont = anObs.setNextDescriptor();
         //e ++currentIndex;
      }
   }

   return;
}

//_________________________________________________________pickValues_uncompressedData
void
BufrPicker::pickValues_uncompressedData( MvObs& obs )
{
#if 0 //FAMI20171115
   //-- "tighten in" 'startIndex' with outer coordinates
   int startIndex = 0;         // coordinate interval start index in the msg
   for( int i=0; i < (coordDescrCount_ - 1); ++i )
   {
      //-- Method 'specifierIndex':
      //-- look for a coordinate descriptor with the given value,
      //-- starting from the given index, returning the index of the coordinate point

      // FAMI20171011 This code is not working properly even in previous versions.
      std::cout << "BufrPicker::pickValue() -> More than one Coordinate Descriptors not working" << std::endl;
#if 0  //FAMI20171011
      startIndex = anObs.specifierIndex( coordDescr_[ i ]
                                        , coordValue_[ i ]
                                        , startIndex );
#else
       startIndex = -1;
#endif

      if( startIndex < 0 )
      {
         cerr << "BufrPicker: No match for coordinate " << coordDescr_[ i ]
              << " with value " << coordValue_[ i ] << endl;
         // marslog( LOG_WARN, "BufrPicker: No match for coordinate %d", coordDescr_[ i ]);
         pickedValue1_ = kBufrMissingValue;
         pickedValue2_ = kBufrMissingValue;
         add( anObs, coordValue_[ coordDescrCount_ - 1 ]  );
         return;  //-- no match for this coordinate => give up
      }
   }
#endif

   //-- We have reached the innnermost coordinate interval, lets pick the value(s)

   if( !pickAll_ )   //-- just pick the value for the single given innermost coordinate
   {
      //-- Method 'valueBySpecifier':
      //--  o  first looks for a coordinate descriptor with the given value
      //--  o     starts looking from 'startIndex'
      //--  o  if found then looks for descriptor 'param_'
      //--  o     starts looking from where the given coordinate descriptor was found
      //--  o  returns the value of descr 'param_' (or a missing value)
      //--
      pickedValue1_ = obs.valueBySpecifier( coordDescr_[ coordDescrCount_ - 1 ], coordValue_[ coordDescrCount_ - 1 ], param_ );
      if( !param2_.empty() )       //-- geovector ?
      {
         pickedValue2_ = obs.valueBySpecifier( coordDescr_[ coordDescrCount_ - 1 ], coordValue_[ coordDescrCount_ - 1 ], param2_ );
      }
      else
         pickedValue2_ = 0;                 //-- a 'non-missing' value

      add( obs, coordValue_[ coordDescrCount_ - 1 ] );
      return;
   }

   //-- Pick all values for the innermost coordinate
   std::string coordDescr = coordDescr_[ coordDescrCount_ - 1 ];
   std::string prevCoordDescr = coordDescrCount_ > 1 ? coordDescr_[ coordDescrCount_ - 2 ] : "";
   //-- use dummy if only one coordinate level

   if ( !obs.setFirstDescriptor() ) // (implicitly) force descriptors to be expanded
      return;

   //e int currentIndex = startIndex + 1;  //-- set start position after the previous coordinate location
   std::string currentDescr;
   bool has_written = false;
   bool cont = true;
   while( cont )
   {
      currentDescr = obs.currentKeyWithoutRank();
      if( currentDescr == prevCoordDescr )
         break;   //-- outer coordinate value changed => break

      if( currentDescr == coordDescr )
      {                                  //-- found new inner coordinate value
         double currentCoordValue = obs.currentValue();

         obs.setNextDescriptor();      //-- advance to the next data
         //e ++currentIndex;

         pickedValue1_ = kBufrMissingValue;
         pickedValue2_ = kBufrMissingValue;

         //-- loop until next innermost coordinate value
         while( obs.currentKeyWithoutRank() != coordDescr )
         {
            currentDescr = obs.currentKeyWithoutRank();
            if( currentDescr == param_ )
               pickedValue1_ = obs.currentValue(); //-- main data found

            if( currentDescr == param2_ )
               pickedValue2_ = obs.currentValue(); //-- secondary data found (only if vector)

            cont = obs.setNextDescriptor();
            //e ++currentIndex;
            if( ! cont )
            {                                    //-- no more data available
               add( obs, currentCoordValue );  //-- add last data value(s) for current innermost coordinate
               has_written = true;
               break;                            //-- end-of-msg reached => break
            }
         }

         if ( has_written )
            has_written = false;
         else
            add( obs, currentCoordValue );  //-- add data value(s) for current innermost coordinate

         // Remove this command later and also variable currentIndex: only valid for BUFRDC
         //e obs[ currentIndex + 1 ];        //-- revert internal pointer
      }
      else
      {
         cont = obs.setNextDescriptor();
         //e ++currentIndex;
      }
   }

   return;
}

//_________________________________________________________add
void
BufrPicker::add( MvObs& anObs, double aCoordValue )
{
   if( pickedValue1_ == kBufrMissingValue && !includeMissingValues_ )
      return;

   if( geoVectorType_ > 0 && pickedValue2_ == kBufrMissingValue && !includeMissingValues_ )
      return;

   MvLocation myLoc = anObs.location();
   TStaticTime myTime = anObs.obsTime();

   outfile_ << myLoc.latitude() << mySeparator;
   outfile_ << myLoc.longitude() << mySeparator;

                                             //-- use innermost coordinate value as "level"
   outfile_ << aCoordValue << mySeparator;

   outfile_ << myTime.GetYear();
   outfile_.width(2); outfile_.fill( '0' ); outfile_ << myTime.GetMonth();
   outfile_.width(2); outfile_.fill( '0' ); outfile_ << myTime.GetDay() << mySeparator;
   outfile_.width(2); outfile_.fill( '0' ); outfile_ << myTime.GetHour();
   outfile_.width(2); outfile_.fill( '0' ); outfile_ << myTime.GetMin() << mySeparator;

   if( pickedValue1_ == kBufrMissingValue )
      pickedValue1_ = missingValue_;        //-- swap to the requested missing value

   outfile_ << pickedValue1_;

   if( geoVectorType_ > 0 )                     //-- two values for geovectors
   {
      if( pickedValue2_ == kBufrMissingValue )
         pickedValue2_ = missingValue_;     //-- swap to the requested missing value

      outfile_ << mySeparator << pickedValue2_;
   }

   outfile_ << endl;

   counter_++;
}

//_________________________________________________________convertDescriptorsToKeys
void
BufrPicker::convertDescriptorsToKeys(MvObs& obs)
{
   param_ = obs.keyC(param_);
   if ( !param2_.empty() )
      param2_ = obs.keyC(param2_);

   for ( int i = 0; i < coordDescrCount_; i++ )
      coordDescr_[i] = obs.keyC(coordDescr_[i]);
}

//_________________________________________________________close
void
BufrPicker::close()
{
   outfile_.close();
}

//_________________________________________________________serve
void
BufrPicker::serve( MvRequest& in, MvRequest& out )
{
   time_t myStartTime;
   time( &myStartTime );
   clock_t myStartCpu = clock();

   in_ = in;
   //in_.print();

   // Get user input parameters
   getParams();                          //-- new inSet_ & iter_

   // Initialise variables and convert descriptors to keys
   MvRequest x = createRequest();
   MvObs myObs;
   convertDescriptorsToKeys(myObs);

   // Main loop
   counter_ = 0;
   try
   {
      // Loop per message. Function pickValue takes care about the subsets
      while( (myObs = (*iter_)(obsOrMsg_) ) )
      {
         pickValues( myObs );
         myObs.clearIterator(); // must do this before the msg handle is deleted by the next assignment
      }
      close();
   }
   catch ( MvException& e )
   {
      marslog( LOG_WARN, "BufrPicker: %s", e.what() );
      if( failOnError_ )
         throw MvException( "BufrPicker failed" );
   }

   // Forward auxiliary information
   time_t myEndTime;
   time( &myEndTime );
   double wallClockTime = difftime( myEndTime, myStartTime );
   clock_t myEndCpu = clock();
   double cpuTime = ( (double)( myEndCpu - myStartCpu ) ) / (double)CLOCKS_PER_SEC; // CLK_TCK;
   std::cout << " Tick Tack for " << wallClockTime << " seconds (" << cpuTime << " CPU secs)\n";

   if( pickAll_ )
      sendProgress( "Picked %d values (out of %d reports) into GeoPoints file in %d secs (cpu %d secs)",
                    counter_, inSet_->obsCount(), (int)wallClockTime, (int)cpuTime );
   else
      sendProgress( "Picked %d observations (out of %d) into GeoPoints file in %d secs (cpu %d secs)",
                    counter_, inSet_->obsCount(), (int)wallClockTime, (int)cpuTime );

   // Clean irternal memory
   clear();

   out = x;
   //out.print();
}

//_________________________________________________________main
int
main( int argc, char **argv )
{
   MvApplication theApp( argc, argv );
   BufrPicker picker;

   theApp.run();
}
