#include <iostream>
#include <sstream>
#include <chrono>
#include <regex>
#include <time.h>
#include <stdio.h>
#include <vector>
#include <map>
#include <chrono>
#include <map>
#include <utility>
#include <algorithm>
#include <random>
#include "bmdbg.h"
static
{
std::cerr
<< "BitMagic Dictionary Search Sample (c) 2018" << std::endl
<< "-idict file-name -- input set file to parse" << std::endl
<< "-svout spase vector output -- sparse vector name to save" << std::endl
<< "-svin sparse vector input -- sparse vector file name to load " << std::endl
<< "-diag -- run diagnostics" << std::endl
<< "-bench -- run benchmarks" << std::endl
<< "-timing -- collect timings" << std::endl
;
}
static
{
for (int i = 1; i < argc; ++i)
{
std::string arg = argv[i];
if ((arg == "-h") || (arg == "--help"))
{
return 0;
}
if (arg == "-svout" || arg == "--svout")
{
if (i + 1 < argc)
{
}
else
{
std::cerr << "Error: -svout requires file name" << std::endl;
return 1;
}
continue;
}
if (arg == "-svin" || arg == "--svin")
{
if (i + 1 < argc)
{
}
else
{
std::cerr << "Error: -svin requires file name" << std::endl;
return 1;
}
continue;
}
if (arg == "-idict" || arg == "--idict" )
{
if (i + 1 < argc)
{
}
else
{
std::cerr << "Error: -idict requires file name" << std::endl;
return 1;
}
continue;
}
if (arg == "-diag" || arg == "--diag" || arg == "-d" || arg == "--d")
{
continue;
}
if (arg == "-timing" || arg == "--timing" || arg == "-t" || arg == "--t")
{
continue;
}
if (arg == "-bench" || arg == "--bench" || arg == "-b" || arg == "--b")
{
continue;
}
std::cerr << "Error: unknown argument: " << arg << std::endl;
return 1;
}
return 0;
}
static
{
std::ifstream fin(fname.c_str(), std::ios::in);
if (!fin.good())
return -1;
std::string line;
std::regex reg("[|]");
std::sregex_token_iterator it_end;
string trim_chars("\" ");
for (unsigned i = 0; std::getline(fin, line); ++i)
{
if (line.empty() || !isdigit(line.front()))
continue;
std::sregex_token_iterator it(line.begin(), line.end(), reg, -1);
std::vector<std::string> line_vec(it, it_end);
if (line_vec.empty())
continue;
try
{
string& col13 = line_vec.at(13);
col13.erase(0, col13.find_first_not_of(trim_chars));
col13.erase(col13.find_last_not_of(trim_chars) + 1);
if (!col13.empty())
str_vec.emplace_back(col13);
}
catch (std::exception&)
{
}
if (i % 10000 == 0)
{
cout << "\rReading input file: " << i << flush;
}
}
cout << endl;
return 0;
}
static
{
if (str_vec.size() != str_sv.
size())
throw runtime_error("Error. size() comparison failed!");
string s;
{
const string& s_control = str_vec[i];
if (s != s_control)
throw runtime_error("Error. element comparison failed!");
}
std::cout <<
"Check ok. Dictionary size = " << str_sv.
size() << std:: endl;
}
static
{
std::uniform_int_distribution<unsigned>
rand_dis(0,
unsigned(str_vec.size()-1));
{
unsigned idx;
while (true)
{
continue;
if (idx < str_vec.size())
bench_vec.push_back(str_vec[idx]);
break;
}
{
string str_nf = str_vec[idx];
string::reverse_iterator rit = str_nf.rbegin();
string::reverse_iterator rit_end = str_nf.rend();
for (; rit != rit_end; ++rit)
{
char ch = *rit;
int a = rand() % 26 + int('A');
ch = char(a);
*rit = ch;
auto it = std::lower_bound(str_vec.begin(), str_vec.end(), str_nf);
if (it == str_vec.end() || *it != str_nf)
{
bench_vec_not_found.push_back(str_nf);
break;
}
}
}
}
cout << endl;
}
static
{
cout <<
"Picked " << bench_vec.
size() <<
" / " << bench_vec_not_found.size() << " samples. Running benchmarks."
<< endl;
unsigned bench_size = unsigned(bench_vec.size());
{
{
for (const string& term : bench_vec)
{
auto it = std::lower_bound(str_vec.begin(), str_vec.end(), term);
if (it != str_vec.end())
{
string_vector::size_type idx =
string_vector::size_type(std::distance(str_vec.begin(), it));
}
}
}
{
for (const string& term : bench_vec_not_found)
{
std::lower_bound(str_vec.begin(), str_vec.end(), term);
}
}
}
{
std::map<string, unsigned> str_map;
for (string_vector::size_type i = 0; i < str_vec.size(); ++i)
{
const string& s = str_vec[i];
str_map[s] = unsigned(i);
}
{
for (const string& term : bench_vec)
{
auto it = str_map.find(term);
if (it != str_map.end())
{
bv2.
set(
unsigned(it->second));
}
}
}
{
for (const string& term : bench_vec_not_found)
{
auto it = str_map.find(term);
if (it != str_map.end())
{
cerr << "empty search returned value..." << endl;
}
}
}
}
{
{
for (const string& term : bench_vec)
{
unsigned pos;
bool found = scanner.
find_eq_str(str_sv, term.c_str(), pos);
if (found)
{
}
}
}
{
for (const string& term : bench_vec_not_found)
{
unsigned pos;
bool found = scanner.
find_eq_str(str_sv, term.c_str(), pos);
if (found)
{
cerr << "scanner empty search returned value..." << endl;
}
}
}
}
{
scanner.
bind(str_sv,
true);
{
for (const string& term : bench_vec)
{
unsigned pos;
if (found)
{
}
}
}
{
for (const string& term : bench_vec_not_found)
{
unsigned pos;
if (found)
{
cerr << "scanner empty search returned value..." << endl;
}
}
}
}
if (cmp != 0)
throw runtime_error("Error. RB-search mismatch!");
if (cmp != 0)
throw runtime_error("Error. scanner mismatch!");
if (cmp != 0)
throw runtime_error("Error. binary scanner mismatch!");
if (bv1.
count() != bench_size)
throw runtime_error("Error. Search result missing elements!");
}
int main(
int argc,
char *argv[])
{
if (argc < 3)
{
return 1;
}
try
{
if (ret != 0)
{
return ret;
}
{
if (res != 0)
{
return res;
}
cout << "Loaded " << str_vec.size() << " dictionary names." << endl;
std::sort(str_vec.begin(), str_vec.end());
}
if (str_vec.size())
{
for (const string& term : str_vec)
{
str_sv.
swap(str_sv_remap);
}
}
{
}
{
{
}
if (str_vec.empty())
{
{
string s;
str_vec.emplace_back(std::move(s));
}
}
}
{
{
print_svector_stat(str_sv, true);
}
if (str_vec.size())
{
size_t total_size = 0;
for (const string& term : str_vec)
{
total_size += term.size();
}
cout << "String dictionary size: "
<< total_size / 1024 << "KB (" << total_size / (1024*1024) << "MB)"
<< endl;
}
if (str_sv.
size() && str_vec.size())
{
cout << "Run full comparison check..." << endl;
cout << "Ok" << endl;
}
}
{
}
{
std::cout << std::endl << "Performance:" << std::endl;
}
}
catch (std::exception& ex)
{
std::cerr << "Error:" << ex.what() << std::endl;
return 1;
}
return 0;
}