OmniSciDB  1dac507f6e
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Importer_NS::Detector Class Reference

#include <Importer.h>

+ Inheritance diagram for Importer_NS::Detector:
+ Collaboration diagram for Importer_NS::Detector:

Public Member Functions

 Detector (const boost::filesystem::path &fp, CopyParams &cp)
 
std::vector< std::string > get_headers ()
 
std::vector< std::vector
< std::string > > 
get_sample_rows (size_t n)
 
- Public Member Functions inherited from Importer_NS::DataStreamSink
 DataStreamSink ()
 
 DataStreamSink (const CopyParams &copy_params, const std::string file_path)
 
virtual ~DataStreamSink ()
 
const CopyParamsget_copy_params () const
 
void import_compressed (std::vector< std::string > &file_paths)
 

Static Public Member Functions

static SQLTypes detect_sqltype (const std::string &str)
 

Public Attributes

std::vector< std::vector
< std::string > > 
raw_rows
 
std::vector< SQLTypesbest_sqltypes
 
std::vector< EncodingTypebest_encodings
 
bool has_headers = false
 

Private Member Functions

void init ()
 
void read_file ()
 
void detect_row_delimiter ()
 
void split_raw_data ()
 
std::vector< SQLTypesdetect_column_types (const std::vector< std::string > &row)
 
void find_best_sqltypes ()
 
std::vector< SQLTypesfind_best_sqltypes (const std::vector< std::vector< std::string >> &raw_rows, const CopyParams &copy_params)
 
std::vector< SQLTypesfind_best_sqltypes (const std::vector< std::vector< std::string >>::const_iterator &row_begin, const std::vector< std::vector< std::string >>::const_iterator &row_end, const CopyParams &copy_params)
 
std::vector< EncodingTypefind_best_encodings (const std::vector< std::vector< std::string >>::const_iterator &row_begin, const std::vector< std::vector< std::string >>::const_iterator &row_end, const std::vector< SQLTypes > &best_types)
 
bool detect_headers (const std::vector< SQLTypes > &first_types, const std::vector< SQLTypes > &rest_types)
 
void find_best_sqltypes_and_headers ()
 
ImportStatus importDelimited (const std::string &file_path, const bool decompressed) override
 

Static Private Member Functions

static bool more_restrictive_sqltype (const SQLTypes a, const SQLTypes b)
 

Private Attributes

std::string raw_data
 
boost::filesystem::path file_path
 
std::chrono::duration< double > timeout {1}
 
std::string line1
 

Additional Inherited Members

- Protected Member Functions inherited from Importer_NS::DataStreamSink
ImportStatus archivePlumber ()
 
- Protected Attributes inherited from Importer_NS::DataStreamSink
CopyParams copy_params
 
const std::string file_path
 
FILE * p_file = nullptr
 
ImportStatus import_status
 
bool load_failed = false
 
size_t total_file_size {0}
 
std::vector< size_t > file_offsets
 
std::mutex file_offsets_mutex
 

Detailed Description

Definition at line 663 of file Importer.h.

Constructor & Destructor Documentation

Importer_NS::Detector::Detector ( const boost::filesystem::path &  fp,
CopyParams cp 
)
inline

Definition at line 665 of file Importer.h.

References init(), and read_file().

666  : DataStreamSink(cp, fp.string()), file_path(fp) {
667  read_file();
668  init();
669  };
boost::filesystem::path file_path
Definition: Importer.h:708

+ Here is the call graph for this function:

Member Function Documentation

std::vector< SQLTypes > Importer_NS::Detector::detect_column_types ( const std::vector< std::string > &  row)
private

Definition at line 2933 of file Importer.cpp.

References detect_sqltype().

Referenced by find_best_sqltypes_and_headers().

2933  {
2934  std::vector<SQLTypes> types(row.size());
2935  for (size_t i = 0; i < row.size(); i++) {
2936  types[i] = detect_sqltype(row[i]);
2937  }
2938  return types;
2939 }
static SQLTypes detect_sqltype(const std::string &str)
Definition: Importer.cpp:2856

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

bool Importer_NS::Detector::detect_headers ( const std::vector< SQLTypes > &  first_types,
const std::vector< SQLTypes > &  rest_types 
)
private

Definition at line 3075 of file Importer.cpp.

References has_headers, and kTEXT.

Referenced by find_best_sqltypes_and_headers().

3076  {
3077  if (head_types.size() != tail_types.size()) {
3078  return false;
3079  }
3080  bool has_headers = false;
3081  for (size_t col_idx = 0; col_idx < tail_types.size(); col_idx++) {
3082  if (head_types[col_idx] != kTEXT) {
3083  return false;
3084  }
3085  has_headers = has_headers || tail_types[col_idx] != kTEXT;
3086  }
3087  return has_headers;
3088 }
Definition: sqltypes.h:55

+ Here is the caller graph for this function:

void Importer_NS::Detector::detect_row_delimiter ( )
private

Definition at line 2800 of file Importer.cpp.

References Importer_NS::DataStreamSink::copy_params, Importer_NS::CopyParams::delimiter, and file_path.

Referenced by init().

2800  {
2801  if (copy_params.delimiter == '\0') {
2802  copy_params.delimiter = ',';
2803  if (boost::filesystem::extension(file_path) == ".tsv") {
2804  copy_params.delimiter = '\t';
2805  }
2806  }
2807 }
boost::filesystem::path file_path
Definition: Importer.h:708

+ Here is the caller graph for this function:

SQLTypes Importer_NS::Detector::detect_sqltype ( const std::string &  str)
static

Definition at line 2856 of file Importer.cpp.

References kBIGINT, kDATE, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, Importer_NS::PROMOTE_POLYGON_TO_MULTIPOLYGON, Importer_NS::try_strptimes(), and run_benchmark_import::type.

Referenced by detect_column_types(), and find_best_sqltypes().

2856  {
2857  SQLTypes type = kTEXT;
2858  if (try_cast<double>(str)) {
2859  type = kDOUBLE;
2860  /*if (try_cast<bool>(str)) {
2861  type = kBOOLEAN;
2862  }*/
2863  if (try_cast<int16_t>(str)) {
2864  type = kSMALLINT;
2865  } else if (try_cast<int32_t>(str)) {
2866  type = kINT;
2867  } else if (try_cast<int64_t>(str)) {
2868  type = kBIGINT;
2869  } else if (try_cast<float>(str)) {
2870  type = kFLOAT;
2871  }
2872  }
2873 
2874  // check for geo types
2875  if (type == kTEXT) {
2876  // convert to upper case
2877  std::string str_upper_case = str;
2878  std::transform(
2879  str_upper_case.begin(), str_upper_case.end(), str_upper_case.begin(), ::toupper);
2880 
2881  // then test for leading words
2882  if (str_upper_case.find("POINT") == 0) {
2883  type = kPOINT;
2884  } else if (str_upper_case.find("LINESTRING") == 0) {
2885  type = kLINESTRING;
2886  } else if (str_upper_case.find("POLYGON") == 0) {
2888  type = kMULTIPOLYGON;
2889  } else {
2890  type = kPOLYGON;
2891  }
2892  } else if (str_upper_case.find("MULTIPOLYGON") == 0) {
2893  type = kMULTIPOLYGON;
2894  } else if (str_upper_case.find_first_not_of("0123456789ABCDEF") ==
2895  std::string::npos &&
2896  (str_upper_case.size() % 2) == 0) {
2897  // could be a WKB hex blob
2898  // we can't handle these yet
2899  // leave as TEXT for now
2900  // deliberate return here, as otherwise this would get matched as TIME
2901  // @TODO
2902  // implement WKB import
2903  return type;
2904  }
2905  }
2906 
2907  // check for time types
2908  if (type == kTEXT) {
2909  // @TODO
2910  // make these tests more robust so they don't match stuff they should not
2911  char* buf;
2912  buf = try_strptimes(str.c_str(), {"%Y-%m-%d", "%m/%d/%Y", "%d-%b-%y", "%d/%b/%Y"});
2913  if (buf) {
2914  type = kDATE;
2915  if (*buf == 'T' || *buf == ' ' || *buf == ':') {
2916  buf++;
2917  }
2918  }
2919  buf = try_strptimes(buf == nullptr ? str.c_str() : buf,
2920  {"%T %z", "%T", "%H%M%S", "%R"});
2921  if (buf) {
2922  if (type == kDATE) {
2923  type = kTIMESTAMP;
2924  } else {
2925  type = kTIME;
2926  }
2927  }
2928  }
2929 
2930  return type;
2931 }
Definition: sqltypes.h:52
SQLTypes
Definition: sqltypes.h:41
char * try_strptimes(const char *str, const std::vector< std::string > &formats)
Definition: Importer.cpp:2844
Definition: sqltypes.h:55
Definition: sqltypes.h:56
Definition: sqltypes.h:48
static constexpr bool PROMOTE_POLYGON_TO_MULTIPOLYGON
Definition: Importer.cpp:144

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::vector< EncodingType > Importer_NS::Detector::find_best_encodings ( const std::vector< std::vector< std::string >>::const_iterator &  row_begin,
const std::vector< std::vector< std::string >>::const_iterator &  row_end,
const std::vector< SQLTypes > &  best_types 
)
private

Definition at line 3039 of file Importer.cpp.

References file_path, IS_STRING, kENCODING_DICT, kENCODING_NONE, and raw_rows.

Referenced by find_best_sqltypes_and_headers().

3042  {
3043  if (raw_rows.size() < 1) {
3044  throw std::runtime_error("No rows found in: " +
3045  boost::filesystem::basename(file_path));
3046  }
3047  size_t num_cols = best_types.size();
3048  std::vector<EncodingType> best_encodes(num_cols, kENCODING_NONE);
3049  std::vector<size_t> num_rows_per_col(num_cols, 1);
3050  std::vector<std::unordered_set<std::string>> count_set(num_cols);
3051  for (auto row = row_begin; row != row_end; row++) {
3052  for (size_t col_idx = 0; col_idx < row->size() && col_idx < num_cols; col_idx++) {
3053  if (IS_STRING(best_types[col_idx])) {
3054  count_set[col_idx].insert(row->at(col_idx));
3055  num_rows_per_col[col_idx]++;
3056  }
3057  }
3058  }
3059  for (size_t col_idx = 0; col_idx < num_cols; col_idx++) {
3060  if (IS_STRING(best_types[col_idx])) {
3061  float uniqueRatio =
3062  static_cast<float>(count_set[col_idx].size()) / num_rows_per_col[col_idx];
3063  if (uniqueRatio < 0.75) {
3064  best_encodes[col_idx] = kENCODING_DICT;
3065  }
3066  }
3067  }
3068  return best_encodes;
3069 }
boost::filesystem::path file_path
Definition: Importer.h:708
std::vector< std::vector< std::string > > raw_rows
Definition: Importer.h:675
#define IS_STRING(T)
Definition: sqltypes.h:166

+ Here is the caller graph for this function:

void Importer_NS::Detector::find_best_sqltypes ( )
private

Definition at line 2986 of file Importer.cpp.

References best_sqltypes, Importer_NS::DataStreamSink::copy_params, and raw_rows.

Referenced by find_best_sqltypes(), and find_best_sqltypes_and_headers().

2986  {
2988 }
std::vector< std::vector< std::string > > raw_rows
Definition: Importer.h:675
std::vector< SQLTypes > best_sqltypes
Definition: Importer.h:677

+ Here is the caller graph for this function:

std::vector< SQLTypes > Importer_NS::Detector::find_best_sqltypes ( const std::vector< std::vector< std::string >> &  raw_rows,
const CopyParams copy_params 
)
private

Definition at line 2990 of file Importer.cpp.

References Importer_NS::DataStreamSink::copy_params, find_best_sqltypes(), and raw_rows.

2992  {
2993  return find_best_sqltypes(raw_rows.begin(), raw_rows.end(), copy_params);
2994 }
std::vector< std::vector< std::string > > raw_rows
Definition: Importer.h:675

+ Here is the call graph for this function:

std::vector< SQLTypes > Importer_NS::Detector::find_best_sqltypes ( const std::vector< std::vector< std::string >>::const_iterator &  row_begin,
const std::vector< std::vector< std::string >>::const_iterator &  row_end,
const CopyParams copy_params 
)
private

Definition at line 2996 of file Importer.cpp.

References detect_sqltype(), run_benchmark_import::end_time, file_path, kCHAR, kTEXT, more_restrictive_sqltype(), Importer_NS::CopyParams::null_str, raw_rows, and timeout.

2999  {
3000  if (raw_rows.size() < 1) {
3001  throw std::runtime_error("No rows found in: " +
3002  boost::filesystem::basename(file_path));
3003  }
3004  auto end_time = std::chrono::steady_clock::now() + timeout;
3005  size_t num_cols = raw_rows.front().size();
3006  std::vector<SQLTypes> best_types(num_cols, kCHAR);
3007  std::vector<size_t> non_null_col_counts(num_cols, 0);
3008  for (auto row = row_begin; row != row_end; row++) {
3009  while (best_types.size() < row->size() || non_null_col_counts.size() < row->size()) {
3010  best_types.push_back(kCHAR);
3011  non_null_col_counts.push_back(0);
3012  }
3013  for (size_t col_idx = 0; col_idx < row->size(); col_idx++) {
3014  // do not count nulls
3015  if (row->at(col_idx) == "" || !row->at(col_idx).compare(copy_params.null_str)) {
3016  continue;
3017  }
3018  SQLTypes t = detect_sqltype(row->at(col_idx));
3019  non_null_col_counts[col_idx]++;
3020  if (!more_restrictive_sqltype(best_types[col_idx], t)) {
3021  best_types[col_idx] = t;
3022  }
3023  }
3024  if (std::chrono::steady_clock::now() > end_time) {
3025  break;
3026  }
3027  }
3028  for (size_t col_idx = 0; col_idx < num_cols; col_idx++) {
3029  // if we don't have any non-null values for this column make it text to be
3030  // safe b/c that is least restrictive type
3031  if (non_null_col_counts[col_idx] == 0) {
3032  best_types[col_idx] = kTEXT;
3033  }
3034  }
3035 
3036  return best_types;
3037 }
std::chrono::duration< double > timeout
Definition: Importer.h:709
std::string null_str
Definition: CopyParams.h:47
SQLTypes
Definition: sqltypes.h:41
static SQLTypes detect_sqltype(const std::string &str)
Definition: Importer.cpp:2856
boost::filesystem::path file_path
Definition: Importer.h:708
Definition: sqltypes.h:55
static bool more_restrictive_sqltype(const SQLTypes a, const SQLTypes b)
Definition: Importer.cpp:2941
std::vector< std::vector< std::string > > raw_rows
Definition: Importer.h:675
Definition: sqltypes.h:44

+ Here is the call graph for this function:

void Importer_NS::Detector::find_best_sqltypes_and_headers ( )
private

Definition at line 2963 of file Importer.cpp.

References Importer_NS::AUTODETECT, best_encodings, best_sqltypes, Importer_NS::DataStreamSink::copy_params, detect_column_types(), detect_headers(), find_best_encodings(), find_best_sqltypes(), Importer_NS::HAS_HEADER, Importer_NS::CopyParams::has_header, has_headers, Importer_NS::NO_HEADER, and raw_rows.

Referenced by init().

2963  {
2965  best_encodings =
2966  find_best_encodings(raw_rows.begin() + 1, raw_rows.end(), best_sqltypes);
2967  std::vector<SQLTypes> head_types = detect_column_types(raw_rows.at(0));
2968  switch (copy_params.has_header) {
2970  has_headers = detect_headers(head_types, best_sqltypes);
2971  if (has_headers) {
2973  } else {
2975  }
2976  break;
2978  has_headers = false;
2979  break;
2981  has_headers = true;
2982  break;
2983  }
2984 }
std::vector< EncodingType > find_best_encodings(const std::vector< std::vector< std::string >>::const_iterator &row_begin, const std::vector< std::vector< std::string >>::const_iterator &row_end, const std::vector< SQLTypes > &best_types)
Definition: Importer.cpp:3039
std::vector< EncodingType > best_encodings
Definition: Importer.h:678
ImportHeaderRow has_header
Definition: CopyParams.h:48
std::vector< SQLTypes > detect_column_types(const std::vector< std::string > &row)
Definition: Importer.cpp:2933
std::vector< std::vector< std::string > > raw_rows
Definition: Importer.h:675
bool detect_headers(const std::vector< SQLTypes > &first_types, const std::vector< SQLTypes > &rest_types)
Definition: Importer.cpp:3075
std::vector< SQLTypes > best_sqltypes
Definition: Importer.h:677

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::vector< std::string > Importer_NS::Detector::get_headers ( )

Definition at line 3098 of file Importer.cpp.

References best_sqltypes, has_headers, raw_rows, and to_string().

Referenced by MapDHandler::detect_column_types().

3098  {
3099  std::vector<std::string> headers(best_sqltypes.size());
3100  for (size_t i = 0; i < best_sqltypes.size(); i++) {
3101  if (has_headers && i < raw_rows[0].size()) {
3102  headers[i] = raw_rows[0][i];
3103  } else {
3104  headers[i] = "column_" + std::to_string(i + 1);
3105  }
3106  }
3107  return headers;
3108 }
std::string to_string(char const *&&v)
std::vector< std::vector< std::string > > raw_rows
Definition: Importer.h:675
std::vector< SQLTypes > best_sqltypes
Definition: Importer.h:677

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::vector< std::vector< std::string > > Importer_NS::Detector::get_sample_rows ( size_t  n)

Definition at line 3090 of file Importer.cpp.

References has_headers, and raw_rows.

Referenced by MapDHandler::detect_column_types().

3090  {
3091  n = std::min(n, raw_rows.size());
3092  size_t offset = (has_headers && raw_rows.size() > 1) ? 1 : 0;
3093  std::vector<std::vector<std::string>> sample_rows(raw_rows.begin() + offset,
3094  raw_rows.begin() + n);
3095  return sample_rows;
3096 }
std::vector< std::vector< std::string > > raw_rows
Definition: Importer.h:675

+ Here is the caller graph for this function:

ImportStatus Importer_NS::Detector::importDelimited ( const std::string &  file_path,
const bool  decompressed 
)
overrideprivatevirtual

Implements Importer_NS::DataStreamSink.

Definition at line 2735 of file Importer.cpp.

References Importer_NS::DataStreamSink::copy_params, run_benchmark_import::end_time, Importer_NS::DataStreamSink::import_status, line1, Importer_NS::CopyParams::line_delim, Importer_NS::DataStreamSink::load_failed, Importer_NS::ImportStatus::load_truncated, Importer_NS::DataStreamSink::p_file, raw_data, Importer_NS::ImportStatus::rows_completed, and timeout.

2736  {
2737  if (!p_file) {
2738  p_file = fopen(file_path.c_str(), "rb");
2739  }
2740  if (!p_file) {
2741  throw std::runtime_error("failed to open file '" + file_path +
2742  "': " + strerror(errno));
2743  }
2744 
2745  // somehow clang does not support ext/stdio_filebuf.h, so
2746  // need to diy readline with customized copy_params.line_delim...
2747  char line[1 << 20];
2748  auto end_time = std::chrono::steady_clock::now() +
2749  timeout * (boost::istarts_with(file_path, "s3://") ? 3 : 1);
2750  try {
2751  while (!feof(p_file)) {
2752  int c;
2753  size_t n = 0;
2754  while (EOF != (c = fgetc(p_file)) && copy_params.line_delim != c) {
2755  line[n++] = c;
2756  if (n >= sizeof(line) - 1) {
2757  break;
2758  }
2759  }
2760  if (0 == n) {
2761  break;
2762  }
2763  line[n] = 0;
2764  // remember the first line, which is possibly a header line, to
2765  // ignore identical header line(s) in 2nd+ files of a archive;
2766  // otherwise, 2nd+ header may be mistaken as an all-string row
2767  // and so be final column types.
2768  if (line1.empty()) {
2769  line1 = line;
2770  } else if (line == line1) {
2771  continue;
2772  }
2773 
2774  raw_data += std::string(line, n);
2777  if (std::chrono::steady_clock::now() > end_time) {
2778  if (import_status.rows_completed > 10000) {
2779  break;
2780  }
2781  }
2782  }
2783  } catch (std::exception& e) {
2784  }
2785 
2786  // as if load truncated
2788  load_failed = true;
2789 
2790  fclose(p_file);
2791  p_file = nullptr;
2792  return import_status;
2793 }
std::chrono::duration< double > timeout
Definition: Importer.h:709
std::string line1
Definition: Importer.h:710
std::string raw_data
Definition: Importer.h:707
boost::filesystem::path file_path
Definition: Importer.h:708
ImportStatus import_status
Definition: Importer.h:656
void Importer_NS::Detector::init ( )
private

Definition at line 2729 of file Importer.cpp.

References detect_row_delimiter(), find_best_sqltypes_and_headers(), and split_raw_data().

Referenced by Detector().

2729  {
2731  split_raw_data();
2733 }
void find_best_sqltypes_and_headers()
Definition: Importer.cpp:2963

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

bool Importer_NS::Detector::more_restrictive_sqltype ( const SQLTypes  a,
const SQLTypes  b 
)
staticprivate

Definition at line 2941 of file Importer.cpp.

References kBIGINT, kBOOLEAN, kCHAR, kDATE, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, and kTIMESTAMP.

Referenced by find_best_sqltypes().

2941  {
2942  static std::array<int, kSQLTYPE_LAST> typeorder;
2943  typeorder[kCHAR] = 0;
2944  typeorder[kBOOLEAN] = 2;
2945  typeorder[kSMALLINT] = 3;
2946  typeorder[kINT] = 4;
2947  typeorder[kBIGINT] = 5;
2948  typeorder[kFLOAT] = 6;
2949  typeorder[kDOUBLE] = 7;
2950  typeorder[kTIMESTAMP] = 8;
2951  typeorder[kTIME] = 9;
2952  typeorder[kDATE] = 10;
2953  typeorder[kPOINT] = 11;
2954  typeorder[kLINESTRING] = 11;
2955  typeorder[kPOLYGON] = 11;
2956  typeorder[kMULTIPOLYGON] = 11;
2957  typeorder[kTEXT] = 12;
2958 
2959  // note: b < a instead of a < b because the map is ordered most to least restrictive
2960  return typeorder[b] < typeorder[a];
2961 }
Definition: sqltypes.h:52
Definition: sqltypes.h:55
Definition: sqltypes.h:56
Definition: sqltypes.h:44
Definition: sqltypes.h:48

+ Here is the caller graph for this function:

void Importer_NS::Detector::read_file ( )
private

Definition at line 2795 of file Importer.cpp.

References Importer_NS::DataStreamSink::archivePlumber().

Referenced by Detector().

2795  {
2796  // this becomes analogous to Importer::import()
2798 }
ImportStatus archivePlumber()
Definition: Importer.cpp:3146

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Importer_NS::Detector::split_raw_data ( )
private

Definition at line 2809 of file Importer.cpp.

References Importer_NS::DataStreamSink::copy_params, Importer_NS::DelimitedParserUtils::get_row(), raw_data, raw_rows, and Importer_NS::CopyParams::threads.

Referenced by init().

2809  {
2810  const char* buf = raw_data.c_str();
2811  const char* buf_end = buf + raw_data.size();
2812  bool try_single_thread = false;
2813  for (const char* p = buf; p < buf_end; p++) {
2814  std::vector<std::string> row;
2816  p, buf_end, buf_end, copy_params, nullptr, row, try_single_thread);
2817  raw_rows.push_back(row);
2818  if (try_single_thread) {
2819  break;
2820  }
2821  }
2822  if (try_single_thread) {
2823  copy_params.threads = 1;
2824  raw_rows.clear();
2825  for (const char* p = buf; p < buf_end; p++) {
2826  std::vector<std::string> row;
2828  p, buf_end, buf_end, copy_params, nullptr, row, try_single_thread);
2829  raw_rows.push_back(row);
2830  }
2831  }
2832 }
static const char * get_row(const char *buf, const char *buf_end, const char *entire_buf_end, const Importer_NS::CopyParams &copy_params, const bool *is_array, std::vector< std::string > &row, bool &try_single_thread)
Parses the first row in the given buffer and inserts fields into given vector.
std::string raw_data
Definition: Importer.h:707
std::vector< std::vector< std::string > > raw_rows
Definition: Importer.h:675

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Member Data Documentation

std::vector<EncodingType> Importer_NS::Detector::best_encodings

Definition at line 678 of file Importer.h.

Referenced by MapDHandler::detect_column_types(), and find_best_sqltypes_and_headers().

std::vector<SQLTypes> Importer_NS::Detector::best_sqltypes
boost::filesystem::path Importer_NS::Detector::file_path
private

Definition at line 708 of file Importer.h.

Referenced by detect_row_delimiter(), find_best_encodings(), and find_best_sqltypes().

bool Importer_NS::Detector::has_headers = false
std::string Importer_NS::Detector::line1
private

Definition at line 710 of file Importer.h.

Referenced by importDelimited().

std::string Importer_NS::Detector::raw_data
private

Definition at line 707 of file Importer.h.

Referenced by importDelimited(), and split_raw_data().

std::vector<std::vector<std::string> > Importer_NS::Detector::raw_rows
std::chrono::duration<double> Importer_NS::Detector::timeout {1}
private

Definition at line 709 of file Importer.h.

Referenced by find_best_sqltypes(), and importDelimited().


The documentation for this class was generated from the following files: