OmniSciDB  0fdbebe030
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Importer_NS::Detector Class Reference

#include <Importer.h>

+ Inheritance diagram for Importer_NS::Detector:
+ Collaboration diagram for Importer_NS::Detector:

Public Member Functions

 Detector (const boost::filesystem::path &fp, CopyParams &cp)
 
std::vector< std::string > get_headers ()
 
std::vector< std::vector
< std::string > > 
get_sample_rows (size_t n)
 
- Public Member Functions inherited from Importer_NS::DataStreamSink
 DataStreamSink ()
 
 DataStreamSink (const CopyParams &copy_params, const std::string file_path)
 
virtual ~DataStreamSink ()
 
const CopyParamsget_copy_params () const
 
void import_compressed (std::vector< std::string > &file_paths)
 

Static Public Member Functions

static SQLTypes detect_sqltype (const std::string &str)
 

Public Attributes

std::vector< std::vector
< std::string > > 
raw_rows
 
std::vector< SQLTypesbest_sqltypes
 
std::vector< EncodingTypebest_encodings
 
bool has_headers = false
 

Private Member Functions

void init ()
 
void read_file ()
 
void detect_row_delimiter ()
 
void split_raw_data ()
 
std::vector< SQLTypesdetect_column_types (const std::vector< std::string > &row)
 
void find_best_sqltypes ()
 
std::vector< SQLTypesfind_best_sqltypes (const std::vector< std::vector< std::string >> &raw_rows, const CopyParams &copy_params)
 
std::vector< SQLTypesfind_best_sqltypes (const std::vector< std::vector< std::string >>::const_iterator &row_begin, const std::vector< std::vector< std::string >>::const_iterator &row_end, const CopyParams &copy_params)
 
std::vector< EncodingTypefind_best_encodings (const std::vector< std::vector< std::string >>::const_iterator &row_begin, const std::vector< std::vector< std::string >>::const_iterator &row_end, const std::vector< SQLTypes > &best_types)
 
bool detect_headers (const std::vector< SQLTypes > &first_types, const std::vector< SQLTypes > &rest_types)
 
void find_best_sqltypes_and_headers ()
 
ImportStatus importDelimited (const std::string &file_path, const bool decompressed) override
 

Static Private Member Functions

static bool more_restrictive_sqltype (const SQLTypes a, const SQLTypes b)
 

Private Attributes

std::string raw_data
 
boost::filesystem::path file_path
 
std::chrono::duration< double > timeout {1}
 
std::string line1
 

Additional Inherited Members

- Protected Member Functions inherited from Importer_NS::DataStreamSink
ImportStatus archivePlumber ()
 
- Protected Attributes inherited from Importer_NS::DataStreamSink
CopyParams copy_params
 
const std::string file_path
 
FILE * p_file = nullptr
 
ImportStatus import_status
 
bool load_failed = false
 
size_t total_file_size {0}
 
std::vector< size_t > file_offsets
 
std::mutex file_offsets_mutex
 

Detailed Description

Definition at line 638 of file Importer.h.

Constructor & Destructor Documentation

Importer_NS::Detector::Detector ( const boost::filesystem::path &  fp,
CopyParams cp 
)
inline

Definition at line 640 of file Importer.h.

References init(), and read_file().

641  : DataStreamSink(cp, fp.string()), file_path(fp) {
642  read_file();
643  init();
644  };
boost::filesystem::path file_path
Definition: Importer.h:683

+ Here is the call graph for this function:

Member Function Documentation

std::vector< SQLTypes > Importer_NS::Detector::detect_column_types ( const std::vector< std::string > &  row)
private

Definition at line 2991 of file Importer.cpp.

References detect_sqltype().

Referenced by find_best_sqltypes_and_headers().

2991  {
2992  std::vector<SQLTypes> types(row.size());
2993  for (size_t i = 0; i < row.size(); i++) {
2994  types[i] = detect_sqltype(row[i]);
2995  }
2996  return types;
2997 }
static SQLTypes detect_sqltype(const std::string &str)
Definition: Importer.cpp:2914

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

bool Importer_NS::Detector::detect_headers ( const std::vector< SQLTypes > &  first_types,
const std::vector< SQLTypes > &  rest_types 
)
private

Definition at line 3133 of file Importer.cpp.

References has_headers, and kTEXT.

Referenced by find_best_sqltypes_and_headers().

3134  {
3135  if (head_types.size() != tail_types.size()) {
3136  return false;
3137  }
3138  bool has_headers = false;
3139  for (size_t col_idx = 0; col_idx < tail_types.size(); col_idx++) {
3140  if (head_types[col_idx] != kTEXT) {
3141  return false;
3142  }
3143  has_headers = has_headers || tail_types[col_idx] != kTEXT;
3144  }
3145  return has_headers;
3146 }
Definition: sqltypes.h:53

+ Here is the caller graph for this function:

void Importer_NS::Detector::detect_row_delimiter ( )
private

Definition at line 2858 of file Importer.cpp.

References Importer_NS::DataStreamSink::copy_params, Importer_NS::CopyParams::delimiter, and file_path.

Referenced by init().

2858  {
2859  if (copy_params.delimiter == '\0') {
2860  copy_params.delimiter = ',';
2861  if (boost::filesystem::extension(file_path) == ".tsv") {
2862  copy_params.delimiter = '\t';
2863  }
2864  }
2865 }
boost::filesystem::path file_path
Definition: Importer.h:683

+ Here is the caller graph for this function:

SQLTypes Importer_NS::Detector::detect_sqltype ( const std::string &  str)
static

Definition at line 2914 of file Importer.cpp.

References kBIGINT, kDATE, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, Importer_NS::PROMOTE_POLYGON_TO_MULTIPOLYGON, Importer_NS::try_strptimes(), and run_benchmark_import::type.

Referenced by detect_column_types(), and find_best_sqltypes().

2914  {
2915  SQLTypes type = kTEXT;
2916  if (try_cast<double>(str)) {
2917  type = kDOUBLE;
2918  /*if (try_cast<bool>(str)) {
2919  type = kBOOLEAN;
2920  }*/
2921  if (try_cast<int16_t>(str)) {
2922  type = kSMALLINT;
2923  } else if (try_cast<int32_t>(str)) {
2924  type = kINT;
2925  } else if (try_cast<int64_t>(str)) {
2926  type = kBIGINT;
2927  } else if (try_cast<float>(str)) {
2928  type = kFLOAT;
2929  }
2930  }
2931 
2932  // check for geo types
2933  if (type == kTEXT) {
2934  // convert to upper case
2935  std::string str_upper_case = str;
2936  std::transform(
2937  str_upper_case.begin(), str_upper_case.end(), str_upper_case.begin(), ::toupper);
2938 
2939  // then test for leading words
2940  if (str_upper_case.find("POINT") == 0) {
2941  type = kPOINT;
2942  } else if (str_upper_case.find("LINESTRING") == 0) {
2943  type = kLINESTRING;
2944  } else if (str_upper_case.find("POLYGON") == 0) {
2946  type = kMULTIPOLYGON;
2947  } else {
2948  type = kPOLYGON;
2949  }
2950  } else if (str_upper_case.find("MULTIPOLYGON") == 0) {
2951  type = kMULTIPOLYGON;
2952  } else if (str_upper_case.find_first_not_of("0123456789ABCDEF") ==
2953  std::string::npos &&
2954  (str_upper_case.size() % 2) == 0) {
2955  // could be a WKB hex blob
2956  // we can't handle these yet
2957  // leave as TEXT for now
2958  // deliberate return here, as otherwise this would get matched as TIME
2959  // @TODO
2960  // implement WKB import
2961  return type;
2962  }
2963  }
2964 
2965  // check for time types
2966  if (type == kTEXT) {
2967  // @TODO
2968  // make these tests more robust so they don't match stuff they should not
2969  char* buf;
2970  buf = try_strptimes(str.c_str(), {"%Y-%m-%d", "%m/%d/%Y", "%d-%b-%y", "%d/%b/%Y"});
2971  if (buf) {
2972  type = kDATE;
2973  if (*buf == 'T' || *buf == ' ' || *buf == ':') {
2974  buf++;
2975  }
2976  }
2977  buf = try_strptimes(buf == nullptr ? str.c_str() : buf,
2978  {"%T %z", "%T", "%H%M%S", "%R"});
2979  if (buf) {
2980  if (type == kDATE) {
2981  type = kTIMESTAMP;
2982  } else {
2983  type = kTIME;
2984  }
2985  }
2986  }
2987 
2988  return type;
2989 }
Definition: sqltypes.h:50
SQLTypes
Definition: sqltypes.h:39
char * try_strptimes(const char *str, const std::vector< std::string > &formats)
Definition: Importer.cpp:2902
Definition: sqltypes.h:53
Definition: sqltypes.h:54
Definition: sqltypes.h:46
static constexpr bool PROMOTE_POLYGON_TO_MULTIPOLYGON
Definition: Importer.cpp:144

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::vector< EncodingType > Importer_NS::Detector::find_best_encodings ( const std::vector< std::vector< std::string >>::const_iterator &  row_begin,
const std::vector< std::vector< std::string >>::const_iterator &  row_end,
const std::vector< SQLTypes > &  best_types 
)
private

Definition at line 3097 of file Importer.cpp.

References file_path, IS_STRING, kENCODING_DICT, kENCODING_NONE, and raw_rows.

Referenced by find_best_sqltypes_and_headers().

3100  {
3101  if (raw_rows.size() < 1) {
3102  throw std::runtime_error("No rows found in: " +
3103  boost::filesystem::basename(file_path));
3104  }
3105  size_t num_cols = best_types.size();
3106  std::vector<EncodingType> best_encodes(num_cols, kENCODING_NONE);
3107  std::vector<size_t> num_rows_per_col(num_cols, 1);
3108  std::vector<std::unordered_set<std::string>> count_set(num_cols);
3109  for (auto row = row_begin; row != row_end; row++) {
3110  for (size_t col_idx = 0; col_idx < row->size() && col_idx < num_cols; col_idx++) {
3111  if (IS_STRING(best_types[col_idx])) {
3112  count_set[col_idx].insert(row->at(col_idx));
3113  num_rows_per_col[col_idx]++;
3114  }
3115  }
3116  }
3117  for (size_t col_idx = 0; col_idx < num_cols; col_idx++) {
3118  if (IS_STRING(best_types[col_idx])) {
3119  float uniqueRatio =
3120  static_cast<float>(count_set[col_idx].size()) / num_rows_per_col[col_idx];
3121  if (uniqueRatio < 0.75) {
3122  best_encodes[col_idx] = kENCODING_DICT;
3123  }
3124  }
3125  }
3126  return best_encodes;
3127 }
boost::filesystem::path file_path
Definition: Importer.h:683
std::vector< std::vector< std::string > > raw_rows
Definition: Importer.h:650
#define IS_STRING(T)
Definition: sqltypes.h:162

+ Here is the caller graph for this function:

void Importer_NS::Detector::find_best_sqltypes ( )
private

Definition at line 3044 of file Importer.cpp.

References best_sqltypes, Importer_NS::DataStreamSink::copy_params, and raw_rows.

Referenced by find_best_sqltypes(), and find_best_sqltypes_and_headers().

3044  {
3046 }
std::vector< std::vector< std::string > > raw_rows
Definition: Importer.h:650
std::vector< SQLTypes > best_sqltypes
Definition: Importer.h:652

+ Here is the caller graph for this function:

std::vector< SQLTypes > Importer_NS::Detector::find_best_sqltypes ( const std::vector< std::vector< std::string >> &  raw_rows,
const CopyParams copy_params 
)
private

Definition at line 3048 of file Importer.cpp.

References Importer_NS::DataStreamSink::copy_params, find_best_sqltypes(), and raw_rows.

3050  {
3051  return find_best_sqltypes(raw_rows.begin(), raw_rows.end(), copy_params);
3052 }
std::vector< std::vector< std::string > > raw_rows
Definition: Importer.h:650

+ Here is the call graph for this function:

std::vector< SQLTypes > Importer_NS::Detector::find_best_sqltypes ( const std::vector< std::vector< std::string >>::const_iterator &  row_begin,
const std::vector< std::vector< std::string >>::const_iterator &  row_end,
const CopyParams copy_params 
)
private

Definition at line 3054 of file Importer.cpp.

References detect_sqltype(), run_benchmark_import::end_time, file_path, kCHAR, kTEXT, more_restrictive_sqltype(), Importer_NS::CopyParams::null_str, raw_rows, and timeout.

3057  {
3058  if (raw_rows.size() < 1) {
3059  throw std::runtime_error("No rows found in: " +
3060  boost::filesystem::basename(file_path));
3061  }
3062  auto end_time = std::chrono::steady_clock::now() + timeout;
3063  size_t num_cols = raw_rows.front().size();
3064  std::vector<SQLTypes> best_types(num_cols, kCHAR);
3065  std::vector<size_t> non_null_col_counts(num_cols, 0);
3066  for (auto row = row_begin; row != row_end; row++) {
3067  while (best_types.size() < row->size() || non_null_col_counts.size() < row->size()) {
3068  best_types.push_back(kCHAR);
3069  non_null_col_counts.push_back(0);
3070  }
3071  for (size_t col_idx = 0; col_idx < row->size(); col_idx++) {
3072  // do not count nulls
3073  if (row->at(col_idx) == "" || !row->at(col_idx).compare(copy_params.null_str)) {
3074  continue;
3075  }
3076  SQLTypes t = detect_sqltype(row->at(col_idx));
3077  non_null_col_counts[col_idx]++;
3078  if (!more_restrictive_sqltype(best_types[col_idx], t)) {
3079  best_types[col_idx] = t;
3080  }
3081  }
3082  if (std::chrono::steady_clock::now() > end_time) {
3083  break;
3084  }
3085  }
3086  for (size_t col_idx = 0; col_idx < num_cols; col_idx++) {
3087  // if we don't have any non-null values for this column make it text to be
3088  // safe b/c that is least restrictive type
3089  if (non_null_col_counts[col_idx] == 0) {
3090  best_types[col_idx] = kTEXT;
3091  }
3092  }
3093 
3094  return best_types;
3095 }
std::chrono::duration< double > timeout
Definition: Importer.h:684
std::string null_str
Definition: CopyParams.h:47
SQLTypes
Definition: sqltypes.h:39
static SQLTypes detect_sqltype(const std::string &str)
Definition: Importer.cpp:2914
boost::filesystem::path file_path
Definition: Importer.h:683
Definition: sqltypes.h:53
static bool more_restrictive_sqltype(const SQLTypes a, const SQLTypes b)
Definition: Importer.cpp:2999
std::vector< std::vector< std::string > > raw_rows
Definition: Importer.h:650
Definition: sqltypes.h:42

+ Here is the call graph for this function:

void Importer_NS::Detector::find_best_sqltypes_and_headers ( )
private

Definition at line 3021 of file Importer.cpp.

References Importer_NS::AUTODETECT, best_encodings, best_sqltypes, Importer_NS::DataStreamSink::copy_params, detect_column_types(), detect_headers(), find_best_encodings(), find_best_sqltypes(), Importer_NS::HAS_HEADER, Importer_NS::CopyParams::has_header, has_headers, Importer_NS::NO_HEADER, and raw_rows.

Referenced by init().

3021  {
3023  best_encodings =
3024  find_best_encodings(raw_rows.begin() + 1, raw_rows.end(), best_sqltypes);
3025  std::vector<SQLTypes> head_types = detect_column_types(raw_rows.at(0));
3026  switch (copy_params.has_header) {
3028  has_headers = detect_headers(head_types, best_sqltypes);
3029  if (has_headers) {
3031  } else {
3033  }
3034  break;
3036  has_headers = false;
3037  break;
3039  has_headers = true;
3040  break;
3041  }
3042 }
std::vector< EncodingType > find_best_encodings(const std::vector< std::vector< std::string >>::const_iterator &row_begin, const std::vector< std::vector< std::string >>::const_iterator &row_end, const std::vector< SQLTypes > &best_types)
Definition: Importer.cpp:3097
std::vector< EncodingType > best_encodings
Definition: Importer.h:653
ImportHeaderRow has_header
Definition: CopyParams.h:48
std::vector< SQLTypes > detect_column_types(const std::vector< std::string > &row)
Definition: Importer.cpp:2991
std::vector< std::vector< std::string > > raw_rows
Definition: Importer.h:650
bool detect_headers(const std::vector< SQLTypes > &first_types, const std::vector< SQLTypes > &rest_types)
Definition: Importer.cpp:3133
std::vector< SQLTypes > best_sqltypes
Definition: Importer.h:652

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::vector< std::string > Importer_NS::Detector::get_headers ( )

Definition at line 3156 of file Importer.cpp.

References best_sqltypes, has_headers, raw_rows, and to_string().

Referenced by MapDHandler::detect_column_types().

3156  {
3157  std::vector<std::string> headers(best_sqltypes.size());
3158  for (size_t i = 0; i < best_sqltypes.size(); i++) {
3159  if (has_headers && i < raw_rows[0].size()) {
3160  headers[i] = raw_rows[0][i];
3161  } else {
3162  headers[i] = "column_" + std::to_string(i + 1);
3163  }
3164  }
3165  return headers;
3166 }
std::string to_string(char const *&&v)
std::vector< std::vector< std::string > > raw_rows
Definition: Importer.h:650
std::vector< SQLTypes > best_sqltypes
Definition: Importer.h:652

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::vector< std::vector< std::string > > Importer_NS::Detector::get_sample_rows ( size_t  n)

Definition at line 3148 of file Importer.cpp.

References has_headers, and raw_rows.

Referenced by MapDHandler::detect_column_types().

3148  {
3149  n = std::min(n, raw_rows.size());
3150  size_t offset = (has_headers && raw_rows.size() > 1) ? 1 : 0;
3151  std::vector<std::vector<std::string>> sample_rows(raw_rows.begin() + offset,
3152  raw_rows.begin() + n);
3153  return sample_rows;
3154 }
std::vector< std::vector< std::string > > raw_rows
Definition: Importer.h:650

+ Here is the caller graph for this function:

ImportStatus Importer_NS::Detector::importDelimited ( const std::string &  file_path,
const bool  decompressed 
)
overrideprivatevirtual

Implements Importer_NS::DataStreamSink.

Definition at line 2791 of file Importer.cpp.

References Importer_NS::DataStreamSink::copy_params, run_benchmark_import::end_time, Importer_NS::DataStreamSink::import_status, line1, Importer_NS::CopyParams::line_delim, Importer_NS::DataStreamSink::load_failed, Importer_NS::ImportStatus::load_truncated, Importer_NS::DataStreamSink::p_file, raw_data, Importer_NS::ImportStatus::rows_completed, and timeout.

2792  {
2793  if (!p_file) {
2794  p_file = fopen(file_path.c_str(), "rb");
2795  }
2796  if (!p_file) {
2797  throw std::runtime_error("failed to open file '" + file_path +
2798  "': " + strerror(errno));
2799  }
2800 
2801  // somehow clang does not support ext/stdio_filebuf.h, so
2802  // need to diy readline with customized copy_params.line_delim...
2803  std::string line;
2804  line.reserve(1 * 1024 * 1024);
2805  auto end_time = std::chrono::steady_clock::now() +
2806  timeout * (boost::istarts_with(file_path, "s3://") ? 3 : 1);
2807  try {
2808  while (!feof(p_file)) {
2809  int c;
2810  size_t n = 0;
2811  while (EOF != (c = fgetc(p_file)) && copy_params.line_delim != c) {
2812  if (n++ >= line.capacity()) {
2813  break;
2814  }
2815  line += c;
2816  }
2817  if (0 == n) {
2818  break;
2819  }
2820  // remember the first line, which is possibly a header line, to
2821  // ignore identical header line(s) in 2nd+ files of a archive;
2822  // otherwise, 2nd+ header may be mistaken as an all-string row
2823  // and so be final column types.
2824  if (line1.empty()) {
2825  line1 = line;
2826  } else if (line == line1) {
2827  line.clear();
2828  continue;
2829  }
2830 
2831  raw_data += line;
2833  line.clear();
2835  if (std::chrono::steady_clock::now() > end_time) {
2836  if (import_status.rows_completed > 10000) {
2837  break;
2838  }
2839  }
2840  }
2841  } catch (std::exception& e) {
2842  }
2843 
2844  // as if load truncated
2846  load_failed = true;
2847 
2848  fclose(p_file);
2849  p_file = nullptr;
2850  return import_status;
2851 }
std::chrono::duration< double > timeout
Definition: Importer.h:684
std::string line1
Definition: Importer.h:685
std::string raw_data
Definition: Importer.h:682
boost::filesystem::path file_path
Definition: Importer.h:683
ImportStatus import_status
Definition: Importer.h:631
void Importer_NS::Detector::init ( )
private

Definition at line 2785 of file Importer.cpp.

References detect_row_delimiter(), find_best_sqltypes_and_headers(), and split_raw_data().

Referenced by Detector().

2785  {
2787  split_raw_data();
2789 }
void find_best_sqltypes_and_headers()
Definition: Importer.cpp:3021

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

bool Importer_NS::Detector::more_restrictive_sqltype ( const SQLTypes  a,
const SQLTypes  b 
)
staticprivate

Definition at line 2999 of file Importer.cpp.

References kBIGINT, kBOOLEAN, kCHAR, kDATE, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, and kTIMESTAMP.

Referenced by find_best_sqltypes().

2999  {
3000  static std::array<int, kSQLTYPE_LAST> typeorder;
3001  typeorder[kCHAR] = 0;
3002  typeorder[kBOOLEAN] = 2;
3003  typeorder[kSMALLINT] = 3;
3004  typeorder[kINT] = 4;
3005  typeorder[kBIGINT] = 5;
3006  typeorder[kFLOAT] = 6;
3007  typeorder[kDOUBLE] = 7;
3008  typeorder[kTIMESTAMP] = 8;
3009  typeorder[kTIME] = 9;
3010  typeorder[kDATE] = 10;
3011  typeorder[kPOINT] = 11;
3012  typeorder[kLINESTRING] = 11;
3013  typeorder[kPOLYGON] = 11;
3014  typeorder[kMULTIPOLYGON] = 11;
3015  typeorder[kTEXT] = 12;
3016 
3017  // note: b < a instead of a < b because the map is ordered most to least restrictive
3018  return typeorder[b] < typeorder[a];
3019 }
Definition: sqltypes.h:50
Definition: sqltypes.h:53
Definition: sqltypes.h:54
Definition: sqltypes.h:42
Definition: sqltypes.h:46

+ Here is the caller graph for this function:

void Importer_NS::Detector::read_file ( )
private

Definition at line 2853 of file Importer.cpp.

References Importer_NS::DataStreamSink::archivePlumber().

Referenced by Detector().

2853  {
2854  // this becomes analogous to Importer::import()
2856 }
ImportStatus archivePlumber()
Definition: Importer.cpp:3204

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Importer_NS::Detector::split_raw_data ( )
private

Definition at line 2867 of file Importer.cpp.

References Importer_NS::DataStreamSink::copy_params, Importer_NS::DelimitedParserUtils::get_row(), raw_data, raw_rows, and Importer_NS::CopyParams::threads.

Referenced by init().

2867  {
2868  const char* buf = raw_data.c_str();
2869  const char* buf_end = buf + raw_data.size();
2870  bool try_single_thread = false;
2871  for (const char* p = buf; p < buf_end; p++) {
2872  std::vector<std::string> row;
2874  p, buf_end, buf_end, copy_params, nullptr, row, try_single_thread);
2875  raw_rows.push_back(row);
2876  if (try_single_thread) {
2877  break;
2878  }
2879  }
2880  if (try_single_thread) {
2881  copy_params.threads = 1;
2882  raw_rows.clear();
2883  for (const char* p = buf; p < buf_end; p++) {
2884  std::vector<std::string> row;
2886  p, buf_end, buf_end, copy_params, nullptr, row, try_single_thread);
2887  raw_rows.push_back(row);
2888  }
2889  }
2890 }
static const char * get_row(const char *buf, const char *buf_end, const char *entire_buf_end, const Importer_NS::CopyParams &copy_params, const bool *is_array, std::vector< std::string > &row, bool &try_single_thread)
Parses the first row in the given buffer and inserts fields into given vector.
std::string raw_data
Definition: Importer.h:682
std::vector< std::vector< std::string > > raw_rows
Definition: Importer.h:650

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Member Data Documentation

std::vector<EncodingType> Importer_NS::Detector::best_encodings

Definition at line 653 of file Importer.h.

Referenced by MapDHandler::detect_column_types(), and find_best_sqltypes_and_headers().

std::vector<SQLTypes> Importer_NS::Detector::best_sqltypes
boost::filesystem::path Importer_NS::Detector::file_path
private

Definition at line 683 of file Importer.h.

Referenced by detect_row_delimiter(), find_best_encodings(), and find_best_sqltypes().

bool Importer_NS::Detector::has_headers = false
std::string Importer_NS::Detector::line1
private

Definition at line 685 of file Importer.h.

Referenced by importDelimited().

std::string Importer_NS::Detector::raw_data
private

Definition at line 682 of file Importer.h.

Referenced by importDelimited(), and split_raw_data().

std::vector<std::vector<std::string> > Importer_NS::Detector::raw_rows
std::chrono::duration<double> Importer_NS::Detector::timeout {1}
private

Definition at line 684 of file Importer.h.

Referenced by find_best_sqltypes(), and importDelimited().


The documentation for this class was generated from the following files: