OmniSciDB  04ee39c94c
RowToColumnLoader.cpp File Reference

Based on StreamInsert code but using binary columnar format for inserting a stream of rows with optional transformations from stdin to a MapD table. More...

#include "RowToColumnLoader.h"
#include "Shared/Logger.h"
+ Include dependency graph for RowToColumnLoader.cpp:

Go to the source code of this file.

Functions

SQLTypes get_sql_types (const TColumnType &ct)
 
SQLTypeInfo create_sql_type_info_from_col_type (const TColumnType &ct)
 
SQLTypeInfo create_array_sql_type_info_from_col_type (const TColumnType &ct)
 
void remove_partial_row (size_t failed_column, std::vector< SQLTypeInfo > column_type_info_vector, std::vector< TColumn > &input_col_vec)
 
void populate_TColumn (TStringValue ts, SQLTypeInfo column_type_info, TColumn &input_col, const Importer_NS::CopyParams &copy_params)
 

Detailed Description

Based on StreamInsert code but using binary columnar format for inserting a stream of rows with optional transformations from stdin to a MapD table.

Author
Michael micha.nosp@m.el@m.nosp@m.apd.c.nosp@m.om Copyright (c) 2017 MapD Technologies, Inc. All rights reserved.

Definition in file RowToColumnLoader.cpp.

Function Documentation

◆ create_array_sql_type_info_from_col_type()

SQLTypeInfo create_array_sql_type_info_from_col_type ( const TColumnType &  ct)

Definition at line 89 of file RowToColumnLoader.cpp.

References get_sql_types(), kENCODING_NONE, and kNULLT.

Referenced by RowToColumnLoader::RowToColumnLoader().

89  {
90  return SQLTypeInfo(get_sql_types(ct),
91  ct.col_type.precision,
92  ct.col_type.scale,
93  ct.col_type.nullable,
95  0,
97 }
SQLTypeInfoCore< ArrayContextTypeSizer, ExecutorTypePackaging, DateTimeFacilities > SQLTypeInfo
Definition: sqltypes.h:823
SQLTypes get_sql_types(const TColumnType &ct)
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ create_sql_type_info_from_col_type()

SQLTypeInfo create_sql_type_info_from_col_type ( const TColumnType &  ct)

Definition at line 66 of file RowToColumnLoader.cpp.

References get_sql_types(), kARRAY, kENCODING_NONE, and kNULLT.

Referenced by RowToColumnLoader::RowToColumnLoader().

66  {
67  if (ct.col_type.is_array) {
69  ct.col_type.precision,
70  ct.col_type.scale,
71  ct.col_type.nullable,
73  0,
74  get_sql_types(ct));
75  } else {
76  // normal column
77  return SQLTypeInfo(get_sql_types(ct),
78  ct.col_type.precision,
79  ct.col_type.scale,
80  ct.col_type.nullable,
82  0,
84  }
85 }
SQLTypeInfoCore< ArrayContextTypeSizer, ExecutorTypePackaging, DateTimeFacilities > SQLTypeInfo
Definition: sqltypes.h:823
SQLTypes get_sql_types(const TColumnType &ct)
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ get_sql_types()

SQLTypes get_sql_types ( const TColumnType &  ct)

Definition at line 31 of file RowToColumnLoader.cpp.

References logger::FATAL, kBIGINT, kBOOLEAN, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kNULLT, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kVARCHAR, and LOG.

Referenced by create_array_sql_type_info_from_col_type(), and create_sql_type_info_from_col_type().

31  {
32  switch (ct.col_type.type) {
33  case TDatumType::BIGINT:
34  return SQLTypes::kBIGINT;
35  case TDatumType::BOOL:
36  return SQLTypes::kBOOLEAN;
37  case TDatumType::DATE:
38  return SQLTypes::kDATE;
39  case TDatumType::DECIMAL:
40  return SQLTypes::kDECIMAL;
41  case TDatumType::DOUBLE:
42  return SQLTypes::kDOUBLE;
43  case TDatumType::FLOAT:
44  return SQLTypes::kFLOAT;
45  case TDatumType::INT:
46  return SQLTypes::kINT;
47  case TDatumType::STR:
48  // Tdataum is lossy here so need to look at precision to see if it was defined
49  if (ct.col_type.precision == 0) {
50  return SQLTypes::kTEXT;
51  } else {
52  return SQLTypes::kVARCHAR;
53  }
54  case TDatumType::TIME:
55  return SQLTypes::kTIME;
56  case TDatumType::TIMESTAMP:
57  return SQLTypes::kTIMESTAMP;
58  case TDatumType::SMALLINT:
59  return SQLTypes::kSMALLINT;
60  default:
61  LOG(FATAL) << "Unsupported TColumnType found, should not be possible";
62  return SQLTypes::kNULLT; // satisfy return-type warning
63  }
64 }
Definition: sqltypes.h:51
#define LOG(tag)
Definition: Logger.h:182
Definition: sqltypes.h:54
Definition: sqltypes.h:55
Definition: sqltypes.h:47
+ Here is the caller graph for this function:

◆ populate_TColumn()

void populate_TColumn ( TStringValue  ts,
SQLTypeInfo  column_type_info,
TColumn &  input_col,
const Importer_NS::CopyParams copy_params 
)

Definition at line 155 of file RowToColumnLoader.cpp.

References Datum::bigintval, anonymous_namespace{ImportTest.cpp}::d(), Datum::doubleval, logger::FATAL, Datum::floatval, SQLTypeInfoCore< TYPE_FACET_PACK >::get_precision(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_type(), Datum::intval, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kNUMERIC, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kVARCHAR, LOG, Datum::smallintval, and StringToDatum().

Referenced by RowToColumnLoader::convert_string_to_column().

158  {
159  // create datum and push data to column structure from row data
160  switch (column_type_info.get_type()) {
161  case SQLTypes::kARRAY:
162  LOG(FATAL) << "Trying to process ARRAY at item level something is wrong";
163  break;
164  case SQLTypes::kTEXT:
165  case SQLTypes::kCHAR:
166  case SQLTypes::kVARCHAR:
167  if (ts.is_null) {
168  input_col.nulls.push_back(true);
169  input_col.data.str_col.emplace_back("");
170 
171  } else {
172  input_col.nulls.push_back(false);
173  switch (column_type_info.get_type()) {
174  case SQLTypes::kCHAR:
175  case SQLTypes::kVARCHAR:
176  input_col.data.str_col.push_back(
177  ts.str_val.substr(0, column_type_info.get_precision()));
178  break;
179  case SQLTypes::kTEXT:
180 
181  input_col.data.str_col.push_back(ts.str_val);
182  break;
183  default:
184  LOG(FATAL) << " trying to process a STRING transport type not handled "
185  << column_type_info.get_type();
186  }
187  }
188  break;
189  case SQLTypes::kINT:
190  case SQLTypes::kBIGINT:
191  case SQLTypes::kSMALLINT:
192  case SQLTypes::kDATE:
193  case SQLTypes::kTIME:
195  case SQLTypes::kNUMERIC:
196  case SQLTypes::kDECIMAL:
197  case SQLTypes::kBOOLEAN:
198  if (ts.is_null) {
199  input_col.nulls.push_back(true);
200  input_col.data.int_col.push_back(0);
201  } else {
202  input_col.nulls.push_back(false);
203  Datum d = StringToDatum(ts.str_val, column_type_info);
204  switch (column_type_info.get_type()) {
205  case SQLTypes::kINT:
206  case SQLTypes::kBOOLEAN:
207  input_col.data.int_col.push_back(d.intval);
208  break;
209  case SQLTypes::kBIGINT:
210  case SQLTypes::kNUMERIC:
211  case SQLTypes::kDECIMAL:
212  input_col.data.int_col.push_back(d.bigintval);
213  break;
214  case SQLTypes::kSMALLINT:
215  input_col.data.int_col.push_back(d.smallintval);
216  break;
217  case SQLTypes::kDATE:
218  case SQLTypes::kTIME:
220  input_col.data.int_col.push_back(d.bigintval);
221  break;
222  default:
223  LOG(FATAL) << " trying to process an INT transport type not handled "
224  << column_type_info.get_type();
225  }
226  }
227  break;
228  case SQLTypes::kFLOAT:
229  case SQLTypes::kDOUBLE:
230  if (ts.is_null) {
231  input_col.nulls.push_back(true);
232  input_col.data.real_col.push_back(0);
233 
234  } else {
235  input_col.nulls.push_back(false);
236  Datum d = StringToDatum(ts.str_val, column_type_info);
237  switch (column_type_info.get_type()) {
238  case SQLTypes::kFLOAT:
239  input_col.data.real_col.push_back(d.floatval);
240  break;
241  case SQLTypes::kDOUBLE:
242  input_col.data.real_col.push_back(d.doubleval);
243  break;
244  default:
245  LOG(FATAL) << " trying to process a REAL transport type not handled "
246  << column_type_info.get_type();
247  }
248  }
249  break;
250  default:
251  LOG(FATAL) << "Trying to process an unsupported datatype, should be impossible";
252  }
253 }
void d(const SQLTypes expected_type, const std::string &str)
Definition: ImportTest.cpp:268
int get_precision() const
Definition: sqltypes.h:326
Definition: sqltypes.h:51
#define LOG(tag)
Definition: Logger.h:182
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:323
int32_t intval
Definition: sqltypes.h:125
float floatval
Definition: sqltypes.h:127
int64_t bigintval
Definition: sqltypes.h:126
int16_t smallintval
Definition: sqltypes.h:124
Definition: sqltypes.h:54
Definition: sqltypes.h:55
Datum StringToDatum(const std::string &s, SQLTypeInfo &ti)
Definition: Datum.cpp:90
Definition: sqltypes.h:43
Definition: sqltypes.h:47
double doubleval
Definition: sqltypes.h:128
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ remove_partial_row()

void remove_partial_row ( size_t  failed_column,
std::vector< SQLTypeInfo column_type_info_vector,
std::vector< TColumn > &  input_col_vec 
)

Definition at line 117 of file RowToColumnLoader.cpp.

References logger::FATAL, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kNUMERIC, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kVARCHAR, and LOG.

Referenced by RowToColumnLoader::convert_string_to_column().

119  {
120  for (size_t idx = 0; idx < failed_column; idx++) {
121  switch (column_type_info_vector[idx].get_type()) {
122  case SQLTypes::kARRAY:
123  input_col_vec[idx].nulls.pop_back();
124  input_col_vec[idx].data.arr_col.pop_back();
125  break;
126  case SQLTypes::kTEXT:
127  case SQLTypes::kCHAR:
128  case SQLTypes::kVARCHAR:
129  input_col_vec[idx].nulls.pop_back();
130  input_col_vec[idx].data.str_col.pop_back();
131  break;
132  case SQLTypes::kINT:
133  case SQLTypes::kBIGINT:
134  case SQLTypes::kSMALLINT:
135  case SQLTypes::kDATE:
136  case SQLTypes::kTIME:
138  case SQLTypes::kNUMERIC:
139  case SQLTypes::kDECIMAL:
140  case SQLTypes::kBOOLEAN:
141  input_col_vec[idx].nulls.pop_back();
142  input_col_vec[idx].data.int_col.pop_back();
143  break;
144  case SQLTypes::kFLOAT:
145  case SQLTypes::kDOUBLE:
146  input_col_vec[idx].nulls.pop_back();
147  input_col_vec[idx].data.real_col.pop_back();
148  break;
149  default:
150  LOG(FATAL) << "Trying to process an unsupported datatype, should be impossible";
151  }
152  }
153 }
Definition: sqltypes.h:51
#define LOG(tag)
Definition: Logger.h:182
Definition: sqltypes.h:54
Definition: sqltypes.h:55
Definition: sqltypes.h:43
Definition: sqltypes.h:47
+ Here is the caller graph for this function: