OmniSciDB  2c44a3935d
RowToColumnLoader.cpp File Reference

Based on StreamInsert code but using binary columnar format for inserting a stream of rows with optional transformations from stdin to a MapD table. More...

+ Include dependency graph for RowToColumnLoader.cpp:

Go to the source code of this file.

Functions

SQLTypes get_sql_types (const TColumnType &ct)
 
SQLTypeInfo create_sql_type_info_from_col_type (const TColumnType &ct)
 
SQLTypeInfo create_array_sql_type_info_from_col_type (const TColumnType &ct)
 
void remove_partial_row (size_t failed_column, std::vector< SQLTypeInfo > column_type_info_vector, std::vector< TColumn > &input_col_vec)
 
void populate_TColumn (TStringValue ts, SQLTypeInfo column_type_info, TColumn &input_col, const import_export::CopyParams &copy_params)
 

Detailed Description

Based on StreamInsert code but using binary columnar format for inserting a stream of rows with optional transformations from stdin to a MapD table.

Author
Michael micha.nosp@m.el@m.nosp@m.apd.c.nosp@m.om Copyright (c) 2017 MapD Technologies, Inc. All rights reserved.

Definition in file RowToColumnLoader.cpp.

Function Documentation

◆ create_array_sql_type_info_from_col_type()

SQLTypeInfo create_array_sql_type_info_from_col_type ( const TColumnType &  ct)

Definition at line 103 of file RowToColumnLoader.cpp.

References get_sql_types(), kENCODING_NONE, and kNULLT.

Referenced by RowToColumnLoader::RowToColumnLoader().

103  {
104  return SQLTypeInfo(get_sql_types(ct),
105  ct.col_type.precision,
106  ct.col_type.scale,
107  ct.col_type.nullable,
109  0,
111 }
SQLTypes get_sql_types(const TColumnType &ct)
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ create_sql_type_info_from_col_type()

SQLTypeInfo create_sql_type_info_from_col_type ( const TColumnType &  ct)

Definition at line 77 of file RowToColumnLoader.cpp.

References get_sql_types(), kARRAY, kENCODING_NONE, and kNULLT.

Referenced by RowToColumnLoader::RowToColumnLoader().

77  {
78  if (ct.col_type.is_array) {
80  ct.col_type.precision,
81  ct.col_type.scale,
82  ct.col_type.nullable,
84  0,
85  get_sql_types(ct));
86  } else {
87  // normal column
88  // NOTE(se)
89  // for geo types, the values inserted for the other fields
90  // may not be valid, but only the type field is ever used
91  return SQLTypeInfo(get_sql_types(ct),
92  ct.col_type.precision,
93  ct.col_type.scale,
94  ct.col_type.nullable,
96  0,
98  }
99 }
SQLTypes get_sql_types(const TColumnType &ct)
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ get_sql_types()

SQLTypes get_sql_types ( const TColumnType &  ct)

Definition at line 32 of file RowToColumnLoader.cpp.

References logger::FATAL, kBIGINT, kBOOLEAN, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNULLT, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, and LOG.

Referenced by create_array_sql_type_info_from_col_type(), and create_sql_type_info_from_col_type().

32  {
33  switch (ct.col_type.type) {
34  case TDatumType::BIGINT:
35  return SQLTypes::kBIGINT;
36  case TDatumType::BOOL:
37  return SQLTypes::kBOOLEAN;
38  case TDatumType::DATE:
39  return SQLTypes::kDATE;
40  case TDatumType::DECIMAL:
41  return SQLTypes::kDECIMAL;
42  case TDatumType::DOUBLE:
43  return SQLTypes::kDOUBLE;
44  case TDatumType::FLOAT:
45  return SQLTypes::kFLOAT;
46  case TDatumType::INT:
47  return SQLTypes::kINT;
48  case TDatumType::STR:
49  // Tdataum is lossy here so need to look at precision to see if it was defined
50  if (ct.col_type.precision == 0) {
51  return SQLTypes::kTEXT;
52  } else {
53  return SQLTypes::kVARCHAR;
54  }
55  case TDatumType::TIME:
56  return SQLTypes::kTIME;
57  case TDatumType::TIMESTAMP:
58  return SQLTypes::kTIMESTAMP;
59  case TDatumType::SMALLINT:
60  return SQLTypes::kSMALLINT;
61  case TDatumType::TINYINT:
62  return SQLTypes::kTINYINT;
63  case TDatumType::POINT:
64  return SQLTypes::kPOINT;
65  case TDatumType::LINESTRING:
66  return SQLTypes::kLINESTRING;
67  case TDatumType::POLYGON:
68  return SQLTypes::kPOLYGON;
69  case TDatumType::MULTIPOLYGON:
71  default:
72  LOG(FATAL) << "Unsupported TColumnType found, should not be possible";
73  return SQLTypes::kNULLT; // satisfy return-type warning
74  }
75 }
Definition: sqltypes.h:50
#define LOG(tag)
Definition: Logger.h:188
Definition: sqltypes.h:53
Definition: sqltypes.h:54
Definition: sqltypes.h:46
+ Here is the caller graph for this function:

◆ populate_TColumn()

void populate_TColumn ( TStringValue  ts,
SQLTypeInfo  column_type_info,
TColumn &  input_col,
const import_export::CopyParams copy_params 
)

Definition at line 177 of file RowToColumnLoader.cpp.

References Datum::bigintval, Datum::doubleval, logger::FATAL, Datum::floatval, SQLTypeInfo::get_precision(), SQLTypeInfo::get_type(), Datum::intval, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, LOG, Datum::smallintval, StringToDatum(), and Datum::tinyintval.

Referenced by RowToColumnLoader::convert_string_to_column().

180  {
181  // create datum and push data to column structure from row data
182  switch (column_type_info.get_type()) {
183  case SQLTypes::kARRAY:
184  LOG(FATAL) << "Trying to process ARRAY at item level something is wrong";
185  break;
186  case SQLTypes::kTEXT:
187  case SQLTypes::kCHAR:
188  case SQLTypes::kVARCHAR:
189  case SQLTypes::kPOINT:
191  case SQLTypes::kPOLYGON:
193  if (ts.is_null) {
194  input_col.nulls.push_back(true);
195  input_col.data.str_col.emplace_back("");
196 
197  } else {
198  input_col.nulls.push_back(false);
199  switch (column_type_info.get_type()) {
200  case SQLTypes::kCHAR:
201  case SQLTypes::kVARCHAR:
202  input_col.data.str_col.push_back(
203  ts.str_val.substr(0, column_type_info.get_precision()));
204  break;
205  case SQLTypes::kTEXT:
206  case SQLTypes::kPOINT:
208  case SQLTypes::kPOLYGON:
210  input_col.data.str_col.push_back(ts.str_val);
211  break;
212  default:
213  LOG(FATAL) << " trying to process a STRING transport type not handled "
214  << column_type_info.get_type();
215  }
216  }
217  break;
218  case SQLTypes::kINT:
219  case SQLTypes::kBIGINT:
220  case SQLTypes::kSMALLINT:
221  case SQLTypes::kTINYINT:
222  case SQLTypes::kDATE:
223  case SQLTypes::kTIME:
225  case SQLTypes::kNUMERIC:
226  case SQLTypes::kDECIMAL:
227  case SQLTypes::kBOOLEAN:
228  if (ts.is_null) {
229  input_col.nulls.push_back(true);
230  input_col.data.int_col.push_back(0);
231  } else {
232  input_col.nulls.push_back(false);
233  Datum d = StringToDatum(ts.str_val, column_type_info);
234  switch (column_type_info.get_type()) {
235  case SQLTypes::kINT:
236  case SQLTypes::kBOOLEAN:
237  input_col.data.int_col.push_back(d.intval);
238  break;
239  case SQLTypes::kBIGINT:
240  case SQLTypes::kNUMERIC:
241  case SQLTypes::kDECIMAL:
242  input_col.data.int_col.push_back(d.bigintval);
243  break;
244  case SQLTypes::kSMALLINT:
245  input_col.data.int_col.push_back(d.smallintval);
246  break;
247  case SQLTypes::kTINYINT:
248  input_col.data.int_col.push_back(d.tinyintval);
249  break;
250  case SQLTypes::kDATE:
251  case SQLTypes::kTIME:
253  input_col.data.int_col.push_back(d.bigintval);
254  break;
255  default:
256  LOG(FATAL) << " trying to process an INT transport type not handled "
257  << column_type_info.get_type();
258  }
259  }
260  break;
261  case SQLTypes::kFLOAT:
262  case SQLTypes::kDOUBLE:
263  if (ts.is_null) {
264  input_col.nulls.push_back(true);
265  input_col.data.real_col.push_back(0);
266 
267  } else {
268  input_col.nulls.push_back(false);
269  Datum d = StringToDatum(ts.str_val, column_type_info);
270  switch (column_type_info.get_type()) {
271  case SQLTypes::kFLOAT:
272  input_col.data.real_col.push_back(d.floatval);
273  break;
274  case SQLTypes::kDOUBLE:
275  input_col.data.real_col.push_back(d.doubleval);
276  break;
277  default:
278  LOG(FATAL) << " trying to process a REAL transport type not handled "
279  << column_type_info.get_type();
280  }
281  }
282  break;
283  default:
284  LOG(FATAL) << "Trying to process an unsupported datatype, should be impossible";
285  }
286 }
int8_t tinyintval
Definition: sqltypes.h:133
int get_precision() const
Definition: sqltypes.h:260
Definition: sqltypes.h:50
#define LOG(tag)
Definition: Logger.h:188
int32_t intval
Definition: sqltypes.h:135
float floatval
Definition: sqltypes.h:137
int64_t bigintval
Definition: sqltypes.h:136
int16_t smallintval
Definition: sqltypes.h:134
Datum StringToDatum(std::string_view s, SQLTypeInfo &ti)
Definition: Datum.cpp:122
Definition: sqltypes.h:53
Definition: sqltypes.h:54
Definition: sqltypes.h:42
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:257
Definition: sqltypes.h:46
double doubleval
Definition: sqltypes.h:138
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ remove_partial_row()

void remove_partial_row ( size_t  failed_column,
std::vector< SQLTypeInfo column_type_info_vector,
std::vector< TColumn > &  input_col_vec 
)

Definition at line 131 of file RowToColumnLoader.cpp.

References logger::FATAL, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, and LOG.

Referenced by RowToColumnLoader::convert_string_to_column().

133  {
134  for (size_t idx = 0; idx < failed_column; idx++) {
135  switch (column_type_info_vector[idx].get_type()) {
136  case SQLTypes::kARRAY:
137  input_col_vec[idx].nulls.pop_back();
138  input_col_vec[idx].data.arr_col.pop_back();
139  break;
140  case SQLTypes::kTEXT:
141  case SQLTypes::kCHAR:
142  case SQLTypes::kVARCHAR:
143  input_col_vec[idx].nulls.pop_back();
144  input_col_vec[idx].data.str_col.pop_back();
145  break;
146  case SQLTypes::kTINYINT:
147  case SQLTypes::kINT:
148  case SQLTypes::kBIGINT:
149  case SQLTypes::kSMALLINT:
150  case SQLTypes::kDATE:
151  case SQLTypes::kTIME:
153  case SQLTypes::kNUMERIC:
154  case SQLTypes::kDECIMAL:
155  case SQLTypes::kBOOLEAN:
156  input_col_vec[idx].nulls.pop_back();
157  input_col_vec[idx].data.int_col.pop_back();
158  break;
159  case SQLTypes::kFLOAT:
160  case SQLTypes::kDOUBLE:
161  input_col_vec[idx].nulls.pop_back();
162  input_col_vec[idx].data.real_col.pop_back();
163  break;
164  case SQLTypes::kPOINT:
166  case SQLTypes::kPOLYGON:
168  input_col_vec[idx].nulls.pop_back();
169  input_col_vec[idx].data.str_col.pop_back();
170  break;
171  default:
172  LOG(FATAL) << "Trying to process an unsupported datatype, should be impossible";
173  }
174  }
175 }
Definition: sqltypes.h:50
#define LOG(tag)
Definition: Logger.h:188
Definition: sqltypes.h:53
Definition: sqltypes.h:54
Definition: sqltypes.h:42
Definition: sqltypes.h:46
+ Here is the caller graph for this function: