OmniSciDB  c07336695a
ParserNode.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
25 #include "ParserNode.h"
26 #include <rapidjson/document.h>
27 #include <rapidjson/stringbuffer.h>
28 #include <rapidjson/writer.h>
29 #include <boost/algorithm/string.hpp>
30 #include <boost/algorithm/string/predicate.hpp>
31 #include <boost/filesystem.hpp>
32 #include <boost/function.hpp>
33 #include <cassert>
34 #include <cmath>
35 #include <limits>
36 #include <random>
37 #include <stdexcept>
38 #include <type_traits>
39 #include <typeinfo>
40 
41 #include "../Analyzer/RangeTableEntry.h"
42 #include "../Catalog/Catalog.h"
43 #include "../Catalog/SharedDictionaryValidator.h"
44 #include "../Fragmenter/InsertOrderFragmenter.h"
45 #include "../Fragmenter/TargetValueConvertersFactories.h"
46 #include "../Import/Importer.h"
47 #include "../Planner/Planner.h"
48 #include "../QueryEngine/CalciteAdapter.h"
49 #include "../QueryEngine/Execute.h"
50 #include "../QueryEngine/ExtensionFunctionsWhitelist.h"
51 #include "../QueryEngine/RelAlgExecutor.h"
52 #include "../Shared/TimeGM.h"
53 #include "../Shared/geo_types.h"
54 #include "../Shared/mapd_glob.h"
55 #include "../Shared/measure.h"
56 #include "../Shared/shard_key.h"
57 #include "LockMgr/LockMgr.h"
58 #include "LockMgr/TableLockMgr.h"
59 #include "ReservedKeywords.h"
60 #include "gen-cpp/CalciteServer.h"
61 #include "parser.h"
62 
63 size_t g_leaf_count{0};
65 
66 using namespace Lock_Namespace;
68 using namespace std::string_literals;
69 
70 using TableDefFuncPtr = boost::function<void(TableDescriptor&,
71  const NameValueAssign*,
72  const std::list<ColumnDescriptor>& columns)>;
73 
74 namespace Importer_NS {
75 std::vector<uint8_t> compress_coords(std::vector<double>& coords, const SQLTypeInfo& ti);
76 } // namespace Importer_NS
77 
78 namespace Parser {
79 std::shared_ptr<Analyzer::Expr> NullLiteral::analyze(
80  const Catalog_Namespace::Catalog& catalog,
82  TlistRefType allow_tlist_ref) const {
83  return makeExpr<Analyzer::Constant>(kNULLT, true);
84 }
85 
86 std::shared_ptr<Analyzer::Expr> StringLiteral::analyze(
87  const Catalog_Namespace::Catalog& catalog,
89  TlistRefType allow_tlist_ref) const {
90  return analyzeValue(*stringval);
91 }
92 
93 std::shared_ptr<Analyzer::Expr> StringLiteral::analyzeValue(
94  const std::string& stringval) {
95  SQLTypeInfo ti(kVARCHAR, stringval.length(), 0, true);
96  Datum d;
97  d.stringval = new std::string(stringval);
98  return makeExpr<Analyzer::Constant>(ti, false, d);
99 }
100 
101 std::shared_ptr<Analyzer::Expr> IntLiteral::analyze(
102  const Catalog_Namespace::Catalog& catalog,
104  TlistRefType allow_tlist_ref) const {
105  return analyzeValue(intval);
106 }
107 
108 std::shared_ptr<Analyzer::Expr> IntLiteral::analyzeValue(const int64_t intval) {
109  SQLTypes t;
110  Datum d;
111  if (intval >= INT16_MIN && intval <= INT16_MAX) {
112  t = kSMALLINT;
113  d.smallintval = (int16_t)intval;
114  } else if (intval >= INT32_MIN && intval <= INT32_MAX) {
115  t = kINT;
116  d.intval = (int32_t)intval;
117  } else {
118  t = kBIGINT;
119  d.bigintval = intval;
120  }
121  return makeExpr<Analyzer::Constant>(t, false, d);
122 }
123 
124 std::shared_ptr<Analyzer::Expr> FixedPtLiteral::analyze(
125  const Catalog_Namespace::Catalog& catalog,
127  TlistRefType allow_tlist_ref) const {
128  SQLTypeInfo ti(kNUMERIC, 0, 0, false);
129  Datum d = StringToDatum(*fixedptval, ti);
130  return makeExpr<Analyzer::Constant>(ti, false, d);
131 }
132 
133 std::shared_ptr<Analyzer::Expr> FixedPtLiteral::analyzeValue(const int64_t numericval,
134  const int scale,
135  const int precision) {
136  SQLTypeInfo ti(kNUMERIC, 0, 0, false);
137  ti.set_scale(scale);
138  ti.set_precision(precision);
139  Datum d;
140  d.bigintval = numericval;
141  return makeExpr<Analyzer::Constant>(ti, false, d);
142 }
143 
144 std::shared_ptr<Analyzer::Expr> FloatLiteral::analyze(
145  const Catalog_Namespace::Catalog& catalog,
147  TlistRefType allow_tlist_ref) const {
148  Datum d;
149  d.floatval = floatval;
150  return makeExpr<Analyzer::Constant>(kFLOAT, false, d);
151 }
152 
153 std::shared_ptr<Analyzer::Expr> DoubleLiteral::analyze(
154  const Catalog_Namespace::Catalog& catalog,
156  TlistRefType allow_tlist_ref) const {
157  Datum d;
158  d.doubleval = doubleval;
159  return makeExpr<Analyzer::Constant>(kDOUBLE, false, d);
160 }
161 
162 std::shared_ptr<Analyzer::Expr> TimestampLiteral::analyze(
163  const Catalog_Namespace::Catalog& catalog,
165  TlistRefType allow_tlist_ref) const {
166  return get(timestampval_);
167 }
168 
169 std::shared_ptr<Analyzer::Expr> TimestampLiteral::get(const int64_t timestampval) {
170  Datum d;
171  d.bigintval = timestampval;
172  return makeExpr<Analyzer::Constant>(kTIMESTAMP, false, d);
173 }
174 
175 std::shared_ptr<Analyzer::Expr> UserLiteral::analyze(
176  const Catalog_Namespace::Catalog& catalog,
178  TlistRefType allow_tlist_ref) const {
179  throw std::runtime_error("USER literal not supported yet.");
180  return nullptr;
181 }
182 
183 std::shared_ptr<Analyzer::Expr> ArrayLiteral::analyze(
184  const Catalog_Namespace::Catalog& catalog,
186  TlistRefType allow_tlist_ref) const {
187  SQLTypeInfo ti = SQLTypeInfo(kARRAY, true);
188  bool set_subtype = true;
189  std::list<std::shared_ptr<Analyzer::Expr>> value_exprs;
190  for (auto& p : value_list) {
191  auto e = p->analyze(catalog, query, allow_tlist_ref);
192  CHECK(e);
193  auto c = std::dynamic_pointer_cast<Analyzer::Constant>(e);
194  if (c != nullptr && c->get_is_null()) {
195  value_exprs.push_back(c);
196  continue;
197  }
198  auto subtype = e->get_type_info().get_type();
199  if (subtype == kNULLT) {
200  // NULL element
201  } else if (set_subtype) {
202  ti.set_subtype(subtype);
203  set_subtype = false;
204  } else {
205  if (ti.get_subtype() != subtype) {
206  throw std::runtime_error("ARRAY element literals should be of the same type.");
207  }
208  }
209  value_exprs.push_back(e);
210  }
211  std::shared_ptr<Analyzer::Expr> result =
212  makeExpr<Analyzer::Constant>(ti, false, value_exprs);
213  return result;
214 }
215 
216 std::string ArrayLiteral::to_string() const {
217  std::string str = "{";
218  bool notfirst = false;
219  for (auto& p : value_list) {
220  if (notfirst) {
221  str += ", ";
222  } else {
223  notfirst = true;
224  }
225  str += p->to_string();
226  }
227  str += "}";
228  return str;
229 }
230 
231 std::shared_ptr<Analyzer::Expr> OperExpr::analyze(
232  const Catalog_Namespace::Catalog& catalog,
234  TlistRefType allow_tlist_ref) const {
235  auto left_expr = left->analyze(catalog, query, allow_tlist_ref);
236  const auto& left_type = left_expr->get_type_info();
237  if (right == nullptr) {
238  return makeExpr<Analyzer::UOper>(
239  left_type, left_expr->get_contains_agg(), optype, left_expr->decompress());
240  }
241  if (optype == kARRAY_AT) {
242  if (left_type.get_type() != kARRAY) {
243  throw std::runtime_error(left->to_string() + " is not of array type.");
244  }
245  auto right_expr = right->analyze(catalog, query, allow_tlist_ref);
246  const auto& right_type = right_expr->get_type_info();
247  if (!right_type.is_integer()) {
248  throw std::runtime_error(right->to_string() + " is not of integer type.");
249  }
250  return makeExpr<Analyzer::BinOper>(
251  left_type.get_elem_type(), false, kARRAY_AT, kONE, left_expr, right_expr);
252  }
253  auto right_expr = right->analyze(catalog, query, allow_tlist_ref);
254  return normalize(optype, opqualifier, left_expr, right_expr);
255 }
256 
257 std::shared_ptr<Analyzer::Expr> OperExpr::normalize(
258  const SQLOps optype,
259  const SQLQualifier qual,
260  std::shared_ptr<Analyzer::Expr> left_expr,
261  std::shared_ptr<Analyzer::Expr> right_expr) {
262  if (left_expr->get_type_info().is_date_in_days() ||
263  right_expr->get_type_info().is_date_in_days()) {
264  // Do not propogate encoding
265  left_expr = left_expr->decompress();
266  right_expr = right_expr->decompress();
267  }
268  const auto& left_type = left_expr->get_type_info();
269  auto right_type = right_expr->get_type_info();
270  if (qual != kONE) {
271  // subquery not supported yet.
272  CHECK(!std::dynamic_pointer_cast<Analyzer::Subquery>(right_expr));
273  if (right_type.get_type() != kARRAY) {
274  throw std::runtime_error(
275  "Existential or universal qualifiers can only be used in front of a subquery "
276  "or an "
277  "expression of array type.");
278  }
279  right_type = right_type.get_elem_type();
280  }
281  SQLTypeInfo new_left_type;
282  SQLTypeInfo new_right_type;
283  auto result_type = Analyzer::BinOper::analyze_type_info(
284  optype, left_type, right_type, &new_left_type, &new_right_type);
285  if (result_type.is_timeinterval()) {
286  return makeExpr<Analyzer::BinOper>(
287  result_type, false, optype, qual, left_expr, right_expr);
288  }
289  if (left_type != new_left_type) {
290  left_expr = left_expr->add_cast(new_left_type);
291  }
292  if (right_type != new_right_type) {
293  if (qual == kONE) {
294  right_expr = right_expr->add_cast(new_right_type);
295  } else {
296  right_expr = right_expr->add_cast(new_right_type.get_array_type());
297  }
298  }
299  auto check_compression = IS_COMPARISON(optype);
300  if (check_compression) {
301  if (new_left_type.get_compression() == kENCODING_DICT &&
302  new_right_type.get_compression() == kENCODING_DICT &&
303  new_left_type.get_comp_param() == new_right_type.get_comp_param()) {
304  // do nothing
305  } else if (new_left_type.get_compression() == kENCODING_DICT &&
306  new_right_type.get_compression() == kENCODING_NONE) {
307  SQLTypeInfo ti(new_right_type);
308  ti.set_compression(new_left_type.get_compression());
309  ti.set_comp_param(new_left_type.get_comp_param());
310  ti.set_fixed_size();
311  right_expr = right_expr->add_cast(ti);
312  } else if (new_right_type.get_compression() == kENCODING_DICT &&
313  new_left_type.get_compression() == kENCODING_NONE) {
314  SQLTypeInfo ti(new_left_type);
315  ti.set_compression(new_right_type.get_compression());
316  ti.set_comp_param(new_right_type.get_comp_param());
317  ti.set_fixed_size();
318  left_expr = left_expr->add_cast(ti);
319  } else {
320  left_expr = left_expr->decompress();
321  right_expr = right_expr->decompress();
322  }
323  } else {
324  left_expr = left_expr->decompress();
325  right_expr = right_expr->decompress();
326  }
327  bool has_agg = (left_expr->get_contains_agg() || right_expr->get_contains_agg());
328  return makeExpr<Analyzer::BinOper>(
329  result_type, has_agg, optype, qual, left_expr, right_expr);
330 }
331 
332 std::shared_ptr<Analyzer::Expr> SubqueryExpr::analyze(
333  const Catalog_Namespace::Catalog& catalog,
335  TlistRefType allow_tlist_ref) const {
336  throw std::runtime_error("Subqueries are not supported yet.");
337  return nullptr;
338 }
339 
340 std::shared_ptr<Analyzer::Expr> IsNullExpr::analyze(
341  const Catalog_Namespace::Catalog& catalog,
343  TlistRefType allow_tlist_ref) const {
344  auto arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
345  auto result = makeExpr<Analyzer::UOper>(kBOOLEAN, kISNULL, arg_expr);
346  if (is_not) {
347  result = makeExpr<Analyzer::UOper>(kBOOLEAN, kNOT, result);
348  }
349  return result;
350 }
351 
352 std::shared_ptr<Analyzer::Expr> InSubquery::analyze(
353  const Catalog_Namespace::Catalog& catalog,
355  TlistRefType allow_tlist_ref) const {
356  throw std::runtime_error("Subqueries are not supported yet.");
357  return nullptr;
358 }
359 
360 std::shared_ptr<Analyzer::Expr> InValues::analyze(
361  const Catalog_Namespace::Catalog& catalog,
363  TlistRefType allow_tlist_ref) const {
364  auto arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
365  SQLTypeInfo ti = arg_expr->get_type_info();
366  bool dict_comp = ti.get_compression() == kENCODING_DICT;
367  std::list<std::shared_ptr<Analyzer::Expr>> value_exprs;
368  for (auto& p : value_list) {
369  auto e = p->analyze(catalog, query, allow_tlist_ref);
370  if (ti != e->get_type_info()) {
371  if (ti.is_string() && e->get_type_info().is_string()) {
372  ti = Analyzer::BinOper::common_string_type(ti, e->get_type_info());
373  } else if (ti.is_number() && e->get_type_info().is_number()) {
374  ti = Analyzer::BinOper::common_numeric_type(ti, e->get_type_info());
375  } else {
376  throw std::runtime_error("IN expressions must contain compatible types.");
377  }
378  }
379  if (dict_comp) {
380  value_exprs.push_back(e->add_cast(arg_expr->get_type_info()));
381  } else {
382  value_exprs.push_back(e);
383  }
384  }
385  if (!dict_comp) {
386  arg_expr = arg_expr->decompress();
387  arg_expr = arg_expr->add_cast(ti);
388  std::list<std::shared_ptr<Analyzer::Expr>> cast_vals;
389  for (auto p : value_exprs) {
390  cast_vals.push_back(p->add_cast(ti));
391  }
392  value_exprs.swap(cast_vals);
393  }
394  std::shared_ptr<Analyzer::Expr> result =
395  makeExpr<Analyzer::InValues>(arg_expr, value_exprs);
396  if (is_not) {
397  result = makeExpr<Analyzer::UOper>(kBOOLEAN, kNOT, result);
398  }
399  return result;
400 }
401 
402 std::shared_ptr<Analyzer::Expr> BetweenExpr::analyze(
403  const Catalog_Namespace::Catalog& catalog,
405  TlistRefType allow_tlist_ref) const {
406  auto arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
407  auto lower_expr = lower->analyze(catalog, query, allow_tlist_ref);
408  auto upper_expr = upper->analyze(catalog, query, allow_tlist_ref);
409  SQLTypeInfo new_left_type, new_right_type;
411  arg_expr->get_type_info(),
412  lower_expr->get_type_info(),
413  &new_left_type,
414  &new_right_type);
415  auto lower_pred =
416  makeExpr<Analyzer::BinOper>(kBOOLEAN,
417  kGE,
418  kONE,
419  arg_expr->add_cast(new_left_type)->decompress(),
420  lower_expr->add_cast(new_right_type)->decompress());
422  arg_expr->get_type_info(),
423  lower_expr->get_type_info(),
424  &new_left_type,
425  &new_right_type);
426  auto upper_pred = makeExpr<Analyzer::BinOper>(
427  kBOOLEAN,
428  kLE,
429  kONE,
430  arg_expr->deep_copy()->add_cast(new_left_type)->decompress(),
431  upper_expr->add_cast(new_right_type)->decompress());
432  std::shared_ptr<Analyzer::Expr> result =
433  makeExpr<Analyzer::BinOper>(kBOOLEAN, kAND, kONE, lower_pred, upper_pred);
434  if (is_not) {
435  result = makeExpr<Analyzer::UOper>(kBOOLEAN, kNOT, result);
436  }
437  return result;
438 }
439 
440 std::shared_ptr<Analyzer::Expr> CharLengthExpr::analyze(
441  const Catalog_Namespace::Catalog& catalog,
443  TlistRefType allow_tlist_ref) const {
444  auto arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
445  if (!arg_expr->get_type_info().is_string()) {
446  throw std::runtime_error(
447  "expression in char_length clause must be of a string type.");
448  }
449  std::shared_ptr<Analyzer::Expr> result =
450  makeExpr<Analyzer::CharLengthExpr>(arg_expr->decompress(), calc_encoded_length);
451  return result;
452 }
453 
454 std::shared_ptr<Analyzer::Expr> CardinalityExpr::analyze(
455  const Catalog_Namespace::Catalog& catalog,
457  TlistRefType allow_tlist_ref) const {
458  auto arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
459  if (!arg_expr->get_type_info().is_array()) {
460  throw std::runtime_error(
461  "expression in cardinality clause must be of an array type.");
462  }
463  std::shared_ptr<Analyzer::Expr> result =
464  makeExpr<Analyzer::CardinalityExpr>(arg_expr->decompress());
465  return result;
466 }
467 
468 void LikeExpr::check_like_expr(const std::string& like_str, char escape_char) {
469  if (like_str.back() == escape_char) {
470  throw std::runtime_error("LIKE pattern must not end with escape character.");
471  }
472 }
473 
474 bool LikeExpr::test_is_simple_expr(const std::string& like_str, char escape_char) {
475  // if not bounded by '%' then not a simple string
476  if (like_str.size() < 2 || like_str[0] != '%' || like_str[like_str.size() - 1] != '%') {
477  return false;
478  }
479  // if the last '%' is escaped then not a simple string
480  if (like_str[like_str.size() - 2] == escape_char &&
481  like_str[like_str.size() - 3] != escape_char) {
482  return false;
483  }
484  for (size_t i = 1; i < like_str.size() - 1; i++) {
485  if (like_str[i] == '%' || like_str[i] == '_' || like_str[i] == '[' ||
486  like_str[i] == ']') {
487  if (like_str[i - 1] != escape_char) {
488  return false;
489  }
490  }
491  }
492  return true;
493 }
494 
495 void LikeExpr::erase_cntl_chars(std::string& like_str, char escape_char) {
496  char prev_char = '\0';
497  // easier to create new string of allowable chars
498  // rather than erase chars from
499  // existing string
500  std::string new_str;
501  for (char& cur_char : like_str) {
502  if (cur_char == '%' || cur_char == escape_char) {
503  if (prev_char != escape_char) {
504  prev_char = cur_char;
505  continue;
506  }
507  }
508  new_str.push_back(cur_char);
509  prev_char = cur_char;
510  }
511  like_str = new_str;
512 }
513 
514 std::shared_ptr<Analyzer::Expr> LikeExpr::analyze(
515  const Catalog_Namespace::Catalog& catalog,
517  TlistRefType allow_tlist_ref) const {
518  auto arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
519  auto like_expr = like_string->analyze(catalog, query, allow_tlist_ref);
520  auto escape_expr = escape_string == nullptr
521  ? nullptr
522  : escape_string->analyze(catalog, query, allow_tlist_ref);
523  return LikeExpr::get(arg_expr, like_expr, escape_expr, is_ilike, is_not);
524 }
525 
526 std::shared_ptr<Analyzer::Expr> LikeExpr::get(std::shared_ptr<Analyzer::Expr> arg_expr,
527  std::shared_ptr<Analyzer::Expr> like_expr,
528  std::shared_ptr<Analyzer::Expr> escape_expr,
529  const bool is_ilike,
530  const bool is_not) {
531  if (!arg_expr->get_type_info().is_string()) {
532  throw std::runtime_error("expression before LIKE must be of a string type.");
533  }
534  if (!like_expr->get_type_info().is_string()) {
535  throw std::runtime_error("expression after LIKE must be of a string type.");
536  }
537  char escape_char = '\\';
538  if (escape_expr != nullptr) {
539  if (!escape_expr->get_type_info().is_string()) {
540  throw std::runtime_error("expression after ESCAPE must be of a string type.");
541  }
542  if (!escape_expr->get_type_info().is_string()) {
543  throw std::runtime_error("expression after ESCAPE must be of a string type.");
544  }
545  auto c = std::dynamic_pointer_cast<Analyzer::Constant>(escape_expr);
546  if (c != nullptr && c->get_constval().stringval->length() > 1) {
547  throw std::runtime_error("String after ESCAPE must have a single character.");
548  }
549  escape_char = (*c->get_constval().stringval)[0];
550  }
551  auto c = std::dynamic_pointer_cast<Analyzer::Constant>(like_expr);
552  bool is_simple = false;
553  if (c != nullptr) {
554  std::string& pattern = *c->get_constval().stringval;
555  if (is_ilike) {
556  std::transform(pattern.begin(), pattern.end(), pattern.begin(), ::tolower);
557  }
558  check_like_expr(pattern, escape_char);
559  is_simple = test_is_simple_expr(pattern, escape_char);
560  if (is_simple) {
561  erase_cntl_chars(pattern, escape_char);
562  }
563  }
564  std::shared_ptr<Analyzer::Expr> result = makeExpr<Analyzer::LikeExpr>(
565  arg_expr->decompress(), like_expr, escape_expr, is_ilike, is_simple);
566  if (is_not) {
567  result = makeExpr<Analyzer::UOper>(kBOOLEAN, kNOT, result);
568  }
569  return result;
570 }
571 
572 void RegexpExpr::check_pattern_expr(const std::string& pattern_str, char escape_char) {
573  if (pattern_str.back() == escape_char) {
574  throw std::runtime_error("REGEXP pattern must not end with escape character.");
575  }
576 }
577 
578 bool RegexpExpr::translate_to_like_pattern(std::string& pattern_str, char escape_char) {
579  char prev_char = '\0';
580  char prev_prev_char = '\0';
581  std::string like_str;
582  for (char& cur_char : pattern_str) {
583  if (prev_char == escape_char || isalnum(cur_char) || cur_char == ' ' ||
584  cur_char == '.') {
585  like_str.push_back((cur_char == '.') ? '_' : cur_char);
586  prev_prev_char = prev_char;
587  prev_char = cur_char;
588  continue;
589  }
590  if (prev_char == '.' && prev_prev_char != escape_char) {
591  if (cur_char == '*' || cur_char == '+') {
592  if (cur_char == '*') {
593  like_str.pop_back();
594  }
595  // .* --> %
596  // .+ --> _%
597  like_str.push_back('%');
598  prev_prev_char = prev_char;
599  prev_char = cur_char;
600  continue;
601  }
602  }
603  return false;
604  }
605  pattern_str = like_str;
606  return true;
607 }
608 
609 std::shared_ptr<Analyzer::Expr> RegexpExpr::analyze(
610  const Catalog_Namespace::Catalog& catalog,
612  TlistRefType allow_tlist_ref) const {
613  auto arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
614  auto pattern_expr = pattern_string->analyze(catalog, query, allow_tlist_ref);
615  auto escape_expr = escape_string == nullptr
616  ? nullptr
617  : escape_string->analyze(catalog, query, allow_tlist_ref);
618  return RegexpExpr::get(arg_expr, pattern_expr, escape_expr, is_not);
619 }
620 
621 std::shared_ptr<Analyzer::Expr> RegexpExpr::get(
622  std::shared_ptr<Analyzer::Expr> arg_expr,
623  std::shared_ptr<Analyzer::Expr> pattern_expr,
624  std::shared_ptr<Analyzer::Expr> escape_expr,
625  const bool is_not) {
626  if (!arg_expr->get_type_info().is_string()) {
627  throw std::runtime_error("expression before REGEXP must be of a string type.");
628  }
629  if (!pattern_expr->get_type_info().is_string()) {
630  throw std::runtime_error("expression after REGEXP must be of a string type.");
631  }
632  char escape_char = '\\';
633  if (escape_expr != nullptr) {
634  if (!escape_expr->get_type_info().is_string()) {
635  throw std::runtime_error("expression after ESCAPE must be of a string type.");
636  }
637  if (!escape_expr->get_type_info().is_string()) {
638  throw std::runtime_error("expression after ESCAPE must be of a string type.");
639  }
640  auto c = std::dynamic_pointer_cast<Analyzer::Constant>(escape_expr);
641  if (c != nullptr && c->get_constval().stringval->length() > 1) {
642  throw std::runtime_error("String after ESCAPE must have a single character.");
643  }
644  escape_char = (*c->get_constval().stringval)[0];
645  if (escape_char != '\\') {
646  throw std::runtime_error("Only supporting '\\' escape character.");
647  }
648  }
649  auto c = std::dynamic_pointer_cast<Analyzer::Constant>(pattern_expr);
650  if (c != nullptr) {
651  std::string& pattern = *c->get_constval().stringval;
652  if (translate_to_like_pattern(pattern, escape_char)) {
653  return LikeExpr::get(arg_expr, pattern_expr, escape_expr, false, is_not);
654  }
655  }
656  std::shared_ptr<Analyzer::Expr> result =
657  makeExpr<Analyzer::RegexpExpr>(arg_expr->decompress(), pattern_expr, escape_expr);
658  if (is_not) {
659  result = makeExpr<Analyzer::UOper>(kBOOLEAN, kNOT, result);
660  }
661  return result;
662 }
663 
664 std::shared_ptr<Analyzer::Expr> LikelihoodExpr::analyze(
665  const Catalog_Namespace::Catalog& catalog,
667  TlistRefType allow_tlist_ref) const {
668  auto arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
669  return LikelihoodExpr::get(arg_expr, likelihood, is_not);
670 }
671 
672 std::shared_ptr<Analyzer::Expr> LikelihoodExpr::get(
673  std::shared_ptr<Analyzer::Expr> arg_expr,
674  float likelihood,
675  const bool is_not) {
676  if (!arg_expr->get_type_info().is_boolean()) {
677  throw std::runtime_error("likelihood expression expects boolean type.");
678  }
679  std::shared_ptr<Analyzer::Expr> result = makeExpr<Analyzer::LikelihoodExpr>(
680  arg_expr->decompress(), is_not ? 1 - likelihood : likelihood);
681  return result;
682 }
683 
684 std::shared_ptr<Analyzer::Expr> ExistsExpr::analyze(
685  const Catalog_Namespace::Catalog& catalog,
687  TlistRefType allow_tlist_ref) const {
688  throw std::runtime_error("Subqueries are not supported yet.");
689  return nullptr;
690 }
691 
692 std::shared_ptr<Analyzer::Expr> ColumnRef::analyze(
693  const Catalog_Namespace::Catalog& catalog,
695  TlistRefType allow_tlist_ref) const {
696  int table_id{0};
697  int rte_idx{0};
698  const ColumnDescriptor* cd{nullptr};
699  if (column == nullptr) {
700  throw std::runtime_error("invalid column name *.");
701  }
702  if (table != nullptr) {
703  rte_idx = query.get_rte_idx(*table);
704  if (rte_idx < 0) {
705  throw std::runtime_error("range variable or table name " + *table +
706  " does not exist.");
707  }
708  Analyzer::RangeTableEntry* rte = query.get_rte(rte_idx);
709  cd = rte->get_column_desc(catalog, *column);
710  if (cd == nullptr) {
711  throw std::runtime_error("Column name " + *column + " does not exist.");
712  }
713  table_id = rte->get_table_id();
714  } else {
715  bool found = false;
716  int i = 0;
717  for (auto rte : query.get_rangetable()) {
718  cd = rte->get_column_desc(catalog, *column);
719  if (cd != nullptr && !found) {
720  found = true;
721  rte_idx = i;
722  table_id = rte->get_table_id();
723  } else if (cd != nullptr && found) {
724  throw std::runtime_error("Column name " + *column + " is ambiguous.");
725  }
726  i++;
727  }
728  if (cd == nullptr && allow_tlist_ref != TlistRefType::TLIST_NONE) {
729  // check if this is a reference to a targetlist entry
730  bool found = false;
731  int varno = -1;
732  int i = 1;
733  std::shared_ptr<Analyzer::TargetEntry> tle;
734  for (auto p : query.get_targetlist()) {
735  if (*column == p->get_resname() && !found) {
736  found = true;
737  varno = i;
738  tle = p;
739  } else if (*column == p->get_resname() && found) {
740  throw std::runtime_error("Output alias " + *column + " is ambiguous.");
741  }
742  i++;
743  }
744  if (found) {
745  if (dynamic_cast<Analyzer::Var*>(tle->get_expr())) {
746  Analyzer::Var* v = static_cast<Analyzer::Var*>(tle->get_expr());
748  return v->deep_copy();
749  }
750  }
751  if (allow_tlist_ref == TlistRefType::TLIST_COPY) {
752  return tle->get_expr()->deep_copy();
753  } else {
754  return makeExpr<Analyzer::Var>(
755  tle->get_expr()->get_type_info(), Analyzer::Var::kOUTPUT, varno);
756  }
757  }
758  }
759  if (cd == nullptr) {
760  throw std::runtime_error("Column name " + *column + " does not exist.");
761  }
762  }
763  return makeExpr<Analyzer::ColumnVar>(cd->columnType, table_id, cd->columnId, rte_idx);
764 }
765 
766 std::shared_ptr<Analyzer::Expr> FunctionRef::analyze(
767  const Catalog_Namespace::Catalog& catalog,
769  TlistRefType allow_tlist_ref) const {
770  SQLTypeInfo result_type;
771  SQLAgg agg_type;
772  std::shared_ptr<Analyzer::Expr> arg_expr;
773  bool is_distinct = false;
774  if (boost::iequals(*name, "count")) {
775  result_type = SQLTypeInfo(kBIGINT, false);
776  agg_type = kCOUNT;
777  if (arg) {
778  arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
779  const SQLTypeInfo& ti = arg_expr->get_type_info();
780  if (ti.is_string() && (ti.get_compression() != kENCODING_DICT || !distinct)) {
781  throw std::runtime_error(
782  "Strings must be dictionary-encoded in COUNT(DISTINCT).");
783  }
784  if (ti.get_type() == kARRAY && !distinct) {
785  throw std::runtime_error("Only COUNT(DISTINCT) is supported on arrays.");
786  }
787  }
788  is_distinct = distinct;
789  } else {
790  if (!arg) {
791  throw std::runtime_error("Cannot compute " + *name + " with argument '*'.");
792  }
793  if (boost::iequals(*name, "min")) {
794  agg_type = kMIN;
795  arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
796  arg_expr = arg_expr->decompress();
797  result_type = arg_expr->get_type_info();
798  } else if (boost::iequals(*name, "max")) {
799  agg_type = kMAX;
800  arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
801  arg_expr = arg_expr->decompress();
802  result_type = arg_expr->get_type_info();
803  } else if (boost::iequals(*name, "avg")) {
804  agg_type = kAVG;
805  arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
806  if (!arg_expr->get_type_info().is_number()) {
807  throw std::runtime_error("Cannot compute AVG on non-number-type arguments.");
808  }
809  arg_expr = arg_expr->decompress();
810  result_type = SQLTypeInfo(kDOUBLE, false);
811  } else if (boost::iequals(*name, "sum")) {
812  agg_type = kSUM;
813  arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
814  if (!arg_expr->get_type_info().is_number()) {
815  throw std::runtime_error("Cannot compute SUM on non-number-type arguments.");
816  }
817  arg_expr = arg_expr->decompress();
818  result_type = arg_expr->get_type_info().is_integer() ? SQLTypeInfo(kBIGINT, false)
819  : arg_expr->get_type_info();
820  } else if (boost::iequals(*name, "unnest")) {
821  arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
822  const SQLTypeInfo& arg_ti = arg_expr->get_type_info();
823  if (arg_ti.get_type() != kARRAY) {
824  throw std::runtime_error(arg->to_string() + " is not of array type.");
825  }
826  return makeExpr<Analyzer::UOper>(arg_ti.get_elem_type(), false, kUNNEST, arg_expr);
827  } else {
828  throw std::runtime_error("invalid function name: " + *name);
829  }
830  if (arg_expr->get_type_info().is_string() ||
831  arg_expr->get_type_info().get_type() == kARRAY) {
832  throw std::runtime_error(
833  "Only COUNT(DISTINCT ) aggregate is supported on strings and arrays.");
834  }
835  }
836  int naggs = query.get_num_aggs();
837  query.set_num_aggs(naggs + 1);
838  return makeExpr<Analyzer::AggExpr>(
839  result_type, agg_type, arg_expr, is_distinct, nullptr);
840 }
841 
842 std::shared_ptr<Analyzer::Expr> CastExpr::analyze(
843  const Catalog_Namespace::Catalog& catalog,
845  TlistRefType allow_tlist_ref) const {
846  target_type->check_type();
847  auto arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
848  SQLTypeInfo ti(target_type->get_type(),
849  target_type->get_param1(),
850  target_type->get_param2(),
851  arg_expr->get_type_info().get_notnull());
852  if (arg_expr->get_type_info().get_type() != target_type->get_type() &&
853  arg_expr->get_type_info().get_compression() != kENCODING_NONE) {
854  arg_expr->decompress();
855  }
856  return arg_expr->add_cast(ti);
857 }
858 
859 std::shared_ptr<Analyzer::Expr> CaseExpr::analyze(
860  const Catalog_Namespace::Catalog& catalog,
862  TlistRefType allow_tlist_ref) const {
863  SQLTypeInfo ti;
864  std::list<std::pair<std::shared_ptr<Analyzer::Expr>, std::shared_ptr<Analyzer::Expr>>>
865  expr_pair_list;
866  for (auto& p : when_then_list) {
867  auto e1 = p->get_expr1()->analyze(catalog, query, allow_tlist_ref);
868  if (e1->get_type_info().get_type() != kBOOLEAN) {
869  throw std::runtime_error("Only boolean expressions can be used after WHEN.");
870  }
871  auto e2 = p->get_expr2()->analyze(catalog, query, allow_tlist_ref);
872  expr_pair_list.emplace_back(e1, e2);
873  }
874  auto else_e = else_expr ? else_expr->analyze(catalog, query, allow_tlist_ref) : nullptr;
875  return normalize(expr_pair_list, else_e);
876 }
877 
878 namespace {
879 
880 bool expr_is_null(const Analyzer::Expr* expr) {
881  if (expr->get_type_info().get_type() == kNULLT) {
882  return true;
883  }
884  const auto const_expr = dynamic_cast<const Analyzer::Constant*>(expr);
885  return const_expr && const_expr->get_is_null();
886 }
887 
888 } // namespace
889 
890 std::shared_ptr<Analyzer::Expr> CaseExpr::normalize(
891  const std::list<std::pair<std::shared_ptr<Analyzer::Expr>,
892  std::shared_ptr<Analyzer::Expr>>>& expr_pair_list,
893  const std::shared_ptr<Analyzer::Expr> else_e_in) {
894  SQLTypeInfo ti;
895  bool has_agg = false;
896  for (auto& p : expr_pair_list) {
897  auto e1 = p.first;
898  CHECK(e1->get_type_info().is_boolean());
899  auto e2 = p.second;
900  if (ti.get_type() == kNULLT) {
901  ti = e2->get_type_info();
902  } else if (e2->get_type_info().get_type() == kNULLT) {
903  ti.set_notnull(false);
904  e2->set_type_info(ti);
905  } else if (ti != e2->get_type_info()) {
906  if (ti.is_string() && e2->get_type_info().is_string()) {
907  ti = Analyzer::BinOper::common_string_type(ti, e2->get_type_info());
908  } else if (ti.is_number() && e2->get_type_info().is_number()) {
909  ti = Analyzer::BinOper::common_numeric_type(ti, e2->get_type_info());
910  } else if (ti.is_boolean() && e2->get_type_info().is_boolean()) {
911  ti = Analyzer::BinOper::common_numeric_type(ti, e2->get_type_info());
912  } else {
913  throw std::runtime_error(
914  "expressions in THEN clause must be of the same or compatible types.");
915  }
916  }
917  if (e2->get_contains_agg()) {
918  has_agg = true;
919  }
920  }
921  auto else_e = else_e_in;
922  if (else_e) {
923  if (else_e->get_contains_agg()) {
924  has_agg = true;
925  }
926  if (expr_is_null(else_e.get())) {
927  ti.set_notnull(false);
928  else_e->set_type_info(ti);
929  } else if (ti != else_e->get_type_info()) {
930  ti.set_notnull(false);
931  if (ti.is_string() && else_e->get_type_info().is_string()) {
932  ti = Analyzer::BinOper::common_string_type(ti, else_e->get_type_info());
933  } else if (ti.is_number() && else_e->get_type_info().is_number()) {
934  ti = Analyzer::BinOper::common_numeric_type(ti, else_e->get_type_info());
935  } else if (ti.is_boolean() && else_e->get_type_info().is_boolean()) {
936  ti = Analyzer::BinOper::common_numeric_type(ti, else_e->get_type_info());
937  } else {
938  throw std::runtime_error(
939  "expressions in ELSE clause must be of the same or compatible types as those "
940  "in the THEN clauses.");
941  }
942  }
943  }
944  std::list<std::pair<std::shared_ptr<Analyzer::Expr>, std::shared_ptr<Analyzer::Expr>>>
945  cast_expr_pair_list;
946  for (auto p : expr_pair_list) {
947  ti.set_notnull(false);
948  cast_expr_pair_list.emplace_back(p.first, p.second->add_cast(ti));
949  }
950  if (else_e != nullptr) {
951  else_e = else_e->add_cast(ti);
952  } else {
953  Datum d;
954  // always create an else expr so that executor doesn't need to worry about it
955  ti.set_notnull(false);
956  else_e = makeExpr<Analyzer::Constant>(ti, true, d);
957  }
958  if (ti.get_type() == kNULLT) {
959  throw std::runtime_error(
960  "Can't deduce the type for case expressions, all branches null");
961  }
962  return makeExpr<Analyzer::CaseExpr>(ti, has_agg, cast_expr_pair_list, else_e);
963 }
964 
965 std::string CaseExpr::to_string() const {
966  std::string str("CASE ");
967  for (auto& p : when_then_list) {
968  str += "WHEN " + p->get_expr1()->to_string() + " THEN " +
969  p->get_expr2()->to_string() + " ";
970  }
971  if (else_expr != nullptr) {
972  str += "ELSE " + else_expr->to_string();
973  }
974  str += " END";
975  return str;
976 }
977 
978 void UnionQuery::analyze(const Catalog_Namespace::Catalog& catalog,
979  Analyzer::Query& query) const {
980  left->analyze(catalog, query);
981  Analyzer::Query* right_query = new Analyzer::Query();
982  right->analyze(catalog, *right_query);
983  query.set_next_query(right_query);
984  query.set_is_unionall(is_unionall);
985 }
986 
987 void QuerySpec::analyze_having_clause(const Catalog_Namespace::Catalog& catalog,
988  Analyzer::Query& query) const {
989  std::shared_ptr<Analyzer::Expr> p;
990  if (having_clause != nullptr) {
991  p = having_clause->analyze(catalog, query, Expr::TlistRefType::TLIST_COPY);
992  if (p->get_type_info().get_type() != kBOOLEAN) {
993  throw std::runtime_error("Only boolean expressions can be in HAVING clause.");
994  }
995  p->check_group_by(query.get_group_by());
996  }
997  query.set_having_predicate(p);
998 }
999 
1000 void QuerySpec::analyze_group_by(const Catalog_Namespace::Catalog& catalog,
1001  Analyzer::Query& query) const {
1002  std::list<std::shared_ptr<Analyzer::Expr>> groupby;
1003  if (!groupby_clause.empty()) {
1004  int gexpr_no = 1;
1005  std::shared_ptr<Analyzer::Expr> gexpr;
1006  const std::vector<std::shared_ptr<Analyzer::TargetEntry>>& tlist =
1007  query.get_targetlist();
1008  for (auto& c : groupby_clause) {
1009  // special-case ordinal numbers in GROUP BY
1010  if (dynamic_cast<Literal*>(c.get())) {
1011  IntLiteral* i = dynamic_cast<IntLiteral*>(c.get());
1012  if (!i) {
1013  throw std::runtime_error("Invalid literal in GROUP BY clause.");
1014  }
1015  int varno = (int)i->get_intval();
1016  if (varno <= 0 || varno > static_cast<int>(tlist.size())) {
1017  throw std::runtime_error("Invalid ordinal number in GROUP BY clause.");
1018  }
1019  if (tlist[varno - 1]->get_expr()->get_contains_agg()) {
1020  throw std::runtime_error(
1021  "Ordinal number in GROUP BY cannot reference an expression containing "
1022  "aggregate "
1023  "functions.");
1024  }
1025  gexpr = makeExpr<Analyzer::Var>(
1026  tlist[varno - 1]->get_expr()->get_type_info(), Analyzer::Var::kOUTPUT, varno);
1027  } else {
1028  gexpr = c->analyze(catalog, query, Expr::TlistRefType::TLIST_REF);
1029  }
1030  const SQLTypeInfo gti = gexpr->get_type_info();
1031  bool set_new_type = false;
1032  SQLTypeInfo ti(gti);
1033  if (gti.is_string() && gti.get_compression() == kENCODING_NONE) {
1034  set_new_type = true;
1037  ti.set_fixed_size();
1038  }
1039  std::shared_ptr<Analyzer::Var> v;
1040  if (std::dynamic_pointer_cast<Analyzer::Var>(gexpr)) {
1041  v = std::static_pointer_cast<Analyzer::Var>(gexpr);
1042  int n = v->get_varno();
1043  gexpr = tlist[n - 1]->get_own_expr();
1044  auto cv = std::dynamic_pointer_cast<Analyzer::ColumnVar>(gexpr);
1045  if (cv != nullptr) {
1046  // inherit all ColumnVar info for lineage.
1047  *std::static_pointer_cast<Analyzer::ColumnVar>(v) = *cv;
1048  }
1049  v->set_which_row(Analyzer::Var::kGROUPBY);
1050  v->set_varno(gexpr_no);
1051  tlist[n - 1]->set_expr(v);
1052  }
1053  if (set_new_type) {
1054  auto new_e = gexpr->add_cast(ti);
1055  groupby.push_back(new_e);
1056  if (v != nullptr) {
1057  v->set_type_info(new_e->get_type_info());
1058  }
1059  } else {
1060  groupby.push_back(gexpr);
1061  }
1062  gexpr_no++;
1063  }
1064  }
1065  if (query.get_num_aggs() > 0 || !groupby.empty()) {
1066  for (auto t : query.get_targetlist()) {
1067  auto e = t->get_expr();
1068  e->check_group_by(groupby);
1069  }
1070  }
1071  query.set_group_by(groupby);
1072 }
1073 
1074 void QuerySpec::analyze_where_clause(const Catalog_Namespace::Catalog& catalog,
1075  Analyzer::Query& query) const {
1076  if (where_clause == nullptr) {
1077  query.set_where_predicate(nullptr);
1078  return;
1079  }
1080  auto p = where_clause->analyze(catalog, query, Expr::TlistRefType::TLIST_COPY);
1081  if (p->get_type_info().get_type() != kBOOLEAN) {
1082  throw std::runtime_error("Only boolean expressions can be in WHERE clause.");
1083  }
1084  query.set_where_predicate(p);
1085 }
1086 
1087 void QuerySpec::analyze_select_clause(const Catalog_Namespace::Catalog& catalog,
1088  Analyzer::Query& query) const {
1089  std::vector<std::shared_ptr<Analyzer::TargetEntry>>& tlist =
1090  query.get_targetlist_nonconst();
1091  if (select_clause.empty()) {
1092  // this means SELECT *
1093  int rte_idx = 0;
1094  for (auto rte : query.get_rangetable()) {
1095  rte->expand_star_in_targetlist(catalog, tlist, rte_idx++);
1096  }
1097  } else {
1098  for (auto& p : select_clause) {
1099  const Parser::Expr* select_expr = p->get_select_expr();
1100  // look for the case of range_var.*
1101  if (typeid(*select_expr) == typeid(ColumnRef) &&
1102  dynamic_cast<const ColumnRef*>(select_expr)->get_column() == nullptr) {
1103  const std::string* range_var_name =
1104  dynamic_cast<const ColumnRef*>(select_expr)->get_table();
1105  int rte_idx = query.get_rte_idx(*range_var_name);
1106  if (rte_idx < 0) {
1107  throw std::runtime_error("invalid range variable name: " + *range_var_name);
1108  }
1109  Analyzer::RangeTableEntry* rte = query.get_rte(rte_idx);
1110  rte->expand_star_in_targetlist(catalog, tlist, rte_idx);
1111  } else {
1112  auto e = select_expr->analyze(catalog, query);
1113  std::string resname;
1114 
1115  if (p->get_alias() != nullptr) {
1116  resname = *p->get_alias();
1117  } else if (std::dynamic_pointer_cast<Analyzer::ColumnVar>(e) &&
1118  !std::dynamic_pointer_cast<Analyzer::Var>(e)) {
1119  auto colvar = std::static_pointer_cast<Analyzer::ColumnVar>(e);
1120  const ColumnDescriptor* col_desc = catalog.getMetadataForColumn(
1121  colvar->get_table_id(), colvar->get_column_id());
1122  resname = col_desc->columnName;
1123  }
1124  if (e->get_type_info().get_type() == kNULLT) {
1125  throw std::runtime_error(
1126  "Untyped NULL in SELECT clause. Use CAST to specify a type.");
1127  }
1128  auto o = std::static_pointer_cast<Analyzer::UOper>(e);
1129  bool unnest = (o != nullptr && o->get_optype() == kUNNEST);
1130  auto tle = std::make_shared<Analyzer::TargetEntry>(resname, e, unnest);
1131  tlist.push_back(tle);
1132  }
1133  }
1134  }
1135 }
1136 
1137 void QuerySpec::analyze_from_clause(const Catalog_Namespace::Catalog& catalog,
1138  Analyzer::Query& query) const {
1140  for (auto& p : from_clause) {
1141  const TableDescriptor* table_desc;
1142  table_desc = catalog.getMetadataForTable(*p->get_table_name());
1143  if (table_desc == nullptr) {
1144  throw std::runtime_error("Table " + *p->get_table_name() + " does not exist.");
1145  }
1146  std::string range_var;
1147  if (p->get_range_var() == nullptr) {
1148  range_var = *p->get_table_name();
1149  } else {
1150  range_var = *p->get_range_var();
1151  }
1152  rte = new Analyzer::RangeTableEntry(range_var, table_desc, nullptr);
1153  query.add_rte(rte);
1154  }
1155 }
1156 
1157 void QuerySpec::analyze(const Catalog_Namespace::Catalog& catalog,
1158  Analyzer::Query& query) const {
1160  analyze_from_clause(catalog, query);
1161  analyze_select_clause(catalog, query);
1162  analyze_where_clause(catalog, query);
1163  analyze_group_by(catalog, query);
1164  analyze_having_clause(catalog, query);
1165 }
1166 
1167 void SelectStmt::analyze(const Catalog_Namespace::Catalog& catalog,
1168  Analyzer::Query& query) const {
1169  query.set_stmt_type(kSELECT);
1170  query.set_limit(limit);
1171  if (offset < 0) {
1172  throw std::runtime_error("OFFSET cannot be negative.");
1173  }
1174  query.set_offset(offset);
1175  query_expr->analyze(catalog, query);
1176  if (orderby_clause.empty() && !query.get_is_distinct()) {
1177  query.set_order_by(nullptr);
1178  return;
1179  }
1180  const std::vector<std::shared_ptr<Analyzer::TargetEntry>>& tlist =
1181  query.get_targetlist();
1182  std::list<Analyzer::OrderEntry>* order_by = new std::list<Analyzer::OrderEntry>();
1183  if (!orderby_clause.empty()) {
1184  for (auto& p : orderby_clause) {
1185  int tle_no = p->get_colno();
1186  if (tle_no == 0) {
1187  // use column name
1188  // search through targetlist for matching name
1189  const std::string* name = p->get_column()->get_column();
1190  tle_no = 1;
1191  bool found = false;
1192  for (auto tle : tlist) {
1193  if (tle->get_resname() == *name) {
1194  found = true;
1195  break;
1196  }
1197  tle_no++;
1198  }
1199  if (!found) {
1200  throw std::runtime_error("invalid name in order by: " + *name);
1201  }
1202  }
1203  order_by->push_back(
1204  Analyzer::OrderEntry(tle_no, p->get_is_desc(), p->get_nulls_first()));
1205  }
1206  }
1207  if (query.get_is_distinct()) {
1208  // extend order_by to include all targetlist entries.
1209  for (int i = 1; i <= static_cast<int>(tlist.size()); i++) {
1210  bool in_orderby = false;
1211  std::for_each(order_by->begin(),
1212  order_by->end(),
1213  [&in_orderby, i](const Analyzer::OrderEntry& oe) {
1214  in_orderby = in_orderby || (i == oe.tle_no);
1215  });
1216  if (!in_orderby) {
1217  order_by->push_back(Analyzer::OrderEntry(i, false, false));
1218  }
1219  }
1220  }
1221  query.set_order_by(order_by);
1222 }
1223 
1224 std::string SQLType::to_string() const {
1225  std::string str;
1226  switch (type) {
1227  case kBOOLEAN:
1228  str = "BOOLEAN";
1229  break;
1230  case kCHAR:
1231  str = "CHAR(" + boost::lexical_cast<std::string>(param1) + ")";
1232  break;
1233  case kVARCHAR:
1234  str = "VARCHAR(" + boost::lexical_cast<std::string>(param1) + ")";
1235  break;
1236  case kTEXT:
1237  str = "TEXT";
1238  break;
1239  case kNUMERIC:
1240  str = "NUMERIC(" + boost::lexical_cast<std::string>(param1);
1241  if (param2 > 0) {
1242  str += ", " + boost::lexical_cast<std::string>(param2);
1243  }
1244  str += ")";
1245  break;
1246  case kDECIMAL:
1247  str = "DECIMAL(" + boost::lexical_cast<std::string>(param1);
1248  if (param2 > 0) {
1249  str += ", " + boost::lexical_cast<std::string>(param2);
1250  }
1251  str += ")";
1252  break;
1253  case kBIGINT:
1254  str = "BIGINT";
1255  break;
1256  case kINT:
1257  str = "INT";
1258  break;
1259  case kTINYINT:
1260  str = "TINYINT";
1261  break;
1262  case kSMALLINT:
1263  str = "SMALLINT";
1264  break;
1265  case kFLOAT:
1266  str = "FLOAT";
1267  break;
1268  case kDOUBLE:
1269  str = "DOUBLE";
1270  break;
1271  case kTIME:
1272  str = "TIME";
1273  if (param1 < 6) {
1274  str += "(" + boost::lexical_cast<std::string>(param1) + ")";
1275  }
1276  break;
1277  case kTIMESTAMP:
1278  str = "TIMESTAMP";
1279  if (param1 <= 9) {
1280  str += "(" + boost::lexical_cast<std::string>(param1) + ")";
1281  }
1282  break;
1283  case kDATE:
1284  str = "DATE";
1285  break;
1286  default:
1287  assert(false);
1288  break;
1289  }
1290  if (is_array) {
1291  str += "[";
1292  if (array_size > 0) {
1293  str += boost::lexical_cast<std::string>(array_size);
1294  }
1295  str += "]";
1296  }
1297  return str;
1298 }
1299 
1300 std::string SelectEntry::to_string() const {
1301  std::string str = select_expr->to_string();
1302  if (alias != nullptr) {
1303  str += " AS " + *alias;
1304  }
1305  return str;
1306 }
1307 
1308 std::string TableRef::to_string() const {
1309  std::string str = *table_name;
1310  if (range_var != nullptr) {
1311  str += " " + *range_var;
1312  }
1313  return str;
1314 }
1315 
1316 std::string ColumnRef::to_string() const {
1317  std::string str;
1318  if (table == nullptr) {
1319  str = *column;
1320  } else if (column == nullptr) {
1321  str = *table + ".*";
1322  } else {
1323  str = *table + "." + *column;
1324  }
1325  return str;
1326 }
1327 
1328 std::string OperExpr::to_string() const {
1329  std::string op_str[] = {
1330  "=", "===", "<>", "<", ">", "<=", ">=", " AND ", " OR ", "NOT", "-", "+", "*", "/"};
1331  std::string str;
1332  if (optype == kUMINUS) {
1333  str = "-(" + left->to_string() + ")";
1334  } else if (optype == kNOT) {
1335  str = "NOT (" + left->to_string() + ")";
1336  } else if (optype == kARRAY_AT) {
1337  str = left->to_string() + "[" + right->to_string() + "]";
1338  } else if (optype == kUNNEST) {
1339  str = "UNNEST(" + left->to_string() + ")";
1340  } else if (optype == kIN) {
1341  str = "(" + left->to_string() + " IN " + right->to_string() + ")";
1342  } else {
1343  str = "(" + left->to_string() + op_str[optype] + right->to_string() + ")";
1344  }
1345  return str;
1346 }
1347 
1348 std::string InExpr::to_string() const {
1349  std::string str = arg->to_string();
1350  if (is_not) {
1351  str += " NOT IN ";
1352  } else {
1353  str += " IN ";
1354  }
1355  return str;
1356 }
1357 
1358 std::string ExistsExpr::to_string() const {
1359  return "EXISTS (" + query->to_string() + ")";
1360 }
1361 
1362 std::string SubqueryExpr::to_string() const {
1363  std::string str;
1364  str = "(";
1365  str += query->to_string();
1366  str += ")";
1367  return str;
1368 }
1369 
1370 std::string IsNullExpr::to_string() const {
1371  std::string str = arg->to_string();
1372  if (is_not) {
1373  str += " IS NOT NULL";
1374  } else {
1375  str += " IS NULL";
1376  }
1377  return str;
1378 }
1379 
1380 std::string InSubquery::to_string() const {
1381  std::string str = InExpr::to_string();
1382  str += subquery->to_string();
1383  return str;
1384 }
1385 
1386 std::string InValues::to_string() const {
1387  std::string str = InExpr::to_string() + "(";
1388  bool notfirst = false;
1389  for (auto& p : value_list) {
1390  if (notfirst) {
1391  str += ", ";
1392  } else {
1393  notfirst = true;
1394  }
1395  str += p->to_string();
1396  }
1397  str += ")";
1398  return str;
1399 }
1400 
1401 std::string BetweenExpr::to_string() const {
1402  std::string str = arg->to_string();
1403  if (is_not) {
1404  str += " NOT BETWEEN ";
1405  } else {
1406  str += " BETWEEN ";
1407  }
1408  str += lower->to_string() + " AND " + upper->to_string();
1409  return str;
1410 }
1411 
1412 std::string CharLengthExpr::to_string() const {
1413  std::string str;
1414  if (calc_encoded_length) {
1415  str = "CHAR_LENGTH (" + arg->to_string() + ")";
1416  } else {
1417  str = "LENGTH (" + arg->to_string() + ")";
1418  }
1419  return str;
1420 }
1421 
1422 std::string CardinalityExpr::to_string() const {
1423  std::string str = "CARDINALITY(" + arg->to_string() + ")";
1424  return str;
1425 }
1426 
1427 std::string LikeExpr::to_string() const {
1428  std::string str = arg->to_string();
1429  if (is_not) {
1430  str += " NOT LIKE ";
1431  } else {
1432  str += " LIKE ";
1433  }
1434  str += like_string->to_string();
1435  if (escape_string != nullptr) {
1436  str += " ESCAPE " + escape_string->to_string();
1437  }
1438  return str;
1439 }
1440 
1441 std::string RegexpExpr::to_string() const {
1442  std::string str = arg->to_string();
1443  if (is_not) {
1444  str += " NOT REGEXP ";
1445  } else {
1446  str += " REGEXP ";
1447  }
1448  str += pattern_string->to_string();
1449  if (escape_string != nullptr) {
1450  str += " ESCAPE " + escape_string->to_string();
1451  }
1452  return str;
1453 }
1454 
1455 std::string LikelihoodExpr::to_string() const {
1456  std::string str = " LIKELIHOOD ";
1457  str += arg->to_string();
1458  str += " ";
1459  str += boost::lexical_cast<std::string>(is_not ? 1.0 - likelihood : likelihood);
1460  return str;
1461 }
1462 
1463 std::string FunctionRef::to_string() const {
1464  std::string str = *name + "(";
1465  if (distinct) {
1466  str += "DISTINCT ";
1467  }
1468  if (arg == nullptr) {
1469  str += "*)";
1470  } else {
1471  str += arg->to_string() + ")";
1472  }
1473  return str;
1474 }
1475 
1476 std::string QuerySpec::to_string() const {
1477  std::string query_str = "SELECT ";
1478  if (is_distinct) {
1479  query_str += "DISTINCT ";
1480  }
1481  if (select_clause.empty()) {
1482  query_str += "* ";
1483  } else {
1484  bool notfirst = false;
1485  for (auto& p : select_clause) {
1486  if (notfirst) {
1487  query_str += ", ";
1488  } else {
1489  notfirst = true;
1490  }
1491  query_str += p->to_string();
1492  }
1493  }
1494  query_str += " FROM ";
1495  bool notfirst = false;
1496  for (auto& p : from_clause) {
1497  if (notfirst) {
1498  query_str += ", ";
1499  } else {
1500  notfirst = true;
1501  }
1502  query_str += p->to_string();
1503  }
1504  if (where_clause) {
1505  query_str += " WHERE " + where_clause->to_string();
1506  }
1507  if (!groupby_clause.empty()) {
1508  query_str += " GROUP BY ";
1509  bool notfirst = false;
1510  for (auto& p : groupby_clause) {
1511  if (notfirst) {
1512  query_str += ", ";
1513  } else {
1514  notfirst = true;
1515  }
1516  query_str += p->to_string();
1517  }
1518  }
1519  if (having_clause) {
1520  query_str += " HAVING " + having_clause->to_string();
1521  }
1522  query_str += ";";
1523  return query_str;
1524 }
1525 
1526 void InsertStmt::analyze(const Catalog_Namespace::Catalog& catalog,
1527  Analyzer::Query& query) const {
1528  query.set_stmt_type(kINSERT);
1529  const TableDescriptor* td = catalog.getMetadataForTable(*table);
1530  if (td == nullptr) {
1531  throw std::runtime_error("Table " + *table + " does not exist.");
1532  }
1533  if (td->isView) {
1534  throw std::runtime_error("Insert to views is not supported yet.");
1535  }
1536  query.set_result_table_id(td->tableId);
1537  std::list<int> result_col_list;
1538  if (column_list.empty()) {
1539  const std::list<const ColumnDescriptor*> all_cols =
1540  catalog.getAllColumnMetadataForTable(td->tableId, false, false, true);
1541  for (auto cd : all_cols) {
1542  result_col_list.push_back(cd->columnId);
1543  }
1544  } else {
1545  for (auto& c : column_list) {
1546  const ColumnDescriptor* cd = catalog.getMetadataForColumn(td->tableId, *c);
1547  if (cd == nullptr) {
1548  throw std::runtime_error("Column " + *c + " does not exist.");
1549  }
1550  result_col_list.push_back(cd->columnId);
1551  const auto& col_ti = cd->columnType;
1552  if (col_ti.get_physical_cols() > 0) {
1553  CHECK(cd->columnType.is_geometry());
1554  for (auto i = 1; i <= col_ti.get_physical_cols(); i++) {
1555  const ColumnDescriptor* pcd =
1556  catalog.getMetadataForColumn(td->tableId, cd->columnId + i);
1557  if (pcd == nullptr) {
1558  throw std::runtime_error("Column " + *c + "'s metadata is incomplete.");
1559  }
1560  result_col_list.push_back(pcd->columnId);
1561  }
1562  }
1563  }
1564  if (catalog.getAllColumnMetadataForTable(td->tableId, false, false, true).size() !=
1565  result_col_list.size()) {
1566  throw std::runtime_error("Insert into a subset of columns is not supported yet.");
1567  }
1568  }
1569  query.set_result_col_list(result_col_list);
1570 }
1571 
1572 size_t InsertValuesStmt::determineLeafIndex(const Catalog_Namespace::Catalog& catalog,
1573  size_t num_leafs) {
1574  const TableDescriptor* td = catalog.getMetadataForTable(*table);
1575  if (td == nullptr) {
1576  throw std::runtime_error("Table " + *table + " does not exist.");
1577  }
1578  if (td->isView) {
1579  throw std::runtime_error("Insert to views is not supported yet.");
1580  }
1581 
1582  if (td->partitions == "REPLICATED") {
1583  throw std::runtime_error("Cannot determine leaf on replicated table.");
1584  }
1585 
1586  if (0 == td->nShards) {
1587  std::random_device rd;
1588  std::mt19937_64 gen(rd());
1589  std::uniform_int_distribution<size_t> dis;
1590  const auto leaf_idx = dis(gen) % num_leafs;
1591  return leaf_idx;
1592  }
1593 
1594  size_t indexOfShardColumn = 0;
1595  const ColumnDescriptor* shardColumn = catalog.getShardColumnMetadataForTable(td);
1596  CHECK(shardColumn);
1597 
1598  if (column_list.empty()) {
1599  auto all_cols =
1600  catalog.getAllColumnMetadataForTable(td->tableId, false, false, false);
1601  auto iter = std::find(all_cols.begin(), all_cols.end(), shardColumn);
1602  CHECK(iter != all_cols.end());
1603  indexOfShardColumn = std::distance(all_cols.begin(), iter);
1604  } else {
1605  for (auto& c : column_list) {
1606  if (*c == shardColumn->columnName) {
1607  break;
1608  }
1609  indexOfShardColumn++;
1610  }
1611 
1612  if (indexOfShardColumn == column_list.size()) {
1613  throw std::runtime_error("No value defined for shard column.");
1614  }
1615  }
1616 
1617  if (indexOfShardColumn >= value_list.size()) {
1618  throw std::runtime_error("No value defined for shard column.");
1619  }
1620 
1621  auto& shardColumnValueExpr = *(std::next(value_list.begin(), indexOfShardColumn));
1622 
1624  auto e = shardColumnValueExpr->analyze(catalog, query);
1625  e = e->add_cast(shardColumn->columnType);
1626  const Analyzer::Constant* con = dynamic_cast<Analyzer::Constant*>(e.get());
1627  if (!con) {
1628  auto col_cast = dynamic_cast<const Analyzer::UOper*>(e.get());
1629  CHECK(col_cast);
1630  CHECK_EQ(kCAST, col_cast->get_optype());
1631  con = dynamic_cast<const Analyzer::Constant*>(col_cast->get_operand());
1632  }
1633  CHECK(con);
1634 
1635  Datum d = con->get_constval();
1636 
1637  auto shard_count = td->nShards * num_leafs;
1638  int64_t shardId = 0;
1639 
1640  if (con->get_is_null()) {
1642  shard_count);
1643  } else if (shardColumn->columnType.is_string()) {
1644  auto dictDesc =
1645  catalog.getMetadataForDict(shardColumn->columnType.get_comp_param(), true);
1646  auto str_id = dictDesc->stringDict->getOrAdd(*d.stringval);
1647  bool invalid = false;
1648 
1649  if (4 == shardColumn->columnType.get_size()) {
1650  invalid = str_id > max_valid_int_value<int32_t>();
1651  } else if (2 == shardColumn->columnType.get_size()) {
1652  invalid = str_id > max_valid_int_value<uint16_t>();
1653  } else if (1 == shardColumn->columnType.get_size()) {
1654  invalid = str_id > max_valid_int_value<uint8_t>();
1655  }
1656 
1657  if (invalid || str_id == inline_int_null_value<int32_t>()) {
1658  str_id = inline_fixed_encoding_null_val(shardColumn->columnType);
1659  }
1660  shardId = SHARD_FOR_KEY(str_id, shard_count);
1661  } else {
1662  switch (shardColumn->columnType.get_logical_size()) {
1663  case 8:
1664  shardId = SHARD_FOR_KEY(d.bigintval, shard_count);
1665  break;
1666  case 4:
1667  shardId = SHARD_FOR_KEY(d.intval, shard_count);
1668  break;
1669  case 2:
1670  shardId = SHARD_FOR_KEY(d.smallintval, shard_count);
1671  break;
1672  case 1:
1673  shardId = SHARD_FOR_KEY(d.tinyintval, shard_count);
1674  break;
1675  default:
1676  CHECK(false);
1677  }
1678  }
1679 
1680  return shardId / td->nShards;
1681 }
1682 
1683 void InsertValuesStmt::analyze(const Catalog_Namespace::Catalog& catalog,
1684  Analyzer::Query& query) const {
1685  InsertStmt::analyze(catalog, query);
1686  std::vector<std::shared_ptr<Analyzer::TargetEntry>>& tlist =
1687  query.get_targetlist_nonconst();
1688  const auto tableId = query.get_result_table_id();
1689  const std::list<const ColumnDescriptor*> non_phys_cols =
1690  catalog.getAllColumnMetadataForTable(tableId, false, false, false);
1691  if (non_phys_cols.size() != value_list.size()) {
1692  throw std::runtime_error("Insert has more target columns than expressions.");
1693  }
1694  std::list<int>::const_iterator it = query.get_result_col_list().begin();
1695  for (auto& v : value_list) {
1696  auto e = v->analyze(catalog, query);
1697  const ColumnDescriptor* cd =
1698  catalog.getMetadataForColumn(query.get_result_table_id(), *it);
1699  assert(cd != nullptr);
1700  if (cd->columnType.get_notnull()) {
1701  auto c = std::dynamic_pointer_cast<Analyzer::Constant>(e);
1702  if (c != nullptr && c->get_is_null()) {
1703  throw std::runtime_error("Cannot insert NULL into column " + cd->columnName);
1704  }
1705  }
1706  e = e->add_cast(cd->columnType);
1707  tlist.emplace_back(new Analyzer::TargetEntry("", e, false));
1708  ++it;
1709 
1710  const auto& col_ti = cd->columnType;
1711  if (col_ti.get_physical_cols() > 0) {
1712  CHECK(cd->columnType.is_geometry());
1713  std::string* wkt{nullptr};
1714  auto c = std::dynamic_pointer_cast<Analyzer::Constant>(e);
1715  if (c) {
1716  wkt = c->get_constval().stringval;
1717  } else {
1718  auto uoper = std::dynamic_pointer_cast<Analyzer::UOper>(e);
1719  if (uoper && uoper->get_optype() == kCAST) {
1720  auto c = dynamic_cast<const Analyzer::Constant*>(uoper->get_operand());
1721  if (c) {
1722  wkt = c->get_constval().stringval;
1723  }
1724  }
1725  }
1726  if (!wkt) {
1727  throw std::runtime_error("Expecting a WKT string for column " + cd->columnName);
1728  }
1729  std::vector<double> coords;
1730  std::vector<double> bounds;
1731  std::vector<int> ring_sizes;
1732  std::vector<int> poly_rings;
1733  int render_group =
1734  0; // @TODO simon.eves where to get render_group from in this context?!
1735  SQLTypeInfo import_ti;
1737  *wkt, import_ti, coords, bounds, ring_sizes, poly_rings)) {
1738  throw std::runtime_error("Cannot read geometry to insert into column " +
1739  cd->columnName);
1740  }
1741  if (cd->columnType.get_type() != import_ti.get_type()) {
1742  // allow POLYGON to be inserted into MULTIPOLYGON column
1743  if (!(import_ti.get_type() == SQLTypes::kPOLYGON &&
1745  throw std::runtime_error("Imported geometry doesn't match the type of column " +
1746  cd->columnName);
1747  }
1748  }
1749  // TODO: check if import SRID matches columns SRID, may need to transform before
1750  // inserting
1751 
1752  int nextColumnOffset = 1;
1753 
1754  const ColumnDescriptor* cd_coords = catalog.getMetadataForColumn(
1755  query.get_result_table_id(), cd->columnId + nextColumnOffset);
1756  CHECK(cd_coords);
1757  CHECK_EQ(cd_coords->columnType.get_type(), kARRAY);
1758  CHECK_EQ(cd_coords->columnType.get_subtype(), kTINYINT);
1759  auto compressed_coords = Importer_NS::compress_coords(coords, col_ti);
1760  std::list<std::shared_ptr<Analyzer::Expr>> value_exprs;
1761  for (auto cc : compressed_coords) {
1762  Datum d;
1763  d.tinyintval = cc;
1764  auto e = makeExpr<Analyzer::Constant>(kTINYINT, false, d);
1765  value_exprs.push_back(e);
1766  }
1767  tlist.emplace_back(new Analyzer::TargetEntry(
1768  "",
1769  makeExpr<Analyzer::Constant>(cd_coords->columnType, false, value_exprs),
1770  false));
1771  ++it;
1772  nextColumnOffset++;
1773 
1774  if (cd->columnType.get_type() == kPOLYGON ||
1775  cd->columnType.get_type() == kMULTIPOLYGON) {
1776  // Put ring sizes array into separate physical column
1777  const ColumnDescriptor* cd_ring_sizes = catalog.getMetadataForColumn(
1778  query.get_result_table_id(), cd->columnId + nextColumnOffset);
1779  CHECK(cd_ring_sizes);
1780  CHECK_EQ(cd_ring_sizes->columnType.get_type(), kARRAY);
1781  CHECK_EQ(cd_ring_sizes->columnType.get_subtype(), kINT);
1782  std::list<std::shared_ptr<Analyzer::Expr>> value_exprs;
1783  for (auto c : ring_sizes) {
1784  Datum d;
1785  d.intval = c;
1786  auto e = makeExpr<Analyzer::Constant>(kINT, false, d);
1787  value_exprs.push_back(e);
1788  }
1789  tlist.emplace_back(new Analyzer::TargetEntry(
1790  "",
1791  makeExpr<Analyzer::Constant>(cd_ring_sizes->columnType, false, value_exprs),
1792  false));
1793  ++it;
1794  nextColumnOffset++;
1795 
1796  if (cd->columnType.get_type() == kMULTIPOLYGON) {
1797  // Put poly_rings array into separate physical column
1798  const ColumnDescriptor* cd_poly_rings = catalog.getMetadataForColumn(
1799  query.get_result_table_id(), cd->columnId + nextColumnOffset);
1800  CHECK(cd_poly_rings);
1801  CHECK_EQ(cd_poly_rings->columnType.get_type(), kARRAY);
1802  CHECK_EQ(cd_poly_rings->columnType.get_subtype(), kINT);
1803  std::list<std::shared_ptr<Analyzer::Expr>> value_exprs;
1804  for (auto c : poly_rings) {
1805  Datum d;
1806  d.intval = c;
1807  auto e = makeExpr<Analyzer::Constant>(kINT, false, d);
1808  value_exprs.push_back(e);
1809  }
1810  tlist.emplace_back(new Analyzer::TargetEntry(
1811  "",
1812  makeExpr<Analyzer::Constant>(cd_poly_rings->columnType, false, value_exprs),
1813  false));
1814  ++it;
1815  nextColumnOffset++;
1816  }
1817  }
1818 
1819  if (cd->columnType.get_type() == kLINESTRING ||
1820  cd->columnType.get_type() == kPOLYGON ||
1821  cd->columnType.get_type() == kMULTIPOLYGON) {
1822  const ColumnDescriptor* cd_bounds = catalog.getMetadataForColumn(
1823  query.get_result_table_id(), cd->columnId + nextColumnOffset);
1824  CHECK(cd_bounds);
1825  CHECK_EQ(cd_bounds->columnType.get_type(), kARRAY);
1826  CHECK_EQ(cd_bounds->columnType.get_subtype(), kDOUBLE);
1827  std::list<std::shared_ptr<Analyzer::Expr>> value_exprs;
1828  for (auto b : bounds) {
1829  Datum d;
1830  d.doubleval = b;
1831  auto e = makeExpr<Analyzer::Constant>(kDOUBLE, false, d);
1832  value_exprs.push_back(e);
1833  }
1834  tlist.emplace_back(new Analyzer::TargetEntry(
1835  "",
1836  makeExpr<Analyzer::Constant>(cd_bounds->columnType, false, value_exprs),
1837  false));
1838  ++it;
1839  nextColumnOffset++;
1840  }
1841 
1842  if (cd->columnType.get_type() == kPOLYGON ||
1843  cd->columnType.get_type() == kMULTIPOLYGON) {
1844  // Put render group into separate physical column
1845  const ColumnDescriptor* cd_render_group = catalog.getMetadataForColumn(
1846  query.get_result_table_id(), cd->columnId + nextColumnOffset);
1847  CHECK(cd_render_group);
1848  CHECK_EQ(cd_render_group->columnType.get_type(), kINT);
1849  Datum d;
1850  d.intval = render_group;
1851  tlist.emplace_back(new Analyzer::TargetEntry(
1852  "",
1853  makeExpr<Analyzer::Constant>(cd_render_group->columnType, false, d),
1854  false));
1855  ++it;
1856  nextColumnOffset++;
1857  }
1858  }
1859  }
1860 }
1861 
1862 void InsertQueryStmt::analyze(const Catalog_Namespace::Catalog& catalog,
1863  Analyzer::Query& insert_query) const {
1864  InsertStmt::analyze(catalog, insert_query);
1865  query->analyze(catalog, insert_query);
1866 }
1867 
1868 void UpdateStmt::analyze(const Catalog_Namespace::Catalog& catalog,
1869  Analyzer::Query& query) const {
1870  throw std::runtime_error("UPDATE statement not supported yet.");
1871 }
1872 
1873 void DeleteStmt::analyze(const Catalog_Namespace::Catalog& catalog,
1874  Analyzer::Query& query) const {
1875  throw std::runtime_error("DELETE statement not supported yet.");
1876 }
1877 
1878 void SQLType::check_type() {
1879  switch (type) {
1880  case kCHAR:
1881  case kVARCHAR:
1882  if (param1 <= 0) {
1883  throw std::runtime_error("CHAR and VARCHAR must have a positive dimension.");
1884  }
1885  break;
1886  case kDECIMAL:
1887  case kNUMERIC:
1888  if (param1 <= 0) {
1889  throw std::runtime_error("DECIMAL and NUMERIC must have a positive precision.");
1890  } else if (param1 > 19) {
1891  throw std::runtime_error(
1892  "DECIMAL and NUMERIC precision cannot be larger than 19.");
1893  } else if (param1 <= param2) {
1894  throw std::runtime_error(
1895  "DECIMAL and NUMERIC must have precision larger than scale.");
1896  }
1897  break;
1898  case kTIMESTAMP:
1899  if (param1 == -1) {
1900  param1 = 0; // set default to 0
1901  } else if (param1 != 0 && param1 != 3 && param1 != 6 &&
1902  param1 != 9) { // support ms, us, ns
1903  throw std::runtime_error(
1904  "Only TIMESTAMP(n) where n = (0,3,6,9) are supported now.");
1905  }
1906  break;
1907  case kTIME:
1908  if (param1 == -1) {
1909  param1 = 0; // default precision is 0
1910  }
1911  if (param1 > 0) { // @TODO(wei) support sub-second precision later.
1912  throw std::runtime_error("Only TIME(0) is supported now.");
1913  }
1914  break;
1915  case kPOINT:
1916  case kLINESTRING:
1917  case kPOLYGON:
1918  case kMULTIPOLYGON:
1919  // Storing SRID in param1
1920  break;
1921  default:
1922  param1 = 0;
1923  break;
1924  }
1925 }
1926 
1927 namespace {
1928 
1930  const auto& col_ti = cd.columnType;
1931  if (col_ti.is_integer() ||
1932  (col_ti.is_string() && col_ti.get_compression() == kENCODING_DICT) ||
1933  col_ti.is_time()) {
1934  return;
1935  }
1936  throw std::runtime_error("Cannot shard on type " + col_ti.get_type_name() +
1937  ", encoding " + col_ti.get_compression_name());
1938 }
1939 
1940 size_t shard_column_index(const std::string& name,
1941  const std::list<ColumnDescriptor>& columns) {
1942  size_t index = 1;
1943  for (const auto& cd : columns) {
1944  if (cd.columnName == name) {
1946  return index;
1947  }
1948  ++index;
1949  if (cd.columnType.is_geometry()) {
1950  index += cd.columnType.get_physical_cols();
1951  }
1952  }
1953  // Not found, return 0
1954  return 0;
1955 }
1956 
1957 size_t sort_column_index(const std::string& name,
1958  const std::list<ColumnDescriptor>& columns) {
1959  size_t index = 1;
1960  for (const auto& cd : columns) {
1961  if (boost::to_upper_copy<std::string>(cd.columnName) == name) {
1962  return index;
1963  }
1964  ++index;
1965  if (cd.columnType.is_geometry()) {
1966  index += cd.columnType.get_physical_cols();
1967  }
1968  }
1969  // Not found, return 0
1970  return 0;
1971 }
1972 
1973 void set_string_field(rapidjson::Value& obj,
1974  const std::string& field_name,
1975  const std::string& field_value,
1976  rapidjson::Document& document) {
1977  rapidjson::Value field_name_json_str;
1978  field_name_json_str.SetString(
1979  field_name.c_str(), field_name.size(), document.GetAllocator());
1980  rapidjson::Value field_value_json_str;
1981  field_value_json_str.SetString(
1982  field_value.c_str(), field_value.size(), document.GetAllocator());
1983  obj.AddMember(field_name_json_str, field_value_json_str, document.GetAllocator());
1984 }
1985 
1987  const ShardKeyDef* shard_key_def,
1988  const std::vector<SharedDictionaryDef>& shared_dict_defs) {
1989  rapidjson::Document document;
1990  auto& allocator = document.GetAllocator();
1991  rapidjson::Value arr(rapidjson::kArrayType);
1992  if (shard_key_def) {
1993  rapidjson::Value shard_key_obj(rapidjson::kObjectType);
1994  set_string_field(shard_key_obj, "type", "SHARD KEY", document);
1995  set_string_field(shard_key_obj, "name", shard_key_def->get_column(), document);
1996  arr.PushBack(shard_key_obj, allocator);
1997  }
1998  for (const auto& shared_dict_def : shared_dict_defs) {
1999  rapidjson::Value shared_dict_obj(rapidjson::kObjectType);
2000  set_string_field(shared_dict_obj, "type", "SHARED DICTIONARY", document);
2001  set_string_field(shared_dict_obj, "name", shared_dict_def.get_column(), document);
2003  shared_dict_obj, "foreign_table", shared_dict_def.get_foreign_table(), document);
2004  set_string_field(shared_dict_obj,
2005  "foreign_column",
2006  shared_dict_def.get_foreign_column(),
2007  document);
2008  arr.PushBack(shared_dict_obj, allocator);
2009  }
2010  rapidjson::StringBuffer buffer;
2011  rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
2012  arr.Accept(writer);
2013  return buffer.GetString();
2014 }
2015 
2016 template <typename LITERAL_TYPE,
2017  typename ASSIGNMENT,
2018  typename VALIDATE = DefaultValidate<LITERAL_TYPE>>
2019 decltype(auto) get_property_value(const NameValueAssign* p,
2020  ASSIGNMENT op,
2021  VALIDATE validate = VALIDATE()) {
2022  const auto val = validate(p);
2023  return op(val);
2024 }
2025 
2027  const NameValueAssign* p,
2028  const std::list<ColumnDescriptor>& columns) {
2029  return get_property_value<IntLiteral>(p,
2030  [&td](const auto val) { td.maxFragRows = val; });
2031 }
2032 
2034  const NameValueAssign* p,
2035  const std::list<ColumnDescriptor>& columns) {
2036  return get_property_value<IntLiteral>(p,
2037  [&td](const auto val) { td.maxChunkSize = val; });
2038 }
2039 
2041  const NameValueAssign* p,
2042  const std::list<ColumnDescriptor>& columns) {
2043  return get_property_value<IntLiteral>(p,
2044  [&td](const auto val) { td.fragPageSize = val; });
2045 }
2047  const NameValueAssign* p,
2048  const std::list<ColumnDescriptor>& columns) {
2049  return get_property_value<IntLiteral>(p, [&td](const auto val) { td.maxRows = val; });
2050 }
2052  const NameValueAssign* p,
2053  const std::list<ColumnDescriptor>& columns) {
2054  return get_property_value<StringLiteral>(p, [&td](const auto partitions_uc) {
2055  if (partitions_uc != "SHARDED" && partitions_uc != "REPLICATED") {
2056  throw std::runtime_error("PARTITIONS must be SHARDED or REPLICATED");
2057  }
2058  if (td.shardedColumnId != 0 && partitions_uc == "REPLICATED") {
2059  throw std::runtime_error(
2060  "A table cannot be sharded and replicated at the same time");
2061  };
2062  td.partitions = partitions_uc;
2063  });
2064 }
2066  const NameValueAssign* p,
2067  const std::list<ColumnDescriptor>& columns) {
2068  if (!td.shardedColumnId) {
2069  throw std::runtime_error("SHARD KEY must be defined.");
2070  }
2071  return get_property_value<IntLiteral>(p, [&td](const auto shard_count) {
2072  if (g_leaf_count && shard_count % g_leaf_count) {
2073  throw std::runtime_error(
2074  "SHARD_COUNT must be a multiple of the number of leaves in the cluster.");
2075  }
2076  td.nShards = g_leaf_count ? shard_count / g_leaf_count : shard_count;
2077  if (!td.shardedColumnId && !td.nShards) {
2078  throw std::runtime_error(
2079  "Must specify the number of shards through the SHARD_COUNT option");
2080  };
2081  });
2082 }
2083 
2084 decltype(auto) get_vacuum_def(TableDescriptor& td,
2085  const NameValueAssign* p,
2086  const std::list<ColumnDescriptor>& columns) {
2087  return get_property_value<StringLiteral>(p, [&td](const auto vacuum_uc) {
2088  if (vacuum_uc != "IMMEDIATE" && vacuum_uc != "DELAYED") {
2089  throw std::runtime_error("VACUUM must be IMMEDIATE or DELAYED");
2090  }
2091  td.hasDeletedCol = boost::iequals(vacuum_uc, "IMMEDIATE") ? false : true;
2092  });
2093 }
2094 
2096  const NameValueAssign* p,
2097  const std::list<ColumnDescriptor>& columns) {
2098  return get_property_value<StringLiteral>(p, [&td, &columns](const auto sort_upper) {
2099  td.sortedColumnId = sort_column_index(sort_upper, columns);
2100  if (!td.sortedColumnId) {
2101  throw std::runtime_error("Specified sort column " + sort_upper + " doesn't exist");
2102  }
2103  });
2104 }
2105 
2106 static const std::map<const std::string, const TableDefFuncPtr> tableDefFuncMap = {
2107  {"fragment_size"s, get_frag_size_def},
2108  {"max_chunk_size"s, get_max_chunk_size_def},
2109  {"page_size"s, get_page_size_def},
2110  {"max_rows"s, get_max_rows_def},
2111  {"partitions"s, get_partions_def},
2112  {"shard_count"s, get_shard_count_def},
2113  {"vacuum"s, get_vacuum_def},
2114  {"sort_column"s, get_sort_column_def}};
2115 
2117  const std::unique_ptr<NameValueAssign>& p,
2118  const std::list<ColumnDescriptor>& columns) {
2119  const auto it = tableDefFuncMap.find(boost::to_lower_copy<std::string>(*p->get_name()));
2120  if (it == tableDefFuncMap.end()) {
2121  throw std::runtime_error("Invalid CREATE TABLE option " + *p->get_name() +
2122  ". Should be FRAGMENT_SIZE, PAGE_SIZE, MAX_ROWS, "
2123  "PARTITIONS, VACUUM, SORT_COLUMN or SHARD_COUNT.");
2124  }
2125  return it->second(td, p.get(), columns);
2126 }
2127 
2128 } // namespace
2129 
2130 void CreateTableStmt::execute(const Catalog_Namespace::SessionInfo& session) {
2131  auto& catalog = session.getCatalog();
2132  // check access privileges
2135  throw std::runtime_error("Table " + *table_ +
2136  " will not be created. User has no create privileges.");
2137  }
2138 
2139  if (catalog.getMetadataForTable(*table_) != nullptr) {
2140  if (if_not_exists_) {
2141  return;
2142  }
2143  throw std::runtime_error("Table " + *table_ + " already exists.");
2144  }
2145  std::list<ColumnDescriptor> columns;
2146  std::unordered_set<std::string> uc_col_names;
2147  std::vector<SharedDictionaryDef> shared_dict_defs;
2148  const ShardKeyDef* shard_key_def{nullptr};
2149  for (auto& e : table_element_list_) {
2150  if (dynamic_cast<SharedDictionaryDef*>(e.get())) {
2151  auto shared_dict_def = static_cast<SharedDictionaryDef*>(e.get());
2153  this, shared_dict_def, columns, shared_dict_defs, catalog);
2154  shared_dict_defs.push_back(*shared_dict_def);
2155  continue;
2156  }
2157  if (dynamic_cast<ShardKeyDef*>(e.get())) {
2158  if (shard_key_def) {
2159  throw std::runtime_error("Specified more than one shard key");
2160  }
2161  shard_key_def = static_cast<const ShardKeyDef*>(e.get());
2162  continue;
2163  }
2164  if (!dynamic_cast<ColumnDef*>(e.get())) {
2165  throw std::runtime_error("Table constraints are not supported yet.");
2166  }
2167  ColumnDef* coldef = static_cast<ColumnDef*>(e.get());
2168  ColumnDescriptor cd;
2169  cd.columnName = *coldef->get_column_name();
2170  const auto uc_col_name = boost::to_upper_copy<std::string>(cd.columnName);
2171  if (reserved_keywords.find(uc_col_name) != reserved_keywords.end()) {
2172  throw std::runtime_error("Cannot create column with reserved keyword '" +
2173  cd.columnName + "'");
2174  }
2175  const auto it_ok = uc_col_names.insert(uc_col_name);
2176  if (!it_ok.second) {
2177  throw std::runtime_error("Column '" + cd.columnName + "' defined more than once");
2178  }
2179 
2180  setColumnDescriptor(cd, coldef);
2181  columns.push_back(cd);
2182  }
2183 
2184  TableDescriptor td;
2185  td.tableName = *table_;
2186  td.userId = session.get_currentUser().userId;
2187  td.nColumns = columns.size();
2188  td.isView = false;
2189  td.fragmenter = nullptr;
2195  if (shard_key_def) {
2196  td.shardedColumnId = shard_column_index(shard_key_def->get_column(), columns);
2197  if (!td.shardedColumnId) {
2198  throw std::runtime_error("Specified shard column " + shard_key_def->get_column() +
2199  " doesn't exist");
2200  }
2201  }
2202  if (is_temporary_) {
2204  } else {
2206  }
2207  if (!storage_options_.empty()) {
2208  for (auto& p : storage_options_) {
2209  get_table_definitions(td, p, columns);
2210  }
2211  }
2212  td.keyMetainfo = serialize_key_metainfo(shard_key_def, shared_dict_defs);
2213  catalog.createShardedTable(td, columns, shared_dict_defs);
2214  // TODO (max): It's transactionally unsafe, should be fixed: we may create object w/o
2215  // privileges
2216  SysCatalog::instance().createDBObject(
2217  session.get_currentUser(), td.tableName, TableDBObjectType, catalog);
2218 }
2219 
2220 std::shared_ptr<ResultSet> getResultSet(const Catalog_Namespace::SessionInfo& session,
2221  const std::string select_stmt,
2222  std::vector<TargetMetaInfo>& targets,
2223  bool validate_only = false) {
2224  auto& catalog = session.getCatalog();
2225 
2226  auto executor = Executor::getExecutor(catalog.getCurrentDB().dbId);
2227 
2228 #ifdef HAVE_CUDA
2229  const auto device_type = session.get_executor_device_type();
2230 #else
2231  const auto device_type = ExecutorDeviceType::CPU;
2232 #endif // HAVE_CUDA
2233  auto calcite_mgr = catalog.getCalciteMgr();
2234 
2235  // TODO MAT this should actually get the global or the session parameter for
2236  // view optimization
2237  const auto query_ra =
2238  calcite_mgr->process(session, pg_shim(select_stmt), {}, true, false, false)
2239  .plan_result;
2240  CompilationOptions co = {
2241  device_type, true, ExecutorOptLevel::LoopStrengthReduction, false};
2242  // TODO(adb): Need a better method of dropping constants into this ExecutionOptions
2243  // struct
2244  ExecutionOptions eo = {false,
2245  true,
2246  false,
2247  true,
2248  false,
2249  false,
2250  validate_only,
2251  false,
2252  10000,
2253  false,
2254  false,
2255  0.9};
2256  RelAlgExecutor ra_executor(executor.get(), catalog);
2257  ExecutionResult result{std::make_shared<ResultSet>(std::vector<TargetInfo>{},
2260  nullptr,
2261  nullptr),
2262  {}};
2263  result = ra_executor.executeRelAlgQuery(query_ra, co, eo, nullptr);
2264  targets = result.getTargetsMeta();
2265 
2266  return result.getRows();
2267 }
2268 
2271  std::string& sql_query_string,
2272  bool validate_only) {
2273  auto& catalog = session.getCatalog();
2274  std::string pg_shimmed_select_query = pg_shim(sql_query_string);
2275  const auto query_ra = parse_to_ra(catalog, pg_shimmed_select_query, session);
2276 
2277  // Note that the below command should *only* get read locks on the source table for
2278  // InsertIntoAsSelect. Write locks will be taken on the target table in
2279  // InsertOrderFragmenter::deleteFragments. This potentially leaves a small window where
2280  // the target table is not locked, and should be investigated.
2281  std::vector<TableLock> table_locks;
2282  TableLockMgr::getTableLocks(session.getCatalog(), query_ra, table_locks);
2283 
2284  std::vector<TargetMetaInfo> target_metainfos;
2285 
2286  auto result_rows =
2287  getResultSet(session, sql_query_string, target_metainfos, validate_only);
2288  AggregatedResult res = {result_rows, target_metainfos};
2289  return res;
2290 }
2291 void InsertIntoTableAsSelectStmt::LocalConnector::insertDataToLeaf(
2293  const size_t leaf_idx,
2294  Fragmenter_Namespace::InsertData& insert_data) {
2295  CHECK(leaf_idx == 0);
2296  auto& catalog = session.getCatalog();
2297  auto created_td = catalog.getMetadataForTable(insert_data.tableId);
2298  // get CheckpointLock on the table before trying to create
2299  // its 1st fragment
2300  ChunkKey chunkKey = {catalog.getCurrentDB().dbId, created_td->tableId};
2301  mapd_unique_lock<mapd_shared_mutex> chkptlLock(
2304  // [ TableWriteLock ] lock is deferred in
2305  // InsertOrderFragmenter::deleteFragments
2306  created_td->fragmenter->insertDataNoCheckpoint(insert_data);
2307 }
2308 
2309 void InsertIntoTableAsSelectStmt::LocalConnector::checkpoint(
2311  int tableId) {
2312  auto& catalog = session.getCatalog();
2313  auto dbId = catalog.getCurrentDB().dbId;
2314  catalog.getDataMgr().checkpoint(dbId, tableId);
2315 }
2316 
2317 void InsertIntoTableAsSelectStmt::LocalConnector::rollback(
2319  int tableId) {
2320  auto& catalog = session.getCatalog();
2321  auto dbId = catalog.getCurrentDB().dbId;
2322  catalog.getDataMgr().checkpoint(dbId, tableId);
2323 }
2324 
2325 std::list<ColumnDescriptor>
2326 InsertIntoTableAsSelectStmt::LocalConnector::getColumnDescriptors(
2328  bool for_create) {
2329  std::list<ColumnDescriptor> column_descriptors;
2330  std::list<ColumnDescriptor> column_descriptors_for_create;
2331 
2332  int rowid_suffix = 0;
2333  for (const auto& target_metainfo : result.targets_meta) {
2334  ColumnDescriptor cd;
2335  cd.columnName = target_metainfo.get_resname();
2336  if (cd.columnName == "rowid") {
2337  cd.columnName += std::to_string(rowid_suffix++);
2338  }
2339  cd.columnType = target_metainfo.get_physical_type_info();
2340 
2341  ColumnDescriptor cd_for_create = cd;
2342 
2344  // we need to reset the comp param (as this points to the actual dictionary)
2345  if (cd.columnType.is_array()) {
2346  // for dict encoded arrays, it is always 4 bytes
2347  cd_for_create.columnType.set_comp_param(32);
2348  } else {
2349  cd_for_create.columnType.set_comp_param(cd.columnType.get_size() * 8);
2350  }
2351  }
2352 
2353  column_descriptors_for_create.push_back(cd_for_create);
2354  column_descriptors.push_back(cd);
2355  }
2356 
2357  if (for_create) {
2358  return column_descriptors_for_create;
2359  }
2360 
2361  return column_descriptors;
2362 }
2363 
2364 void InsertIntoTableAsSelectStmt::populateData(
2366  bool is_temporary,
2367  bool validate_table) {
2368  LocalConnector local_connector;
2369 
2370  bool populate_table = false;
2371 
2372  if (leafs_connector_) {
2373  populate_table = true;
2374  } else {
2375  leafs_connector_ = &local_connector;
2376  if (!g_cluster) {
2377  populate_table = true;
2378  }
2379  }
2380 
2381  auto& catalog = session.getCatalog();
2382  auto get_target_column_descriptors = [this, &catalog](const TableDescriptor* td) {
2383  std::vector<const ColumnDescriptor*> target_column_descriptors;
2384  if (column_list_.empty()) {
2385  auto list = catalog.getAllColumnMetadataForTable(td->tableId, false, false, false);
2386  target_column_descriptors = {std::begin(list), std::end(list)};
2387 
2388  } else {
2389  for (auto& c : column_list_) {
2390  const ColumnDescriptor* cd = catalog.getMetadataForColumn(td->tableId, *c);
2391  if (cd == nullptr) {
2392  throw std::runtime_error("Column " + *c + " does not exist.");
2393  }
2394  target_column_descriptors.push_back(cd);
2395  }
2396  }
2397 
2398  return target_column_descriptors;
2399  };
2400 
2401  if (validate_table) {
2402  // check access privileges
2403  const TableDescriptor* td = catalog.getMetadataForTable(table_name_);
2404 
2405  if (td == nullptr) {
2406  throw std::runtime_error("Table " + table_name_ + " does not exist.");
2407  }
2408  if (td->isView) {
2409  throw std::runtime_error("Insert to views is not supported yet.");
2410  }
2411 
2414  table_name_)) {
2415  throw std::runtime_error("User has no insert privileges on " + table_name_ + ".");
2416  }
2417 
2418  // only validate the select query so we get the target types
2419  // correctly, but do not populate the result set
2420  auto result = local_connector.query(session, select_query_, true);
2421  auto source_column_descriptors = local_connector.getColumnDescriptors(result, false);
2422 
2423  std::vector<const ColumnDescriptor*> target_column_descriptors =
2424  get_target_column_descriptors(td);
2425  if (catalog.getAllColumnMetadataForTable(td->tableId, false, false, false).size() !=
2426  target_column_descriptors.size()) {
2427  throw std::runtime_error("Insert into a subset of columns is not supported yet.");
2428  }
2429 
2430  if (source_column_descriptors.size() != target_column_descriptors.size()) {
2431  throw std::runtime_error("The number of source and target columns does not match.");
2432  }
2433 
2434  for (int i = 0; i < source_column_descriptors.size(); i++) {
2435  const ColumnDescriptor* source_cd =
2436  &(*std::next(source_column_descriptors.begin(), i));
2437  const ColumnDescriptor* target_cd = target_column_descriptors.at(i);
2438 
2439  if ((!source_cd->columnType.is_string() && !target_cd->columnType.is_string()) &&
2440  (source_cd->columnType.get_type() != target_cd->columnType.get_type())) {
2441  throw std::runtime_error("Source '" + source_cd->columnName + " " +
2442  source_cd->columnType.get_type_name() +
2443  "' and target '" + target_cd->columnName + " " +
2444  target_cd->columnType.get_type_name() +
2445  "' column types do not match.");
2446  }
2447  if (source_cd->columnType.is_array()) {
2448  if (source_cd->columnType.get_subtype() != target_cd->columnType.get_subtype()) {
2449  throw std::runtime_error("Source '" + source_cd->columnName + " " +
2450  source_cd->columnType.get_type_name() +
2451  "' and target '" + target_cd->columnName + " " +
2452  target_cd->columnType.get_type_name() +
2453  "' array column element types do not match.");
2454  }
2455  }
2456 
2457  if (source_cd->columnType.is_decimal() ||
2458  source_cd->columnType.get_elem_type().is_decimal()) {
2459  SQLTypeInfo sourceType = source_cd->columnType;
2460  SQLTypeInfo targetType = target_cd->columnType;
2461 
2462  if (source_cd->columnType.is_array()) {
2463  sourceType = source_cd->columnType.get_elem_type();
2464  targetType = target_cd->columnType.get_elem_type();
2465  }
2466 
2467  if ((sourceType.get_dimension() != targetType.get_dimension()) ||
2468  (sourceType.get_scale() != targetType.get_scale())) {
2469  throw std::runtime_error(
2470  "Source '" + source_cd->columnName + " " +
2471  source_cd->columnType.get_type_name() + "' and target '" +
2472  target_cd->columnName + " " + target_cd->columnType.get_type_name() +
2473  "' decimal columns dimension and or scale do not match.");
2474  }
2475  }
2476 
2477  if (source_cd->columnType.is_string()) {
2478  if (source_cd->columnType.get_compression() !=
2479  target_cd->columnType.get_compression()) {
2480  throw std::runtime_error("Source '" + source_cd->columnName + " " +
2481  source_cd->columnType.get_type_name() +
2482  "' and target '" + target_cd->columnName + " " +
2483  target_cd->columnType.get_type_name() +
2484  "' columns string encodings do not match.");
2485  }
2486  }
2487 
2488  if (source_cd->columnType.is_timestamp() && target_cd->columnType.is_timestamp()) {
2489  if (source_cd->columnType.get_dimension() !=
2490  target_cd->columnType.get_dimension()) {
2491  throw std::runtime_error("Source '" + source_cd->columnName + " " +
2492  source_cd->columnType.get_type_name() +
2493  "' and target '" + target_cd->columnName + " " +
2494  target_cd->columnType.get_type_name() +
2495  "' timestamp column precisions do not match.");
2496  }
2497  }
2498 
2499  if (!source_cd->columnType.is_string() &&
2500  source_cd->columnType.get_size() > target_cd->columnType.get_size()) {
2501  throw std::runtime_error("Source '" + source_cd->columnName + " " +
2502  source_cd->columnType.get_type_name() +
2503  "' and target '" + target_cd->columnName + " " +
2504  target_cd->columnType.get_type_name() +
2505  "' column encoding sizes do not match.");
2506  }
2507  }
2508  }
2509 
2510  if (!populate_table) {
2511  return;
2512  }
2513 
2514  const TableDescriptor* created_td = catalog.getMetadataForTable(table_name_);
2515  Fragmenter_Namespace::InsertDataLoader insertDataLoader(*leafs_connector_);
2516  AggregatedResult res = leafs_connector_->query(session, select_query_);
2517  auto target_column_descriptors = get_target_column_descriptors(created_td);
2518  auto result_rows = res.rs;
2519  result_rows->setGeoReturnType(ResultSet::GeoReturnType::GeoTargetValue);
2520  const auto num_rows = result_rows->rowCount();
2521 
2522  if (0 == num_rows) {
2523  return;
2524  }
2525 
2526  size_t leaf_count = leafs_connector_->leafCount();
2527 
2528  size_t max_number_of_rows_per_package = std::min(num_rows / leaf_count, 64UL * 1024UL);
2529 
2530  size_t start_row = 0;
2531  size_t num_rows_to_process = std::min(num_rows, max_number_of_rows_per_package);
2532 
2533  // ensure that at least one row is being processed
2534  num_rows_to_process = std::max(num_rows_to_process, 1UL);
2535 
2536  std::vector<std::unique_ptr<TargetValueConverter>> value_converters;
2537 
2539 
2540  const int num_worker_threads = std::thread::hardware_concurrency();
2541 
2542  std::vector<size_t> thread_start_idx(num_worker_threads),
2543  thread_end_idx(num_worker_threads);
2544  bool can_go_parallel = !result_rows->isTruncated() && num_rows_to_process > 20000;
2545 
2546  std::atomic<size_t> row_idx{0};
2547 
2548  auto convert_function = [&result_rows,
2549  &value_converters,
2550  &row_idx,
2551  &num_rows_to_process,
2552  &thread_start_idx,
2553  &thread_end_idx](const int thread_id) {
2554  const int num_cols = value_converters.size();
2555  const size_t start = thread_start_idx[thread_id];
2556  const size_t end = thread_end_idx[thread_id];
2557  size_t idx = 0;
2558  for (idx = start; idx < end; ++idx) {
2559  const auto result_row = result_rows->getRowAtNoTranslations(idx);
2560  if (!result_row.empty()) {
2561  size_t target_row = row_idx.fetch_add(1);
2562 
2563  if (target_row >= num_rows_to_process) {
2564  break;
2565  }
2566 
2567  for (unsigned int col = 0; col < num_cols; col++) {
2568  const auto& mapd_variant = result_row[col];
2569  value_converters[col]->convertToColumnarFormat(target_row, &mapd_variant);
2570  }
2571  }
2572  }
2573 
2574  thread_start_idx[thread_id] = idx;
2575  };
2576 
2577  if (can_go_parallel) {
2578  const size_t entryCount = result_rows->entryCount();
2579  for (size_t i = 0,
2580  start_entry = 0,
2581  stride = (entryCount + num_worker_threads - 1) / num_worker_threads;
2582  i < num_worker_threads && start_entry < entryCount;
2583  ++i, start_entry += stride) {
2584  const auto end_entry = std::min(start_entry + stride, entryCount);
2585  thread_start_idx[i] = start_entry;
2586  thread_end_idx[i] = end_entry;
2587  }
2588 
2589  } else {
2590  thread_start_idx[0] = 0;
2591  thread_end_idx[0] = result_rows->entryCount();
2592  }
2593 
2594  while (start_row < num_rows) {
2595  try {
2596  value_converters.clear();
2597  row_idx = 0;
2598  int colNum = 0;
2599  for (const auto targetDescriptor : target_column_descriptors) {
2600  auto sourceDataMetaInfo = res.targets_meta[colNum++];
2601 
2603  num_rows_to_process,
2604  catalog,
2605  sourceDataMetaInfo,
2606  targetDescriptor,
2607  targetDescriptor->columnType,
2608  !targetDescriptor->columnType.get_notnull(),
2609  result_rows->getRowSetMemOwner()->getLiteralStringDictProxy()};
2610  auto converter = factory.create(param);
2611  value_converters.push_back(std::move(converter));
2612  }
2613 
2614  if (can_go_parallel) {
2615  std::vector<std::future<void>> worker_threads;
2616  for (int i = 0; i < num_worker_threads; ++i) {
2617  worker_threads.push_back(std::async(std::launch::async, convert_function, i));
2618  }
2619 
2620  for (auto& child : worker_threads) {
2621  child.wait();
2622  }
2623  for (auto& child : worker_threads) {
2624  child.get();
2625  }
2626 
2627  } else {
2628  convert_function(0);
2629  }
2630 
2631  // finalize the insert data
2632  {
2633  auto finalizer_func =
2634  [](std::unique_ptr<TargetValueConverter>::pointer targetValueConverter) {
2635  targetValueConverter->finalizeDataBlocksForInsertData();
2636  };
2637  std::vector<std::future<void>> worker_threads;
2638  for (auto& converterPtr : value_converters) {
2639  worker_threads.push_back(
2640  std::async(std::launch::async, finalizer_func, converterPtr.get()));
2641  }
2642 
2643  for (auto& child : worker_threads) {
2644  child.wait();
2645  }
2646  for (auto& child : worker_threads) {
2647  child.get();
2648  }
2649  }
2650 
2652  insert_data.databaseId = catalog.getCurrentDB().dbId;
2653  CHECK(created_td);
2654  insert_data.tableId = created_td->tableId;
2655  insert_data.numRows = num_rows_to_process;
2656 
2657  for (int col_idx = 0; col_idx < target_column_descriptors.size(); col_idx++) {
2658  value_converters[col_idx]->addDataBlocksToInsertData(insert_data);
2659  }
2660 
2661  insertDataLoader.insertData(session, insert_data);
2662  } catch (...) {
2663  try {
2664  if (created_td->nShards) {
2665  const auto shard_tables = catalog.getPhysicalTablesDescriptors(created_td);
2666  for (const auto ptd : shard_tables) {
2667  leafs_connector_->rollback(session, ptd->tableId);
2668  }
2669  }
2670  leafs_connector_->rollback(session, created_td->tableId);
2671  } catch (...) {
2672  // eat it
2673  }
2674  throw;
2675  }
2676  start_row += num_rows_to_process;
2677  num_rows_to_process = std::min(num_rows - start_row, max_number_of_rows_per_package);
2678  }
2679 
2680  if (!is_temporary) {
2681  if (created_td->nShards) {
2682  const auto shard_tables = catalog.getPhysicalTablesDescriptors(created_td);
2683  for (const auto ptd : shard_tables) {
2684  leafs_connector_->checkpoint(session, ptd->tableId);
2685  }
2686  }
2687  leafs_connector_->checkpoint(session, created_td->tableId);
2688  }
2689 }
2690 
2691 void CreateTableAsSelectStmt::execute(const Catalog_Namespace::SessionInfo& session) {
2692  LocalConnector local_connector;
2693  auto& catalog = session.getCatalog();
2694  bool create_table = nullptr == leafs_connector_;
2695 
2696  if (create_table) {
2697  // check access privileges
2700  throw std::runtime_error("CTAS failed. Table " + table_name_ +
2701  " will not be created. User has no create privileges.");
2702  }
2703 
2704  if (catalog.getMetadataForTable(table_name_) != nullptr) {
2705  throw std::runtime_error("Table " + table_name_ +
2706  " already exists and no data was loaded.");
2707  }
2708 
2709  // only validate the select query so we get the target types
2710  // correctly, but do not populate the result set
2711  auto result = local_connector.query(session, select_query_, true);
2712  const auto column_descriptors_for_create =
2713  local_connector.getColumnDescriptors(result, true);
2714 
2715  TableDescriptor td;
2716  td.tableName = table_name_;
2717  td.userId = session.get_currentUser().userId;
2718  td.nColumns = column_descriptors_for_create.size();
2719  td.isView = false;
2720  td.fragmenter = nullptr;
2726  td.keyMetainfo = "[]";
2727  if (is_temporary_) {
2729  } else {
2731  }
2732 
2733  if (!storage_options_.empty()) {
2734  for (auto& p : storage_options_) {
2735  get_table_definitions(td, p, column_descriptors_for_create);
2736  }
2737  }
2738 
2739  catalog.createTable(td, column_descriptors_for_create, {}, true);
2740  // TODO (max): It's transactionally unsafe, should be fixed: we may create object
2741  // w/o privileges
2742  SysCatalog::instance().createDBObject(
2743  session.get_currentUser(), td.tableName, TableDBObjectType, catalog);
2744  }
2745 
2746  try {
2747  populateData(session, is_temporary_, false);
2748  } catch (...) {
2749  if (!g_cluster) {
2750  const TableDescriptor* created_td = catalog.getMetadataForTable(table_name_);
2751  if (created_td) {
2752  catalog.dropTable(created_td);
2753  }
2754  }
2755  throw;
2756  }
2757 }
2758 
2759 void DropTableStmt::execute(const Catalog_Namespace::SessionInfo& session) {
2760  auto& catalog = session.getCatalog();
2761 
2762  const TableDescriptor* td = catalog.getMetadataForTable(*table, false);
2763  if (td == nullptr) {
2764  if (if_exists) {
2765  return;
2766  }
2767  throw std::runtime_error("Table " + *table + " does not exist.");
2768  }
2769 
2770  // check access privileges
2771  if (!session.checkDBAccessPrivileges(
2773  throw std::runtime_error("Table " + *table +
2774  " will not be dropped. User has no proper privileges.");
2775  }
2776 
2777  if (td->isView) {
2778  throw std::runtime_error(*table + " is a view. Use DROP VIEW.");
2779  }
2780 
2781  auto chkptlLock = getTableLock<mapd_shared_mutex, mapd_unique_lock>(
2782  catalog, *table, LockType::CheckpointLock);
2783  auto table_write_lock = TableLockMgr::getWriteLockForTable(catalog, *table);
2784  catalog.dropTable(td);
2785 }
2786 
2787 void TruncateTableStmt::execute(const Catalog_Namespace::SessionInfo& session) {
2788  auto& catalog = session.getCatalog();
2789  const TableDescriptor* td = catalog.getMetadataForTable(*table);
2790  if (td == nullptr) {
2791  throw std::runtime_error("Table " + *table + " does not exist.");
2792  }
2793 
2794  // check access privileges
2795  std::vector<DBObject> privObjects;
2796  DBObject dbObject(*table, TableDBObjectType);
2797  dbObject.loadKey(catalog);
2799  privObjects.push_back(dbObject);
2800  if (!SysCatalog::instance().checkPrivileges(session.get_currentUser(), privObjects)) {
2801  throw std::runtime_error("Table " + *table + " will not be truncated. User " +
2802  session.get_currentUser().userName +
2803  " has no proper privileges.");
2804  }
2805 
2806  if (td->isView) {
2807  throw std::runtime_error(*table + " is a view. Cannot Truncate.");
2808  }
2809  catalog.truncateTable(td);
2810 }
2811 
2813  const TableDescriptor* td) {
2814  if (session.get_currentUser().isSuper ||
2815  session.get_currentUser().userId == td->userId) {
2816  return;
2817  }
2818  std::vector<DBObject> privObjects;
2819  DBObject dbObject(td->tableName, TableDBObjectType);
2820  dbObject.loadKey(session.getCatalog());
2822  privObjects.push_back(dbObject);
2823  if (!SysCatalog::instance().checkPrivileges(session.get_currentUser(), privObjects)) {
2824  throw std::runtime_error("Current user does not have the privilege to alter table: " +
2825  td->tableName);
2826  }
2827 }
2828 
2829 void RenameUserStmt::execute(const Catalog_Namespace::SessionInfo& session) {
2830  if (!session.get_currentUser().isSuper) {
2831  throw std::runtime_error("Only a super user can rename users.");
2832  }
2833 
2835  if (!SysCatalog::instance().getMetadataForUser(*username_, user)) {
2836  throw std::runtime_error("User " + *username_ + " does not exist.");
2837  }
2838 
2839  SysCatalog::instance().renameUser(*username_, *new_username_);
2840 }
2841 
2842 void RenameDatabaseStmt::execute(const Catalog_Namespace::SessionInfo& session) {
2844  if (!SysCatalog::instance().getMetadataForDB(*database_name_, db)) {
2845  throw std::runtime_error("Database " + *database_name_ + " does not exist.");
2846  }
2847 
2848  if (!session.get_currentUser().isSuper &&
2849  session.get_currentUser().userId != db.dbOwner) {
2850  throw std::runtime_error("Only a super user or the owner can rename the database.");
2851  }
2852 
2853  SysCatalog::instance().renameDatabase(*database_name_, *new_database_name_);
2854 }
2855 
2856 void RenameTableStmt::execute(const Catalog_Namespace::SessionInfo& session) {
2857  auto& catalog = session.getCatalog();
2858  const TableDescriptor* td = catalog.getMetadataForTable(*table);
2859  if (td == nullptr) {
2860  throw std::runtime_error("Table " + *table + " does not exist.");
2861  }
2862  check_alter_table_privilege(session, td);
2863  if (catalog.getMetadataForTable(*new_table_name) != nullptr) {
2864  throw std::runtime_error("Table or View " + *new_table_name + " already exists.");
2865  }
2866  catalog.renameTable(td, *new_table_name);
2867 }
2868 
2869 void DDLStmt::setColumnDescriptor(ColumnDescriptor& cd, const ColumnDef* coldef) {
2870  cd.columnName = *coldef->get_column_name();
2871  SQLType* t = coldef->get_column_type();
2872  t->check_type();
2873  if (t->get_is_array()) {
2875  cd.columnType.set_subtype(t->get_type());
2876  } else {
2877  cd.columnType.set_type(t->get_type());
2878  }
2879  if (IS_GEO(t->get_type())) {
2880  cd.columnType.set_subtype(static_cast<SQLTypes>(t->get_param1()));
2883  } else {
2885  cd.columnType.set_scale(t->get_param2());
2886  }
2888  const ColumnConstraintDef* cc = coldef->get_column_constraint();
2889  if (cc == nullptr) {
2890  cd.columnType.set_notnull(false);
2891  } else {
2893  }
2894  const CompressDef* compression = coldef->get_compression();
2895  if (compression == nullptr) {
2896  // Change default TEXT column behaviour to be DICT encoded
2897  if (cd.columnType.is_string() || cd.columnType.is_string_array()) {
2898  // default to 32-bits
2900  cd.columnType.set_comp_param(32);
2901  } else if (cd.columnType.is_decimal() && cd.columnType.get_precision() <= 4) {
2903  cd.columnType.set_comp_param(16);
2904  } else if (cd.columnType.is_decimal() && cd.columnType.get_precision() <= 9) {
2906  cd.columnType.set_comp_param(32);
2907  } else if (cd.columnType.is_decimal() && cd.columnType.get_precision() > 18) {
2908  throw std::runtime_error(cd.columnName + ": Precision too high, max 18.");
2909  } else if (cd.columnType.is_geometry() && cd.columnType.get_output_srid() == 4326) {
2910  // default to GEOINT 32-bits
2912  cd.columnType.set_comp_param(32);
2913  } else if (type == kDATE && g_use_date_in_days_default_encoding) {
2914  // Days encoding for DATE
2916  cd.columnType.set_comp_param(0);
2917  } else {
2919  cd.columnType.set_comp_param(0);
2920  }
2921  } else {
2922  const std::string& comp = *compression->get_encoding_name();
2923  int comp_param;
2924  if (boost::iequals(comp, "fixed")) {
2925  if (!cd.columnType.is_integer() && !cd.columnType.is_time() &&
2926  !cd.columnType.is_decimal()) {
2927  throw std::runtime_error(
2928  cd.columnName +
2929  ": Fixed encoding is only supported for integer or time columns.");
2930  }
2931  // fixed-bits encoding
2932  if (type == kARRAY) {
2933  type = cd.columnType.get_subtype();
2934  }
2935  switch (type) {
2936  case kSMALLINT:
2937  if (compression->get_encoding_param() != 8) {
2938  throw std::runtime_error(
2939  cd.columnName +
2940  ": Compression parameter for Fixed encoding on SMALLINT must be 8.");
2941  }
2942  break;
2943  case kINT:
2944  if (compression->get_encoding_param() != 8 &&
2945  compression->get_encoding_param() != 16) {
2946  throw std::runtime_error(
2947  cd.columnName +
2948  ": Compression parameter for Fixed encoding on INTEGER must be 8 or 16.");
2949  }
2950  break;
2951  case kBIGINT:
2952  if (compression->get_encoding_param() != 8 &&
2953  compression->get_encoding_param() != 16 &&
2954  compression->get_encoding_param() != 32) {
2955  throw std::runtime_error(cd.columnName +
2956  ": Compression parameter for Fixed encoding on "
2957  "BIGINT must be 8 or 16 or 32.");
2958  }
2959  break;
2960  case kTIMESTAMP:
2961  case kTIME:
2962  if (compression->get_encoding_param() != 32) {
2963  throw std::runtime_error(cd.columnName +
2964  ": Compression parameter for Fixed encoding on "
2965  "TIME or TIMESTAMP must be 32.");
2966  } else if (cd.columnType.is_high_precision_timestamp()) {
2967  throw std::runtime_error(
2968  "Fixed encoding is not supported for TIMESTAMP(3|6|9).");
2969  }
2970  break;
2971  case kDECIMAL:
2972  case kNUMERIC:
2973  if (compression->get_encoding_param() != 32 &&
2974  compression->get_encoding_param() != 16) {
2975  throw std::runtime_error(cd.columnName +
2976  ": Compression parameter for Fixed encoding on "
2977  "DECIMAL must be 16 or 32.");
2978  }
2979 
2980  if (compression->get_encoding_param() == 32 &&
2981  cd.columnType.get_precision() > 9) {
2982  throw std::runtime_error(
2983  cd.columnName + ": Precision too high for Fixed(32) encoding, max 9.");
2984  }
2985 
2986  if (compression->get_encoding_param() == 16 &&
2987  cd.columnType.get_precision() > 4) {
2988  throw std::runtime_error(
2989  cd.columnName + ": Precision too high for Fixed(16) encoding, max 4.");
2990  }
2991  break;
2992  case kDATE:
2993  if (compression->get_encoding_param() != 32 &&
2994  compression->get_encoding_param() != 16) {
2995  throw std::runtime_error(cd.columnName +
2996  ": Compression parameter for Fixed encoding on "
2997  "DATE must be 16 or 32.");
2998  }
2999  break;
3000  default:
3001  throw std::runtime_error(cd.columnName + ": Cannot apply FIXED encoding to " +
3002  t->to_string());
3003  }
3004  if (type == kDATE) {
3006  cd.columnType.set_comp_param(16);
3007  } else {
3009  cd.columnType.set_comp_param(compression->get_encoding_param());
3010  }
3011  } else if (boost::iequals(comp, "rl")) {
3012  // run length encoding
3014  cd.columnType.set_comp_param(0);
3015  // throw std::runtime_error("RL(Run Length) encoding not supported yet.");
3016  } else if (boost::iequals(comp, "diff")) {
3017  // differential encoding
3019  cd.columnType.set_comp_param(0);
3020  // throw std::runtime_error("DIFF(differential) encoding not supported yet.");
3021  } else if (boost::iequals(comp, "dict")) {
3022  if (!cd.columnType.is_string() && !cd.columnType.is_string_array()) {
3023  throw std::runtime_error(
3024  cd.columnName +
3025  ": Dictionary encoding is only supported on string or string array columns.");
3026  }
3027  if (compression->get_encoding_param() == 0) {
3028  comp_param = 32; // default to 32-bits
3029  } else {
3030  comp_param = compression->get_encoding_param();
3031  }
3032  if (cd.columnType.is_string_array() && comp_param != 32) {
3033  throw std::runtime_error(cd.columnName +
3034  ": Compression parameter for string arrays must be 32");
3035  }
3036  if (comp_param != 8 && comp_param != 16 && comp_param != 32) {
3037  throw std::runtime_error(
3038  cd.columnName +
3039  ": Compression parameter for Dictionary encoding must be 8 or 16 or 32.");
3040  }
3041  // diciontary encoding
3043  cd.columnType.set_comp_param(comp_param);
3044  } else if (boost::iequals(comp, "NONE")) {
3045  if (cd.columnType.is_geometry()) {
3047  cd.columnType.set_comp_param(64);
3048  } else {
3049  if (!cd.columnType.is_string() && !cd.columnType.is_string_array()) {
3050  throw std::runtime_error(
3051  cd.columnName +
3052  ": None encoding is only supported on string or string array columns.");
3053  }
3055  cd.columnType.set_comp_param(0);
3056  }
3057  } else if (boost::iequals(comp, "sparse")) {
3058  // sparse column encoding with mostly NULL values
3059  if (cd.columnType.get_notnull()) {
3060  throw std::runtime_error(
3061  cd.columnName + ": Cannot do sparse column encoding on a NOT NULL column.");
3062  }
3063  if (compression->get_encoding_param() == 0 ||
3064  compression->get_encoding_param() % 8 != 0 ||
3065  compression->get_encoding_param() > 48) {
3066  throw std::runtime_error(
3067  cd.columnName +
3068  "Must specify number of bits as 8, 16, 24, 32 or 48 as the parameter to "
3069  "sparse-column encoding.");
3070  }
3072  cd.columnType.set_comp_param(compression->get_encoding_param());
3073  // throw std::runtime_error("SPARSE encoding not supported yet.");
3074  } else if (boost::iequals(comp, "compressed")) {
3075  if (!cd.columnType.is_geometry() || cd.columnType.get_output_srid() != 4326) {
3076  throw std::runtime_error(
3077  cd.columnName +
3078  ": COMPRESSED encoding is only supported on WGS84 geo columns.");
3079  }
3080  if (compression->get_encoding_param() == 0) {
3081  comp_param = 32; // default to 32-bits
3082  } else {
3083  comp_param = compression->get_encoding_param();
3084  }
3085  if (comp_param != 32) {
3086  throw std::runtime_error(cd.columnName +
3087  ": only 32-bit COMPRESSED geo encoding is supported");
3088  }
3089  // encoding longitude/latitude as integers
3091  cd.columnType.set_comp_param(comp_param);
3092  } else if (boost::iequals(comp, "days")) {
3093  // days encoding for dates
3094  if (cd.columnType.get_type() != kDATE) {
3095  throw std::runtime_error(cd.columnName +
3096  ": Days encoding is only supported for DATE columns.");
3097  }
3098  if (compression->get_encoding_param() != 32 &&
3099  compression->get_encoding_param() != 16) {
3100  throw std::runtime_error(cd.columnName +
3101  ": Compression parameter for Days encoding on "
3102  "DATE must be 16 or 32.");
3103  }
3105  cd.columnType.set_comp_param((compression->get_encoding_param() == 16) ? 16 : 0);
3106  } else {
3107  throw std::runtime_error(cd.columnName + ": Invalid column compression scheme " +
3108  comp);
3109  }
3110  }
3111  if (cd.columnType.is_string_array() &&
3113  throw std::runtime_error(
3114  cd.columnName +
3115  ": Array of strings must be dictionary encoded. Specify ENCODING DICT");
3116  }
3117  if (t->get_is_array()) {
3118  int s = -1;
3119  auto array_size = t->get_array_size();
3120  if (array_size > 0) {
3121  auto sti = cd.columnType.get_elem_type();
3122  s = array_size * sti.get_size();
3123  if (s <= 0) {
3124  throw std::runtime_error(cd.columnName + ": Unexpected fixed length array size");
3125  }
3126  }
3127  cd.columnType.set_size(s);
3128 
3129  } else {
3131  }
3132  cd.isSystemCol = false;
3133  cd.isVirtualCol = false;
3134 }
3135 
3136 void AddColumnStmt::check_executable(const Catalog_Namespace::SessionInfo& session) {
3137  auto& catalog = session.getCatalog();
3138  const TableDescriptor* td = catalog.getMetadataForTable(*table);
3139  if (nullptr == td) {
3140  throw std::runtime_error("Table " + *table + " does not exist.");
3141  } else {
3142  if (td->isView) {
3143  throw std::runtime_error("Expecting a table , found view " + *table);
3144  }
3145  };
3146 
3147  check_alter_table_privilege(session, td);
3148 
3149  if (0 == coldefs.size()) {
3150  coldefs.push_back(std::move(coldef));
3151  }
3152 
3153  for (const auto& coldef : coldefs) {
3154  auto& new_column_name = *coldef->get_column_name();
3155  if (catalog.getMetadataForColumn(td->tableId, new_column_name) != nullptr) {
3156  throw std::runtime_error("Column " + new_column_name + " already exists.");
3157  }
3158  if (reserved_keywords.find(boost::to_upper_copy<std::string>(new_column_name)) !=
3159  reserved_keywords.end()) {
3160  throw std::runtime_error("Cannot add column with reserved keyword '" +
3161  new_column_name + "'");
3162  }
3163  }
3164 }
3165 
3166 void AddColumnStmt::execute(const Catalog_Namespace::SessionInfo& session) {
3167  auto& catalog = session.getCatalog();
3168  const TableDescriptor* td = catalog.getMetadataForTable(*table);
3169  check_executable(session);
3170 
3171  catalog.getSqliteConnector().query("BEGIN TRANSACTION");
3172  try {
3173  std::map<const std::string, const ColumnDescriptor> cds;
3174  std::map<const int, const ColumnDef*> cid_coldefs;
3175  for (const auto& coldef : coldefs) {
3176  ColumnDescriptor cd;
3177  setColumnDescriptor(cd, coldef.get());
3178  catalog.addColumn(*td, cd);
3179  cds.emplace(*coldef->get_column_name(), cd);
3180  cid_coldefs.emplace(cd.columnId, coldef.get());
3181 
3182  // expand geo column to phy columns
3183  if (cd.columnType.is_geometry()) {
3184  std::list<ColumnDescriptor> phy_geo_columns;
3185  catalog.expandGeoColumn(cd, phy_geo_columns);
3186  for (auto& cd : phy_geo_columns) {
3187  catalog.addColumn(*td, cd);
3188  cds.emplace(cd.columnName, cd);
3189  cid_coldefs.emplace(cd.columnId, nullptr);
3190  }
3191  }
3192  }
3193 
3194  std::unique_ptr<Importer_NS::Loader> loader(new Importer_NS::Loader(catalog, td));
3195  std::vector<std::unique_ptr<Importer_NS::TypedImportBuffer>> import_buffers;
3196 
3197  // a call of Catalog_Namespace::MapDHandler::prepare_columnar_loader
3199  td->tableName,
3200  td->nColumns - 1,
3201  &loader,
3202  &import_buffers);
3203  loader->setReplicating(true);
3204 
3205  // set_geo_physical_import_buffer below needs a sorted import_buffers
3206  std::sort(import_buffers.begin(),
3207  import_buffers.end(),
3208  [](decltype(import_buffers[0])& a, decltype(import_buffers[0])& b) {
3209  return a->getColumnDesc()->columnId < b->getColumnDesc()->columnId;
3210  });
3211 
3212  size_t nrows = td->fragmenter->getNumRows();
3213  // if sharded, get total nrows from all sharded tables
3214  if (td->nShards > 0) {
3215  const auto physical_tds = catalog.getPhysicalTablesDescriptors(td);
3216  nrows = 0;
3217  std::for_each(physical_tds.begin(), physical_tds.end(), [&nrows](const auto& td) {
3218  nrows += td->fragmenter->getNumRows();
3219  });
3220  }
3221  if (nrows > 0) {
3222  int skip_physical_cols = 0;
3223  for (const auto cit : cid_coldefs) {
3224  const auto cd = catalog.getMetadataForColumn(td->tableId, cit.first);
3225  const auto coldef = cit.second;
3226  const auto column_constraint = coldef ? coldef->get_column_constraint() : nullptr;
3227  std::string defaultval = "";
3228  if (column_constraint) {
3229  auto defaultlp = column_constraint->get_defaultval();
3230  auto defaultsp = dynamic_cast<const StringLiteral*>(defaultlp);
3231  defaultval = defaultsp ? *defaultsp->get_stringval()
3232  : defaultlp ? defaultlp->to_string() : "";
3233  }
3234  bool isnull = column_constraint ? (0 == defaultval.size()) : true;
3235  if (boost::to_upper_copy<std::string>(defaultval) == "NULL") {
3236  isnull = true;
3237  }
3238 
3239  // TODO: remove is_geometry below once if null is allowed for geo
3240  if (isnull) {
3241  if (cd->columnType.is_geometry() ||
3242  (column_constraint && column_constraint->get_notnull())) {
3243  throw std::runtime_error("Default value required for column " +
3244  cd->columnName + " (NULL value not supported)");
3245  }
3246  }
3247 
3248  for (auto it = import_buffers.begin(); it < import_buffers.end(); ++it) {
3249  auto& import_buffer = *it;
3250  if (cd->columnId == import_buffer->getColumnDesc()->columnId) {
3251  if (coldef != nullptr ||
3252  skip_physical_cols-- <= 0) { // skip non-null phy col
3253  import_buffer->add_value(
3254  cd, defaultval, isnull, Importer_NS::CopyParams(), nrows);
3255  // tedious non-null geo default value ...
3256  if (cd->columnType.is_geometry() && !isnull) {
3257  std::vector<double> coords, bounds;
3258  std::vector<int> ring_sizes, poly_rings;
3259  int render_group = 0;
3260  SQLTypeInfo tinfo;
3262  tinfo,
3263  coords,
3264  bounds,
3265  ring_sizes,
3266  poly_rings,
3267  false)) {
3268  throw std::runtime_error("Bad geometry data: '" + defaultval + "'");
3269  }
3270  size_t col_idx = 1 + std::distance(import_buffers.begin(), it);
3272  cd,
3273  import_buffers,
3274  col_idx,
3275  coords,
3276  bounds,
3277  ring_sizes,
3278  poly_rings,
3279  render_group,
3280  nrows);
3281  // skip following phy cols
3282  skip_physical_cols = cd->columnType.get_physical_cols();
3283  }
3284  }
3285  break;
3286  }
3287  }
3288  }
3289  }
3290 
3291  if (!loader->loadNoCheckpoint(import_buffers, nrows)) {
3292  throw std::runtime_error("loadNoCheckpoint failed!");
3293  }
3294  loader->checkpoint();
3295  } catch (...) {
3296  catalog.roll(false);
3297  catalog.getSqliteConnector().query("ROLLBACK TRANSACTION");
3298  throw;
3299  }
3300  catalog.getSqliteConnector().query("END TRANSACTION");
3301  catalog.roll(true);
3302 }
3303 
3304 void RenameColumnStmt::execute(const Catalog_Namespace::SessionInfo& session) {
3305  auto& catalog = session.getCatalog();
3306  const TableDescriptor* td = catalog.getMetadataForTable(*table);
3307  if (td == nullptr) {
3308  throw std::runtime_error("Table " + *table + " does not exist.");
3309  }
3310  check_alter_table_privilege(session, td);
3311  const ColumnDescriptor* cd = catalog.getMetadataForColumn(td->tableId, *column);
3312  if (cd == nullptr) {
3313  throw std::runtime_error("Column " + *column + " does not exist.");
3314  }
3315  if (catalog.getMetadataForColumn(td->tableId, *new_column_name) != nullptr) {
3316  throw std::runtime_error("Column " + *new_column_name + " already exists.");
3317  }
3318  if (reserved_keywords.find(boost::to_upper_copy<std::string>(*new_column_name)) !=
3319  reserved_keywords.end()) {
3320  throw std::runtime_error("Cannot create column with reserved keyword '" +
3321  *new_column_name + "'");
3322  }
3323  catalog.renameColumn(td, cd, *new_column_name);
3324 }
3325 
3326 void CopyTableStmt::execute(const Catalog_Namespace::SessionInfo& session) {
3327  auto importer_factory = [](Catalog_Namespace::Catalog& catalog,
3328  const TableDescriptor* td,
3329  const std::string& file_path,
3330  const Importer_NS::CopyParams& copy_params) {
3331  return boost::make_unique<Importer_NS::Importer>(catalog, td, file_path, copy_params);
3332  };
3333  return execute(session, importer_factory);
3334 }
3335 
3336 void CopyTableStmt::execute(const Catalog_Namespace::SessionInfo& session,
3337  const std::function<std::unique_ptr<Importer_NS::Importer>(
3339  const TableDescriptor*,
3340  const std::string&,
3341  const Importer_NS::CopyParams&)>& importer_factory) {
3342  size_t rows_completed = 0;
3343  size_t rows_rejected = 0;
3344  size_t total_time = 0;
3345  bool load_truncated = false;
3346 
3347  auto& catalog = session.getCatalog();
3348  const TableDescriptor* td = catalog.getMetadataForTable(*table);
3349 
3350  // if the table already exists, it's locked, so check access privileges
3351  if (td) {
3352  std::vector<DBObject> privObjects;
3353  DBObject dbObject(*table, TableDBObjectType);
3354  dbObject.loadKey(catalog);
3356  privObjects.push_back(dbObject);
3357  if (!SysCatalog::instance().checkPrivileges(session.get_currentUser(), privObjects)) {
3358  throw std::runtime_error("Violation of access privileges: user " +
3359  session.get_currentUser().userName +
3360  " has no insert privileges for table " + *table + ".");
3361  }
3362  }
3363 
3364  // since we'll have not only posix file names but also s3/hdfs/... url
3365  // we do not expand wildcard or check file existence here.
3366  // from here on, file_path contains something which may be a url
3367  // or a wildcard of file names;
3368  std::string file_path = *file_pattern;
3369  Importer_NS::CopyParams copy_params;
3370  if (!options.empty()) {
3371  for (auto& p : options) {
3372  if (boost::iequals(*p->get_name(), "max_reject")) {
3373  const IntLiteral* int_literal = dynamic_cast<const IntLiteral*>(p->get_value());
3374  if (int_literal == nullptr) {
3375  throw std::runtime_error("max_reject option must be an integer.");
3376  }
3377  copy_params.max_reject = int_literal->get_intval();
3378  } else if (boost::iequals(*p->get_name(), "buffer_size")) {
3379  const IntLiteral* int_literal = dynamic_cast<const IntLiteral*>(p->get_value());
3380  if (int_literal == nullptr) {
3381  throw std::runtime_error("buffer_size option must be an integer.");
3382  }
3383  copy_params.buffer_size = std::max<size_t>(1 << 20, int_literal->get_intval());
3384  } else if (boost::iequals(*p->get_name(), "threads")) {
3385  const IntLiteral* int_literal = dynamic_cast<const IntLiteral*>(p->get_value());
3386  if (int_literal == nullptr) {
3387  throw std::runtime_error("Threads option must be an integer.");
3388  }
3389  copy_params.threads = int_literal->get_intval();
3390  } else if (boost::iequals(*p->get_name(), "delimiter")) {
3391  const StringLiteral* str_literal =
3392  dynamic_cast<const StringLiteral*>(p->get_value());
3393  if (str_literal == nullptr) {
3394  throw std::runtime_error("Delimiter option must be a string.");
3395  } else if (str_literal->get_stringval()->length() != 1) {
3396  throw std::runtime_error("Delimiter must be a single character string.");
3397  }
3398  copy_params.delimiter = (*str_literal->get_stringval())[0];
3399  } else if (boost::iequals(*p->get_name(), "nulls")) {
3400  const StringLiteral* str_literal =
3401  dynamic_cast<const StringLiteral*>(p->get_value());
3402  if (str_literal == nullptr) {
3403  throw std::runtime_error("Nulls option must be a string.");
3404  }
3405  copy_params.null_str = *str_literal->get_stringval();
3406  } else if (boost::iequals(*p->get_name(), "header")) {
3407  const StringLiteral* str_literal =
3408  dynamic_cast<const StringLiteral*>(p->get_value());
3409  if (str_literal == nullptr) {
3410  throw std::runtime_error("Header option must be a boolean.");
3411  }
3412  const std::string* s = str_literal->get_stringval();
3413  if (*s == "t" || *s == "true" || *s == "T" || *s == "True") {
3415  } else if (*s == "f" || *s == "false" || *s == "F" || *s == "False") {
3417  } else {
3418  throw std::runtime_error("Invalid string for boolean " + *s);
3419  }
3420 #ifdef ENABLE_IMPORT_PARQUET
3421  } else if (boost::iequals(*p->get_name(), "parquet")) {
3422  const StringLiteral* str_literal =
3423  dynamic_cast<const StringLiteral*>(p->get_value());
3424  if (str_literal == nullptr) {
3425  throw std::runtime_error("Parquet option must be a boolean.");
3426  }
3427  const std::string* s = str_literal->get_stringval();
3428  if (*s == "t" || *s == "true" || *s == "T" || *s == "True") {
3429  // not sure a parquet "table" type is proper, but to make code
3430  // look consistent in some places, let's set "table" type too
3431  copy_params.file_type = Importer_NS::FileType::PARQUET;
3432  } else if (*s == "f" || *s == "false" || *s == "F" || *s == "False") {
3433  } else {
3434  throw std::runtime_error("Invalid string for boolean " + *s);
3435  }
3436 #endif // ENABLE_IMPORT_PARQUET
3437  } else if (boost::iequals(*p->get_name(), "s3_access_key")) {
3438  const StringLiteral* str_literal =
3439  dynamic_cast<const StringLiteral*>(p->get_value());
3440  if (str_literal == nullptr) {
3441  throw std::runtime_error("Option s3_access_key must be a string.");
3442  }
3443  copy_params.s3_access_key = *str_literal->get_stringval();
3444  } else if (boost::iequals(*p->get_name(), "s3_secret_key")) {
3445  const StringLiteral* str_literal =
3446  dynamic_cast<const StringLiteral*>(p->get_value());
3447  if (str_literal == nullptr) {
3448  throw std::runtime_error("Option s3_secret_key must be a string.");
3449  }
3450  copy_params.s3_secret_key = *str_literal->get_stringval();
3451  } else if (boost::iequals(*p->get_name(), "s3_region")) {
3452  const StringLiteral* str_literal =
3453  dynamic_cast<const StringLiteral*>(p->get_value());
3454  if (str_literal == nullptr) {
3455  throw std::runtime_error("Option s3_region must be a string.");
3456  }
3457  copy_params.s3_region = *str_literal->get_stringval();
3458  } else if (boost::iequals(*p->get_name(), "s3_endpoint")) {
3459  const StringLiteral* str_literal =
3460  dynamic_cast<const StringLiteral*>(p->get_value());
3461  if (str_literal == nullptr) {
3462  throw std::runtime_error("Option s3_endpoint must be a string.");
3463  }
3464  copy_params.s3_endpoint = *str_literal->get_stringval();
3465  } else if (boost::iequals(*p->get_name(), "quote")) {
3466  const StringLiteral* str_literal =
3467  dynamic_cast<const StringLiteral*>(p->get_value());
3468  if (str_literal == nullptr) {
3469  throw std::runtime_error("Quote option must be a string.");
3470  } else if (str_literal->get_stringval()->length() != 1) {
3471  throw std::runtime_error("Quote must be a single character string.");
3472  }
3473  copy_params.quote = (*str_literal->get_stringval())[0];
3474  } else if (boost::iequals(*p->get_name(), "escape")) {
3475  const StringLiteral* str_literal =
3476  dynamic_cast<const StringLiteral*>(p->get_value());
3477  if (str_literal == nullptr) {
3478  throw std::runtime_error("Escape option must be a string.");
3479  } else if (str_literal->get_stringval()->length() != 1) {
3480  throw std::runtime_error("Escape must be a single character string.");
3481  }
3482  copy_params.escape = (*str_literal->get_stringval())[0];
3483  } else if (boost::iequals(*p->get_name(), "line_delimiter")) {
3484  const StringLiteral* str_literal =
3485  dynamic_cast<const StringLiteral*>(p->get_value());
3486  if (str_literal == nullptr) {
3487  throw std::runtime_error("Line_delimiter option must be a string.");
3488  } else if (str_literal->get_stringval()->length() != 1) {
3489  throw std::runtime_error("Line_delimiter must be a single character string.");
3490  }
3491  copy_params.line_delim = (*str_literal->get_stringval())[0];
3492  } else if (boost::iequals(*p->get_name(), "quoted")) {
3493  const StringLiteral* str_literal =
3494  dynamic_cast<const StringLiteral*>(p->get_value());
3495  if (str_literal == nullptr) {
3496  throw std::runtime_error("Quoted option must be a boolean.");
3497  }
3498  const std::string* s = str_literal->get_stringval();
3499  if (*s == "t" || *s == "true" || *s == "T" || *s == "True") {
3500  copy_params.quoted = true;
3501  } else if (*s == "f" || *s == "false" || *s == "F" || *s == "False") {
3502  copy_params.quoted = false;
3503  } else {
3504  throw std::runtime_error("Invalid string for boolean " + *s);
3505  }
3506  } else if (boost::iequals(*p->get_name(), "plain_text")) {
3507  const StringLiteral* str_literal =
3508  dynamic_cast<const StringLiteral*>(p->get_value());
3509  if (str_literal == nullptr) {
3510  throw std::runtime_error("plain_text option must be a boolean.");
3511  }
3512  const std::string* s = str_literal->get_stringval();
3513  if (*s == "t" || *s == "true" || *s == "T" || *s == "True") {
3514  copy_params.plain_text = true;
3515  } else if (*s == "f" || *s == "false" || *s == "F" || *s == "False") {
3516  copy_params.plain_text = false;
3517  } else {
3518  throw std::runtime_error("Invalid string for boolean " + *s);
3519  }
3520  } else if (boost::iequals(*p->get_name(), "array_marker")) {
3521  const StringLiteral* str_literal =
3522  dynamic_cast<const StringLiteral*>(p->get_value());
3523  if (str_literal == nullptr) {
3524  throw std::runtime_error("Array Marker option must be a string.");
3525  } else if (str_literal->get_stringval()->length() != 2) {
3526  throw std::runtime_error(
3527  "Array Marker option must be exactly two characters. Default is {}.");
3528  }
3529  copy_params.array_begin = (*str_literal->get_stringval())[0];
3530  copy_params.array_end = (*str_literal->get_stringval())[1];
3531  } else if (boost::iequals(*p->get_name(), "array_delimiter")) {
3532  const StringLiteral* str_literal =
3533  dynamic_cast<const StringLiteral*>(p->get_value());
3534  if (str_literal == nullptr) {
3535  throw std::runtime_error("Array Delimiter option must be a string.");
3536  } else if (str_literal->get_stringval()->length() != 1) {
3537  throw std::runtime_error("Array Delimiter must be a single character string.");
3538  }
3539  copy_params.array_delim = (*str_literal->get_stringval())[0];
3540  } else if (boost::iequals(*p->get_name(), "lonlat")) {
3541  const StringLiteral* str_literal =
3542  dynamic_cast<const StringLiteral*>(p->get_value());
3543  if (str_literal == nullptr) {
3544  throw std::runtime_error("Lonlat option must be a boolean.");
3545  }
3546  const std::string* s = str_literal->get_stringval();
3547  if (*s == "t" || *s == "true" || *s == "T" || *s == "True") {
3548  copy_params.lonlat = true;
3549  } else if (*s == "f" || *s == "false" || *s == "F" || *s == "False") {
3550  copy_params.lonlat = false;
3551  } else {
3552  throw std::runtime_error("Invalid string for boolean " + *s);
3553  }
3554  } else if (boost::iequals(*p->get_name(), "geo")) {
3555  const StringLiteral* str_literal =
3556  dynamic_cast<const StringLiteral*>(p->get_value());
3557  if (str_literal == nullptr) {
3558  throw std::runtime_error("Geo option must be a boolean.");
3559  }
3560  const std::string* s = str_literal->get_stringval();
3561  if (*s == "t" || *s == "true" || *s == "T" || *s == "True") {
3563  } else if (*s == "f" || *s == "false" || *s == "F" || *s == "False") {
3565  } else {
3566  throw std::runtime_error("Invalid string for boolean " + *s);
3567  }
3568  } else if (boost::iequals(*p->get_name(), "geo_coords_type")) {
3569  const StringLiteral* str_literal =
3570  dynamic_cast<const StringLiteral*>(p->get_value());
3571  if (str_literal == nullptr) {
3572  throw std::runtime_error("'geo_coords_type' option must be a string");
3573  }
3574  const std::string* s = str_literal->get_stringval();
3575  if (boost::iequals(*s, "geography")) {
3576  throw std::runtime_error(
3577  "GEOGRAPHY coords type not yet supported. Please use GEOMETRY.");
3578  // copy_params.geo_coords_type = kGEOGRAPHY;
3579  } else if (boost::iequals(*s, "geometry")) {
3580  copy_params.geo_coords_type = kGEOMETRY;
3581  } else {
3582  throw std::runtime_error(
3583  "Invalid string for 'geo_coords_type' option (must be 'GEOGRAPHY' or "
3584  "'GEOMETRY'): " +
3585  *s);
3586  }
3587  } else if (boost::iequals(*p->get_name(), "geo_coords_encoding")) {
3588  const StringLiteral* str_literal =
3589  dynamic_cast<const StringLiteral*>(p->get_value());
3590  if (str_literal == nullptr) {
3591  throw std::runtime_error("'geo_coords_encoding' option must be a string");
3592  }
3593  const std::string* s = str_literal->get_stringval();
3594  if (boost::iequals(*s, "none")) {
3595  copy_params.geo_coords_encoding = kENCODING_NONE;
3596  copy_params.geo_coords_comp_param = 0;
3597  } else if (boost::iequals(*s, "compressed(32)")) {
3598  copy_params.geo_coords_encoding = kENCODING_GEOINT;
3599  copy_params.geo_coords_comp_param = 32;
3600  } else {
3601  throw std::runtime_error(
3602  "Invalid string for 'geo_coords_encoding' option (must be 'NONE' or "
3603  "'COMPRESSED(32)'): " +
3604  *s);
3605  }
3606  } else if (boost::iequals(*p->get_name(), "geo_coords_srid")) {
3607  const IntLiteral* int_literal = dynamic_cast<const IntLiteral*>(p->get_value());
3608  if (int_literal == nullptr) {
3609  throw std::runtime_error("'geo_coords_srid' option must be an integer");
3610  }
3611  const int srid = int_literal->get_intval();
3612  if (srid == 4326 || srid == 3857 || srid == 900913) {
3613  copy_params.geo_coords_srid = srid;
3614  } else {
3615  throw std::runtime_error(
3616  "Invalid value for 'geo_coords_srid' option (must be 4326, 3857, or "
3617  "900913): " +
3618  std::to_string(srid));
3619  }
3620  } else if (boost::iequals(*p->get_name(), "geo_layer_name")) {
3621  const StringLiteral* str_literal =
3622  dynamic_cast<const StringLiteral*>(p->get_value());
3623  if (str_literal == nullptr) {
3624  throw std::runtime_error("'geo_layer_name' option must be a string");
3625  }
3626  const std::string* layer_name = str_literal->get_stringval();
3627  if (layer_name) {
3628  copy_params.geo_layer_name = *layer_name;
3629  } else {
3630  throw std::runtime_error("Invalid value for 'geo_layer_name' option");
3631  }
3632  } else if (boost::iequals(*p->get_name(), "partitions")) {
3633  if (copy_params.file_type == Importer_NS::FileType::POLYGON) {
3634  const auto partitions =
3635  static_cast<const StringLiteral*>(p->get_value())->get_stringval();
3636  CHECK(partitions);
3637  const auto partitions_uc = boost::to_upper_copy<std::string>(*partitions);
3638  if (partitions_uc != "REPLICATED") {
3639  throw std::runtime_error("PARTITIONS must be REPLICATED for geo COPY");
3640  }
3641  _geo_copy_from_partitions = partitions_uc;
3642  } else {
3643  throw std::runtime_error("PARTITIONS option not supported for non-geo COPY: " +
3644  *p->get_name());
3645  }
3646  } else {
3647  throw std::runtime_error("Invalid option for COPY: " + *p->get_name());
3648  }
3649  }
3650  }
3651 
3652  std::string tr;
3653  if (copy_params.file_type == Importer_NS::FileType::POLYGON) {
3654  // geo import
3655  // we do nothing here, except stash the parameters so we can
3656  // do the import when we unwind to the top of the handler
3657  _geo_copy_from_file_name = file_path;
3658  _geo_copy_from_copy_params = copy_params;
3659  _was_geo_copy_from = true;
3660 
3661  // the result string
3662  // @TODO simon.eves put something more useful in here
3663  // except we really can't because we haven't done the import yet!
3664  if (td) {
3665  tr = std::string("Appending geo to table '") + *table + std::string("'...");
3666  } else {
3667  tr = std::string("Creating table '") + *table +
3668  std::string("' and importing geo...");
3669  }
3670  } else {
3671  if (td) {
3672  // regular import
3673  auto importer = importer_factory(catalog, td, file_path, copy_params);
3674  auto ms = measure<>::execution([&]() {
3675  auto res = importer->import();
3676  rows_completed += res.rows_completed;
3677  rows_rejected += res.rows_rejected;
3678  load_truncated = res.load_truncated;
3679  });
3680  total_time += ms;
3681 
3682  // results
3683  if (load_truncated || rows_rejected > copy_params.max_reject) {
3684  LOG(ERROR) << "COPY exited early due to reject records count during multi file "
3685  "processing ";
3686  // if we have crossed the truncated load threshold
3687  load_truncated = true;
3688  }
3689  if (!load_truncated) {
3690  tr = std::string("Loaded: " + std::to_string(rows_completed) +
3691  " recs, Rejected: " + std::to_string(rows_rejected) +
3692  " recs in " + std::to_string((double)total_time / 1000.0) +
3693  " secs");
3694  } else {
3695  tr = std::string("Loader truncated due to reject count. Processed : " +
3696  std::to_string(rows_completed) + " recs, Rejected: " +
3697  std::to_string(rows_rejected) + " recs in " +
3698  std::to_string((double)total_time / 1000.0) + " secs");
3699  }
3700  } else {
3701  throw std::runtime_error("Table '" + *table + "' must exist before COPY FROM");
3702  }
3703  }
3704 
3705  return_message.reset(new std::string(tr));
3706  LOG(INFO) << tr;
3707 }
3708 
3709 // CREATE ROLE payroll_dept_role;
3710 void CreateRoleStmt::execute(const Catalog_Namespace::SessionInfo& session) {
3711  const auto& currentUser = session.get_currentUser();
3712  if (!currentUser.isSuper) {
3713  throw std::runtime_error("CREATE ROLE " + get_role() +
3714  " failed. It can only be executed by super user.");
3715  }
3716  SysCatalog::instance().createRole(get_role());
3717 }
3718 
3719 // DROP ROLE payroll_dept_role;
3720 void DropRoleStmt::execute(const Catalog_Namespace::SessionInfo& session) {
3721  const auto& currentUser = session.get_currentUser();
3722  if (!currentUser.isSuper) {
3723  throw std::runtime_error("DROP ROLE " + get_role() +
3724  " failed. It can only be executed by super user.");
3725  }
3726  auto* rl = SysCatalog::instance().getRoleGrantee(get_role());
3727  if (!rl) {
3728  throw std::runtime_error("DROP ROLE " + get_role() +
3729  " failed because role with this name does not exist.");
3730  }
3731  SysCatalog::instance().dropRole(get_role());
3732 }
3733 
3734 std::vector<std::string> splitObjectHierName(const std::string& hierName) {
3735  std::vector<std::string> componentNames;
3736  boost::split(componentNames, hierName, boost::is_any_of("."));
3737  return componentNames;
3738 }
3739 
3740 std::string extractObjectNameFromHierName(const std::string& objectHierName,
3741  const std::string& objectType,
3742  const Catalog_Namespace::Catalog& cat) {
3743  std::string objectName;
3744  std::vector<std::string> componentNames = splitObjectHierName(objectHierName);
3745  if (objectType.compare("DATABASE") == 0) {
3746  if (componentNames.size() == 1) {
3747  objectName = componentNames[0];
3748  } else {
3749  throw std::runtime_error("DB object name is not correct " + objectHierName);
3750  }
3751  } else {
3752  if (objectType.compare("TABLE") == 0 || objectType.compare("DASHBOARD") == 0 ||
3753  objectType.compare("VIEW") == 0) {
3754  switch (componentNames.size()) {
3755  case (1): {
3756  objectName = componentNames[0];
3757  break;
3758  }
3759  case (2): {
3760  objectName = componentNames[1];
3761  break;
3762  }
3763  default: {
3764  throw std::runtime_error("DB object name is not correct " + objectHierName);
3765  }
3766  }
3767  } else {
3768  throw std::runtime_error("DB object type " + objectType + " is not supported.");
3769  }
3770  }
3771  return objectName;
3772 }
3773 
3774 static std::pair<AccessPrivileges, DBObjectType> parseStringPrivs(
3775  const std::string& privs,
3776  const DBObjectType& objectType,
3777  const std::string& object_name) {
3778  static const std::map<std::pair<const std::string, const DBObjectType>,
3779  std::pair<const AccessPrivileges, const DBObjectType>>
3780  privileges_lookup{
3781  {{"ALL"s, DatabaseDBObjectType},
3782  {AccessPrivileges::ALL_DATABASE, DatabaseDBObjectType}},
3783  {{"ALL"s, TableDBObjectType}, {AccessPrivileges::ALL_TABLE, TableDBObjectType}},
3784  {{"ALL"s, DashboardDBObjectType},
3785  {AccessPrivileges::ALL_DASHBOARD, DashboardDBObjectType}},
3786  {{"ALL"s, ViewDBObjectType}, {AccessPrivileges::ALL_VIEW, ViewDBObjectType}},
3787 
3788  {{"CREATE TABLE"s, DatabaseDBObjectType},
3789  {AccessPrivileges::CREATE_TABLE, TableDBObjectType}},
3790  {{"CREATE"s, DatabaseDBObjectType},
3791  {AccessPrivileges::CREATE_TABLE, TableDBObjectType}},
3792  {{"SELECT"s, DatabaseDBObjectType},
3793  {AccessPrivileges::SELECT_FROM_TABLE, TableDBObjectType}},
3794  {{"INSERT"s, DatabaseDBObjectType},
3795  {AccessPrivileges::INSERT_INTO_TABLE, TableDBObjectType}},
3796  {{"TRUNCATE"s, DatabaseDBObjectType},
3797  {AccessPrivileges::TRUNCATE_TABLE, TableDBObjectType}},
3798  {{"UPDATE"s, DatabaseDBObjectType},
3799  {AccessPrivileges::UPDATE_IN_TABLE, TableDBObjectType}},
3800  {{"DELETE"s, DatabaseDBObjectType},
3801  {AccessPrivileges::DELETE_FROM_TABLE, TableDBObjectType}},
3802  {{"DROP"s, DatabaseDBObjectType},
3803  {AccessPrivileges::DROP_TABLE, TableDBObjectType}},
3804  {{"ALTER"s, DatabaseDBObjectType},
3805  {AccessPrivileges::ALTER_TABLE, TableDBObjectType}},
3806 
3807  {{"SELECT"s, TableDBObjectType},
3808  {AccessPrivileges::SELECT_FROM_TABLE, TableDBObjectType}},
3809  {{"INSERT"s, TableDBObjectType},
3810  {AccessPrivileges::INSERT_INTO_TABLE, TableDBObjectType}},
3811  {{"TRUNCATE"s, TableDBObjectType},
3812  {AccessPrivileges::TRUNCATE_TABLE, TableDBObjectType}},
3813  {{"UPDATE"s, TableDBObjectType},
3814  {AccessPrivileges::UPDATE_IN_TABLE, TableDBObjectType}},
3815  {{"DELETE"s, TableDBObjectType},
3816  {AccessPrivileges::DELETE_FROM_TABLE, TableDBObjectType}},
3817  {{"DROP"s, TableDBObjectType},
3818  {AccessPrivileges::DROP_TABLE, TableDBObjectType}},
3819  {{"ALTER"s, TableDBObjectType},
3820  {AccessPrivileges::ALTER_TABLE, TableDBObjectType}},
3821 
3822  {{"CREATE VIEW"s, DatabaseDBObjectType},
3823  {AccessPrivileges::CREATE_VIEW, ViewDBObjectType}},
3824  {{"SELECT VIEW"s, DatabaseDBObjectType},
3825  {AccessPrivileges::SELECT_FROM_VIEW, ViewDBObjectType}},
3826  {{"DROP VIEW"s, DatabaseDBObjectType},
3827  {AccessPrivileges::DROP_VIEW, ViewDBObjectType}},
3828  {{"SELECT"s, ViewDBObjectType},
3829  {AccessPrivileges::SELECT_FROM_VIEW, ViewDBObjectType}},
3830  {{"DROP"s, ViewDBObjectType}, {AccessPrivileges::DROP_VIEW, ViewDBObjectType}},
3831 
3832  {{"CREATE DASHBOARD"s, DatabaseDBObjectType},
3833  {AccessPrivileges::CREATE_DASHBOARD, DashboardDBObjectType}},
3834  {{"EDIT DASHBOARD"s, DatabaseDBObjectType},
3835  {AccessPrivileges::EDIT_DASHBOARD, DashboardDBObjectType}},
3836  {{"VIEW DASHBOARD"s, DatabaseDBObjectType},
3837  {AccessPrivileges::VIEW_DASHBOARD, DashboardDBObjectType}},
3838  {{"DELETE DASHBOARD"s, DatabaseDBObjectType},
3839  {AccessPrivileges::DELETE_DASHBOARD, DashboardDBObjectType}},
3840  {{"VIEW"s, DashboardDBObjectType},
3841  {AccessPrivileges::VIEW_DASHBOARD, DashboardDBObjectType}},
3842  {{"EDIT"s, DashboardDBObjectType},
3843  {AccessPrivileges::EDIT_DASHBOARD, DashboardDBObjectType}},
3844  {{"DELETE"s, DashboardDBObjectType},
3845  {AccessPrivileges::DELETE_DASHBOARD, DashboardDBObjectType}},
3846 
3847  {{"VIEW SQL EDITOR"s, DatabaseDBObjectType},
3848  {AccessPrivileges::VIEW_SQL_EDITOR, DatabaseDBObjectType}},
3849  {{"ACCESS"s, DatabaseDBObjectType},
3850  {AccessPrivileges::ACCESS, DatabaseDBObjectType}}};
3851 
3852  auto result = privileges_lookup.find(std::make_pair(privs, objectType));
3853  if (result == privileges_lookup.end()) {
3854  throw std::runtime_error("Privileges " + privs + " on DB object " + object_name +
3855  " are not correct.");
3856  }
3857  return result->second;
3858 }
3859 
3860 static DBObject createObject(const std::string& objectName, DBObjectType objectType) {
3861  if (objectType == DashboardDBObjectType) {
3862  int32_t dashboard_id = -1;
3863  if (!objectName.empty()) {
3864  try {
3865  dashboard_id = stoi(objectName);
3866  } catch (const std::exception&) {
3867  throw std::runtime_error(
3868  "Privileges on dashboards should be changed via integer dashboard ID");
3869  }
3870  }
3871  return DBObject(dashboard_id, objectType);
3872  } else {
3873  return DBObject(objectName, objectType);
3874  }
3875 }
3876 
3877 // GRANT SELECT/INSERT/CREATE ON TABLE payroll_table TO payroll_dept_role;
3878 void GrantPrivilegesStmt::execute(const Catalog_Namespace::SessionInfo& session) {
3879  auto& catalog = session.getCatalog();
3880  const auto& currentUser = session.get_currentUser();
3881  const auto parserObjectType = boost::to_upper_copy<std::string>(get_object_type());
3882  const auto objectName =
3883  extractObjectNameFromHierName(get_object(), parserObjectType, catalog);
3884  auto objectType = DBObjectTypeFromString(parserObjectType);
3885  DBObject dbObject = createObject(objectName, objectType);
3886  /* verify object ownership if not suser */
3887  if (!currentUser.isSuper) {
3888  if (!SysCatalog::instance().verifyDBObjectOwnership(currentUser, dbObject, catalog)) {
3889  throw std::runtime_error(
3890  "GRANT failed. It can only be executed by super user or owner of the object.");
3891  }
3892  }
3893  /* set proper values of privileges & grant them to the object */
3894  std::vector<DBObject> objects(get_privs().size(), dbObject);
3895  for (size_t i = 0; i < get_privs().size(); ++i) {
3896  std::pair<AccessPrivileges, DBObjectType> priv = parseStringPrivs(
3897  boost::to_upper_copy<std::string>(get_privs()[i]), objectType, get_object());
3898  objects[i].setPrivileges(priv.first);
3899  objects[i].setPermissionType(priv.second);
3900  }
3901  SysCatalog::instance().grantDBObjectPrivilegesBatch(grantees, objects, catalog);
3902 }
3903 
3904 // REVOKE SELECT/INSERT/CREATE ON TABLE payroll_table FROM payroll_dept_role;
3905 void RevokePrivilegesStmt::execute(const Catalog_Namespace::SessionInfo& session) {
3906  auto& catalog = session.getCatalog();
3907  const auto& currentUser = session.get_currentUser();
3908  const auto parserObjectType = boost::to_upper_copy<std::string>(get_object_type());
3909  const auto objectName =
3910  extractObjectNameFromHierName(get_object(), parserObjectType, catalog);
3911  auto objectType = DBObjectTypeFromString(parserObjectType);
3912  DBObject dbObject = createObject(objectName, objectType);
3913  /* verify object ownership if not suser */
3914  if (!currentUser.isSuper) {
3915  if (!SysCatalog::instance().verifyDBObjectOwnership(currentUser, dbObject, catalog)) {
3916  throw std::runtime_error(
3917  "REVOKE failed. It can only be executed by super user or owner of the object.");
3918  }
3919  }
3920  /* set proper values of privileges & grant them to the object */
3921  std::vector<DBObject> objects(get_privs().size(), dbObject);
3922  for (size_t i = 0; i < get_privs().size(); ++i) {
3923  std::pair<AccessPrivileges, DBObjectType> priv = parseStringPrivs(
3924  boost::to_upper_copy<std::string>(get_privs()[i]), objectType, get_object());
3925  objects[i].setPrivileges(priv.first);
3926  objects[i].setPermissionType(priv.second);
3927  }
3928  SysCatalog::instance().revokeDBObjectPrivilegesBatch(grantees, objects, catalog);
3929 }
3930 
3931 // NOTE: not used currently, will we ever use it?
3932 // SHOW ON TABLE payroll_table FOR payroll_dept_role;
3933 void ShowPrivilegesStmt::execute(const Catalog_Namespace::SessionInfo& session) {
3934  auto& catalog = session.getCatalog();
3935  const auto& currentUser = session.get_currentUser();
3936  const auto parserObjectType = boost::to_upper_copy<std::string>(get_object_type());
3937  const auto objectName =
3938  extractObjectNameFromHierName(get_object(), parserObjectType, catalog);
3939  auto objectType = DBObjectTypeFromString(parserObjectType);
3940  DBObject dbObject = createObject(objectName, objectType);
3941  /* verify object ownership if not suser */
3942  if (!currentUser.isSuper) {
3943  if (!SysCatalog::instance().verifyDBObjectOwnership(currentUser, dbObject, catalog)) {
3944  throw std::runtime_error(
3945  "SHOW ON " + get_object() + " FOR " + get_role() +
3946  " failed. It can only be executed by super user or owner of the object.");
3947  }
3948  }
3949  /* get values of privileges for the object and report them */
3950  SysCatalog::instance().getDBObjectPrivileges(get_role(), dbObject, catalog);
3951  AccessPrivileges privs = dbObject.getPrivileges();
3952  printf("\nPRIVILEGES ON %s FOR %s ARE SET AS FOLLOWING: ",
3953  get_object().c_str(),
3954  get_role().c_str());
3955 
3956  if (objectType == DBObjectType::DatabaseDBObjectType) {
3958  printf(" CREATE");
3959  }
3961  printf(" DROP");
3962  }
3963  } else if (objectType == DBObjectType::TableDBObjectType) {
3965  printf(" CREATE");
3966  }
3968  printf(" DROP");
3969  }
3971  printf(" SELECT");
3972  }
3974  printf(" INSERT");
3975  }
3977  printf(" UPDATE");
3978  }
3980  printf(" DELETE");
3981  }
3983  printf(" TRUNCATE");
3984  }
3986  printf(" ALTER");
3987  }
3988  } else if (objectType == DBObjectType::DashboardDBObjectType) {
3990  printf(" CREATE");
3991  }
3993  printf(" DELETE");
3994  }
3996  printf(" VIEW");
3997  }
3999  printf(" EDIT");
4000  }
4001  } else if (objectType == DBObjectType::ViewDBObjectType) {
4003  printf(" CREATE");
4004  }
4006  printf(" DROP");
4007  }
4009  printf(" SELECT");
4010  }
4012  printf(" INSERT");
4013  }
4015  printf(" UPDATE");
4016  }
4018  printf(" DELETE");
4019  }
4020  }
4021  printf(".\n");
4022 }
4023 
4024 // GRANT payroll_dept_role TO joe;
4025 void GrantRoleStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4026  const auto& currentUser = session.get_currentUser();
4027  if (!currentUser.isSuper) {
4028  throw std::runtime_error(
4029  "GRANT failed, because it can only be executed by super user.");
4030  }
4031  if (std::find(get_grantees().begin(), get_grantees().end(), OMNISCI_ROOT_USER) !=
4032  get_grantees().end()) {
4033  throw std::runtime_error(
4034  "Request to grant role failed because mapd root user has all privileges by "
4035  "default.");
4036  }
4037  SysCatalog::instance().grantRoleBatch(get_roles(), get_grantees());
4038 }
4039 
4040 // REVOKE payroll_dept_role FROM joe;get_users
4041 void RevokeRoleStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4042  const auto& currentUser = session.get_currentUser();
4043  if (!currentUser.isSuper) {
4044  throw std::runtime_error(
4045  "REVOKE failed, because it can only be executed by super user.");
4046  }
4047  if (std::find(get_grantees().begin(), get_grantees().end(), OMNISCI_ROOT_USER) !=
4048  get_grantees().end()) {
4049  throw std::runtime_error(
4050  "Request to revoke role failed because privileges can not be revoked from mapd "
4051  "root user.");
4052  }
4053  SysCatalog::instance().revokeRoleBatch(get_roles(), get_grantees());
4054 }
4055 
4056 using dbl = std::numeric_limits<double>;
4057 
4058 void ExportQueryStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4059  if (SysCatalog::instance().isAggregator()) {
4060  // allow copy to statement for stand alone leafs
4061  throw std::runtime_error("Distributed export not supported yet");
4062  }
4063  auto& catalog = session.getCatalog();
4064  Importer_NS::CopyParams copy_params;
4065  if (!options.empty()) {
4066  for (auto& p : options) {
4067  if (boost::iequals(*p->get_name(), "delimiter")) {
4068  const StringLiteral* str_literal =
4069  dynamic_cast<const StringLiteral*>(p->get_value());
4070  if (str_literal == nullptr) {
4071  throw std::runtime_error("Delimiter option must be a string.");
4072  } else if (str_literal->get_stringval()->length() != 1) {
4073  throw std::runtime_error("Delimiter must be a single character string.");
4074  }
4075  copy_params.delimiter = (*str_literal->get_stringval())[0];
4076  } else if (boost::iequals(*p->get_name(), "nulls")) {
4077  const StringLiteral* str_literal =
4078  dynamic_cast<const StringLiteral*>(p->get_value());
4079  if (str_literal == nullptr) {
4080  throw std::runtime_error("Nulls option must be a string.");
4081  }
4082  copy_params.null_str = *str_literal->get_stringval();
4083  } else if (boost::iequals(*p->get_name(), "header")) {
4084  const StringLiteral* str_literal =
4085  dynamic_cast<const StringLiteral*>(p->get_value());
4086  if (str_literal == nullptr) {
4087  throw std::runtime_error("Header option must be a boolean.");
4088  }
4089  const std::string* s = str_literal->get_stringval();
4090  if (*s == "t" || *s == "true" || *s == "T" || *s == "True") {
4092  } else if (*s == "f" || *s == "false" || *s == "F" || *s == "False") {
4094  } else {
4095  throw std::runtime_error("Invalid string for boolean " + *s);
4096  }
4097  } else if (boost::iequals(*p->get_name(), "quote")) {
4098  const StringLiteral* str_literal =
4099  dynamic_cast<const StringLiteral*>(p->get_value());
4100  if (str_literal == nullptr) {
4101  throw std::runtime_error("Quote option must be a string.");
4102  } else if (str_literal->get_stringval()->length() != 1) {
4103  throw std::runtime_error("Quote must be a single character string.");
4104  }
4105  copy_params.quote = (*str_literal->get_stringval())[0];
4106  } else if (boost::iequals(*p->get_name(), "escape")) {
4107  const StringLiteral* str_literal =
4108  dynamic_cast<const StringLiteral*>(p->get_value());
4109  if (str_literal == nullptr) {
4110  throw std::runtime_error("Escape option must be a string.");
4111  } else if (str_literal->get_stringval()->length() != 1) {
4112  throw std::runtime_error("Escape must be a single character string.");
4113  }
4114  copy_params.escape = (*str_literal->get_stringval())[0];
4115  } else if (boost::iequals(*p->get_name(), "line_delimiter")) {
4116  const StringLiteral* str_literal =
4117  dynamic_cast<const StringLiteral*>(p->get_value());
4118  if (str_literal == nullptr) {
4119  throw std::runtime_error("Line_delimiter option must be a string.");
4120  } else if (str_literal->get_stringval()->length() != 1) {
4121  throw std::runtime_error("Line_delimiter must be a single character string.");
4122  }
4123  copy_params.line_delim = (*str_literal->get_stringval())[0];
4124  } else if (boost::iequals(*p->get_name(), "quoted")) {
4125  const StringLiteral* str_literal =
4126  dynamic_cast<const StringLiteral*>(p->get_value());
4127  if (str_literal == nullptr) {
4128  throw std::runtime_error("Quoted option must be a boolean.");
4129  }
4130  const std::string* s = str_literal->get_stringval();
4131  if (*s == "t" || *s == "true" || *s == "T" || *s == "True") {
4132  copy_params.quoted = true;
4133  } else if (*s == "f" || *s == "false" || *s == "F" || *s == "False") {
4134  copy_params.quoted = false;
4135  } else {
4136  throw std::runtime_error("Invalid string for boolean " + *s);
4137  }
4138  } else if (boost::iequals(*p->get_name(), "header")) {
4139  const StringLiteral* str_literal =
4140  dynamic_cast<const StringLiteral*>(p->get_value());
4141  if (str_literal == nullptr) {
4142  throw std::runtime_error("Header option must be a boolean.");
4143  }
4144  const std::string* s = str_literal->get_stringval();
4145  if (*s == "t" || *s == "true" || *s == "T" || *s == "True") {
4147  } else if (*s == "f" || *s == "false" || *s == "F" || *s == "False") {
4149  } else {
4150  throw std::runtime_error("Invalid string for boolean " + *s);
4151  }
4152  } else {
4153  throw std::runtime_error("Invalid option for COPY: " + *p->get_name());
4154  }
4155  }
4156  }
4157  std::vector<TargetMetaInfo> targets;
4158  const auto results = getResultSet(session, *select_stmt, targets);
4159  TargetMetaInfo* td = targets.data();
4160 
4161  std::ofstream outfile;
4162  if (file_path->empty() || !boost::filesystem::path(*file_path).is_absolute()) {
4163  std::string file_name;
4164  if (file_path->empty()) {
4165  file_name = session.get_session_id() + ".txt";
4166  } else {
4167  file_name = boost::filesystem::path(*file_path).filename().string();
4168  }
4169  *file_path = catalog.getBasePath() + "/mapd_export/" + session.get_session_id() + "/";
4170  if (!boost::filesystem::exists(*file_path)) {
4171  if (!boost::filesystem::create_directories(*file_path)) {
4172  throw std::runtime_error("Directory " + *file_path + " cannot be created.");
4173  }
4174  }
4175  *file_path += file_name;
4176  }
4177  outfile.open(*file_path);
4178  if (!outfile) {
4179  throw std::runtime_error("Cannot open file: " + *file_path);
4180  }
4182  bool not_first = false;
4183  size_t i = 0;
4184  for (const auto& target : targets) {
4185  std::string col_name = target.get_resname();
4186  if (col_name.empty()) {
4187  col_name = "result_" + std::to_string(i + 1);
4188  }
4189  if (not_first) {
4190  outfile << copy_params.delimiter;
4191  } else {
4192  not_first = true;
4193  }
4194  outfile << col_name;
4195  ++i;
4196  }
4197  outfile << copy_params.line_delim;
4198  }
4199  while (true) {
4200  const auto crt_row = results->getNextRow(true, true);
4201  if (crt_row.empty()) {
4202  break;
4203  }
4204  bool not_first = false;
4205  for (size_t i = 0; i < results->colCount(); ++i) {
4206  bool is_null;
4207  const auto tv = crt_row[i];
4208  const auto scalar_tv = boost::get<ScalarTargetValue>(&tv);
4209  if (not_first) {
4210  outfile << copy_params.delimiter;
4211  } else {
4212  not_first = true;
4213  }
4214  if (copy_params.quoted) {
4215  outfile << copy_params.quote;
4216  }
4217  const auto& ti = td[i].get_type_info();
4218  if (!scalar_tv) {
4219  outfile << datum_to_string(crt_row[i], ti, " | ");
4220  if (copy_params.quoted) {
4221  outfile << copy_params.quote;
4222  }
4223  continue;
4224  }
4225  if (boost::get<int64_t>(scalar_tv)) {
4226  auto int_val = *(boost::get<int64_t>(scalar_tv));
4227  switch (ti.get_type()) {
4228  case kBOOLEAN:
4229  is_null = (int_val == NULL_BOOLEAN);
4230  break;
4231  case kTINYINT:
4232  is_null = (int_val == NULL_TINYINT);
4233  break;
4234  case kSMALLINT:
4235  is_null = (int_val == NULL_SMALLINT);
4236  break;
4237  case kINT:
4238  is_null = (int_val == NULL_INT);
4239  break;
4240  case kBIGINT:
4241  is_null = (int_val == NULL_BIGINT);
4242  break;
4243  case kTIME:
4244  case kTIMESTAMP:
4245  case kDATE:
4246  is_null = (int_val == NULL_BIGINT);
4247  break;
4248  default:
4249  is_null = false;
4250  }
4251  if (is_null) {
4252  outfile << copy_params.null_str;
4253  } else if (ti.get_type() == kTIME) {
4254  const auto t = static_cast<time_t>(int_val);
4255  std::tm tm_struct;
4256  gmtime_r(&t, &tm_struct);
4257  char buf[9];
4258  strftime(buf, 9, "%T", &tm_struct);
4259  outfile << buf;
4260  } else {
4261  outfile << int_val;
4262  }
4263  } else if (boost::get<double>(scalar_tv)) {
4264  auto real_val = *(boost::get<double>(scalar_tv));
4265  if (ti.get_type() == kFLOAT) {
4266  is_null = (real_val == NULL_FLOAT);
4267  } else {
4268  is_null = (real_val == NULL_DOUBLE);
4269  }
4270  if (is_null) {
4271  outfile << copy_params.null_str;
4272  } else if (ti.get_type() == kNUMERIC) {
4273  outfile << std::setprecision(ti.get_precision()) << real_val;
4274  } else {
4275  outfile << std::setprecision(std::numeric_limits<double>::digits10 + 1)
4276  << real_val;
4277  }
4278  } else if (boost::get<float>(scalar_tv)) {
4279  CHECK_EQ(kFLOAT, ti.get_type());
4280  auto real_val = *(boost::get<float>(scalar_tv));
4281  if (real_val == NULL_FLOAT) {
4282  outfile << copy_params.null_str;
4283  } else {
4284  outfile << std::setprecision(std::numeric_limits<float>::digits10 + 1)
4285  << real_val;
4286  }
4287  } else {
4288  auto s = boost::get<NullableString>(scalar_tv);
4289  is_null = !s || boost::get<void*>(s);
4290  if (is_null) {
4291  outfile << copy_params.null_str;
4292  } else {
4293  auto s_notnull = boost::get<std::string>(s);
4294  CHECK(s_notnull);
4295  if (!copy_params.quoted) {
4296  outfile << *s_notnull;
4297  } else {
4298  size_t q = s_notnull->find(copy_params.quote);
4299  if (q == std::string::npos) {
4300  outfile << *s_notnull;
4301  } else {
4302  std::string str(*s_notnull);
4303  while (q != std::string::npos) {
4304  str.insert(q, 1, copy_params.escape);
4305  q = str.find(copy_params.quote, q + 2);
4306  }
4307  outfile << str;
4308  }
4309  }
4310  }
4311  }
4312  if (copy_params.quoted) {
4313  outfile << copy_params.quote;
4314  }
4315  }
4316  outfile << copy_params.line_delim;
4317  }
4318  outfile.close();
4319 }
4320 
4321 void CreateViewStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4322  auto& catalog = session.getCatalog();
4323 
4324  if (catalog.getMetadataForTable(view_name_) != nullptr) {
4325  if (if_not_exists_) {
4326  return;
4327  }
4328  throw std::runtime_error("Table or View " + view_name_ + " already exists.");
4329  }
4330 
4333  throw std::runtime_error("View " + view_name_ +
4334  " will not be created. User has no create view privileges.");
4335  }
4336 
4337  const auto query_after_shim = pg_shim(select_query_);
4338 
4339  // this now also ensures that access permissions are checked
4340  catalog.getCalciteMgr()->process(session, query_after_shim, {}, true, false, false);
4341  TableDescriptor td;
4342  td.tableName = view_name_;
4343  td.userId = session.get_currentUser().userId;
4344  td.nColumns = 0;
4345  td.isView = true;
4346  td.viewSQL = query_after_shim;
4347  td.fragmenter = nullptr;
4349  td.maxFragRows =
4350  DEFAULT_FRAGMENT_ROWS; // @todo this stuff should not be InsertOrderFragmenter
4351  td.maxChunkSize =
4352  DEFAULT_MAX_CHUNK_SIZE; // @todo this stuff should not be InsertOrderFragmenter
4355  catalog.createTable(td, {}, {}, true);
4356 
4357  // TODO (max): It's transactionally unsafe, should be fixed: we may create object w/o
4358  // privileges
4359  SysCatalog::instance().createDBObject(
4360  session.get_currentUser(), view_name_, ViewDBObjectType, catalog);
4361 }
4362 
4363 void DropViewStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4364  auto& catalog = session.getCatalog();
4365  const TableDescriptor* td = catalog.getMetadataForTable(*view_name);
4366  if (td == nullptr) {
4367  if (if_exists) {
4368  return;
4369  }
4370  throw std::runtime_error("View " + *view_name + " does not exist.");
4371  }
4372 
4373  if (!session.checkDBAccessPrivileges(
4375  throw std::runtime_error("View " + *view_name +
4376  " will not be dropped. User has no drop view privileges.");
4377  }
4378 
4379  if (!td->isView) {
4380  throw std::runtime_error(*view_name + " is a table. Use DROP TABLE.");
4381  }
4382  catalog.dropTable(td);
4383 }
4384 
4385 void CreateDBStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4386  if (!session.get_currentUser().isSuper) {
4387  throw std::runtime_error(
4388  "CREATE DATABASE command can only be executed by super user.");
4389  }
4391  if (SysCatalog::instance().getMetadataForDB(*db_name, db_meta) && if_not_exists_) {
4392  return;
4393  }
4394  int ownerId = session.get_currentUser().userId;
4395  if (!name_value_list.empty()) {
4396  for (auto& p : name_value_list) {
4397  if (boost::iequals(*p->get_name(), "owner")) {
4398  if (!dynamic_cast<const StringLiteral*>(p->get_value())) {
4399  throw std::runtime_error("Owner name must be a string literal.");
4400  }
4401  const std::string* str =
4402  static_cast<const StringLiteral*>(p->get_value())->get_stringval();
4404  if (!SysCatalog::instance().getMetadataForUser(*str, user)) {
4405  throw std::runtime_error("User " + *str + " does not exist.");
4406  }
4407  ownerId = user.userId;
4408  } else {
4409  throw std::runtime_error("Invalid CREATE DATABASE option " + *p->get_name() +
4410  ". Only OWNER supported.");
4411  }
4412  }
4413  }
4414  SysCatalog::instance().createDatabase(*db_name, ownerId);
4415 }
4416 
4417 void DropDBStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4418  if (!session.get_currentUser().isSuper) {
4419  throw std::runtime_error("DROP DATABASE command can only be executed by super user.");
4420  }
4422  if (!SysCatalog::instance().getMetadataForDB(*db_name, db)) {
4423  if (if_exists_) {
4424  return;
4425  }
4426  throw std::runtime_error("Database " + *db_name + " does not exist.");
4427  }
4428 
4429  if (!session.get_currentUser().isSuper &&
4430  session.get_currentUser().userId != db.dbOwner) {
4431  throw std::runtime_error("Only the super user or the owner can drop database.");
4432  }
4433 
4434  SysCatalog::instance().dropDatabase(db);
4435 }
4436 
4437 void CreateUserStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4438  std::string passwd;
4439  bool is_super = false;
4440  std::string default_db;
4441  for (auto& p : name_value_list) {
4442  if (boost::iequals(*p->get_name(), "password")) {
4443  if (!dynamic_cast<const StringLiteral*>(p->get_value())) {
4444  throw std::runtime_error("Password must be a string literal.");
4445  }
4446  passwd = *static_cast<const StringLiteral*>(p->get_value())->get_stringval();
4447  } else if (boost::iequals(*p->get_name(), "is_super")) {
4448  if (!dynamic_cast<const StringLiteral*>(p->get_value())) {
4449  throw std::runtime_error("IS_SUPER option must be a string literal.");
4450  }
4451  const std::string* str =
4452  static_cast<const StringLiteral*>(p->get_value())->get_stringval();
4453  if (boost::iequals(*str, "true")) {
4454  is_super = true;
4455  } else if (boost::iequals(*str, "false")) {
4456  is_super = false;
4457  } else {
4458  throw std::runtime_error("Value to IS_SUPER must be TRUE or FALSE.");
4459  }
4460  } else if (boost::iequals(*p->get_name(), "default_db")) {
4461  if (!dynamic_cast<const StringLiteral*>(p->get_value())) {
4462  throw std::runtime_error("DEFAULT_DB option must be a string literal.");
4463  }
4464  default_db = *static_cast<const StringLiteral*>(p->get_value())->get_stringval();
4465  } else {
4466  throw std::runtime_error("Invalid CREATE USER option " + *p->get_name() +
4467  ". Should be PASSWORD, IS_SUPER, or DEFAULT_DB.");
4468  }
4469  }
4470  if (!session.get_currentUser().isSuper) {
4471  throw std::runtime_error("Only super user can create new users.");
4472  }
4473  SysCatalog::instance().createUser(*user_name, passwd, is_super, default_db);
4474 }
4475 
4476 void AlterUserStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4477  // Parse the statement
4478  const std::string* passwd = nullptr;
4479  bool is_super = false;
4480  bool* is_superp = nullptr;
4481  const std::string* default_db = nullptr;
4482  for (auto& p : name_value_list) {
4483  if (boost::iequals(*p->get_name(), "password")) {
4484  if (!dynamic_cast<const StringLiteral*>(p->get_value())) {
4485  throw std::runtime_error("Password must be a string literal.");
4486  }
4487  passwd = static_cast<const StringLiteral*>(p->get_value())->get_stringval();
4488  } else if (boost::iequals(*p->get_name(), "is_super")) {
4489  if (!dynamic_cast<const StringLiteral*>(p->get_value())) {
4490  throw std::runtime_error("IS_SUPER option must be a string literal.");
4491  }
4492  const std::string* str =
4493  static_cast<const StringLiteral*>(p->get_value())->get_stringval();
4494  if (boost::iequals(*str, "true")) {
4495  is_super = true;
4496  is_superp = &is_super;
4497  } else if (boost::iequals(*str, "false")) {
4498  is_super = false;
4499  is_superp = &is_super;
4500  } else {
4501  throw std::runtime_error("Value to IS_SUPER must be TRUE or FALSE.");
4502  }
4503  } else if (boost::iequals(*p->get_name(), "default_db")) {
4504  if (dynamic_cast<const StringLiteral*>(p->get_value())) {
4505  default_db = static_cast<const StringLiteral*>(p->get_value())->get_stringval();
4506  } else if (dynamic_cast<const NullLiteral*>(p->get_value())) {
4507  static std::string blank;
4508  default_db = &blank;
4509  } else {
4510  throw std::runtime_error(
4511  "DEFAULT_DB option must be either a string literal or a NULL literal.");
4512  }
4513  } else {
4514  throw std::runtime_error("Invalid ALTER USER option " + *p->get_name() +
4515  ". Should be PASSWORD, DEFAULT_DB or IS_SUPER.");
4516  }
4517  }
4518 
4519  // Check if the user is authorized to execute ALTER USER statement
4521  if (!SysCatalog::instance().getMetadataForUser(*user_name, user)) {
4522  throw std::runtime_error("User " + *user_name + " does not exist.");
4523  }
4524  if (!session.get_currentUser().isSuper) {
4525  if (session.get_currentUser().userId != user.userId) {
4526  throw std::runtime_error("Only super user can change another user's attributes.");
4527  } else if (is_superp) {
4528  throw std::runtime_error(
4529  "A user can only update their own password or default database.");
4530  }
4531  }
4532 
4533  if (passwd || is_superp || default_db) {
4534  SysCatalog::instance().alterUser(user.userId, passwd, is_superp, default_db);
4535  }
4536 }
4537 
4538 void DropUserStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4539  if (!session.get_currentUser().isSuper) {
4540  throw std::runtime_error("Only super user can drop users.");
4541  }
4542  SysCatalog::instance().dropUser(*user_name);
4543 }
4544 } // namespace Parser
4545 
4546 // this is a non-clustered version of MapDHandler::prepare_columnar_loader,
4547 // exists for non-thrift builds, specifically for test cases and bin/initdb.
4549  const std::string& session,
4550  const std::string& table_name,
4551  size_t num_cols,
4552  std::unique_ptr<Importer_NS::Loader>* loader,
4553  std::vector<std::unique_ptr<Importer_NS::TypedImportBuffer>>* import_buffers) {
4554  auto col_descs = (*loader)->get_column_descs();
4555  for (auto cd : col_descs) {
4556  import_buffers->push_back(std::unique_ptr<Importer_NS::TypedImportBuffer>(
4557  new Importer_NS::TypedImportBuffer(cd, (*loader)->getStringDict(cd))));
4558  }
4559 }
int8_t tinyintval
Definition: sqltypes.h:123
int32_t get_table_id() const
decltype(auto) get_max_rows_def(TableDescriptor &td, const NameValueAssign *p, const std::list< ColumnDescriptor > &columns)
static std::set< std::string > reserved_keywords
static const AccessPrivileges VIEW_SQL_EDITOR
Definition: DBObject.h:145
decltype(auto) get_max_chunk_size_def(TableDescriptor &td, const NameValueAssign *p, const std::list< ColumnDescriptor > &columns)
static const std::map< const std::string, const TableDefFuncPtr > tableDefFuncMap
SQLAgg
Definition: sqldefs.h:71
#define CHECK_EQ(x, y)
Definition: Logger.h:195
size_t shard_column_index(const std::string &name, const std::list< ColumnDescriptor > &columns)
std::string partitions
static const AccessPrivileges VIEW_DASHBOARD
Definition: DBObject.h:164
#define NULL_DOUBLE
Definition: sqltypes.h:177
static const int32_t DROP_VIEW
Definition: DBObject.h:114
SQLTypes get_type() const
Definition: ParserNode.h:67
int64_t get_intval() const
Definition: ParserNode.h:157
bool is_string_array() const
Definition: sqltypes.h:447
void d(const SQLTypes expected_type, const std::string &str)
Definition: ImportTest.cpp:268
bool is_time() const
Definition: sqltypes.h:452
HOST DEVICE int get_size() const
Definition: sqltypes.h:329
static std::shared_ptr< Executor > getExecutor(const int db_id, const std::string &debug_dir="", const std::string &debug_file="", const MapDParameters mapd_parameters=MapDParameters(), ::QueryRenderer::QueryRenderManager *render_manager=nullptr)
Definition: Execute.cpp:122
int get_precision() const
Definition: sqltypes.h:322
static const int32_t SELECT_FROM_VIEW
Definition: DBObject.h:115
const std::vector< std::shared_ptr< TargetEntry > > & get_targetlist() const
Definition: Analyzer.h:1449
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:321
const int8_t const int64_t * num_rows
Definition: Analyzer.h:1406
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:81
std::string null_str
Definition: Importer.h:100
Definition: sqltypes.h:51
SQLTypes
Definition: sqltypes.h:40
static const int32_t UPDATE_IN_VIEW
Definition: DBObject.h:117
std::string tableName
const std::vector< TargetMetaInfo > targets_meta
void add_rte(RangeTableEntry *rte)
Definition: Analyzer.cpp:1266
static const AccessPrivileges ALL_DATABASE
Definition: DBObject.h:144
SQLType * get_column_type() const
Definition: ParserNode.h:805
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
decltype(auto) get_vacuum_def(TableDescriptor &td, const NameValueAssign *p, const std::list< ColumnDescriptor > &columns)
static const AccessPrivileges ALTER_TABLE
Definition: DBObject.h:158
DBObjectType
Definition: DBObject.h:42
static const int32_t CREATE_VIEW
Definition: DBObject.h:113
const std::list< std::shared_ptr< Analyzer::Expr > > & get_group_by() const
Definition: Analyzer.h:1457
virtual void prepare_columnar_loader(const std::string &session, const std::string &table_name, size_t num_cols, std::unique_ptr< Importer_NS::Loader > *loader, std::vector< std::unique_ptr< Importer_NS::TypedImportBuffer >> *import_buffers)
#define NULL_BIGINT
Definition: sqltypes.h:175
void expand_star_in_targetlist(const Catalog_Namespace::Catalog &catalog, std::vector< std::shared_ptr< TargetEntry >> &tlist, int rte_idx)
static const AccessPrivileges TRUNCATE_TABLE
Definition: DBObject.h:157
SQLQualifier
Definition: sqldefs.h:69
#define LOG(tag)
Definition: Logger.h:182
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:326
bool get_is_array() const
Definition: ParserNode.h:70
std::string datum_to_string(const TargetValue &tv, const SQLTypeInfo &ti, const std::string &delim)
SQLOps
Definition: sqldefs.h:29
Definition: sqldefs.h:35
void set_size(int s)
Definition: sqltypes.h:417
void c(const std::string &query_string, const ExecutorDeviceType device_type)
static const AccessPrivileges INSERT_INTO_TABLE
Definition: DBObject.h:154
HOST DEVICE int get_scale() const
Definition: sqltypes.h:324
std::shared_ptr< Analyzer::Expr > deep_copy() const override
Definition: Analyzer.cpp:80
Definition: sqldefs.h:36
static const AccessPrivileges CREATE_DASHBOARD
Definition: DBObject.h:163
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:319
int get_result_table_id() const
Definition: Analyzer.h:1465
std::vector< uint8_t > compress_coords(std::vector< double > &coords, const SQLTypeInfo &ti)
Definition: Importer.cpp:1546
SQLTypes geo_coords_type
Definition: Importer.h:128
void set_input_srid(int d)
Definition: sqltypes.h:413
Definition: sqldefs.h:49
#define DEFAULT_MAX_ROWS
const UserMetadata & get_currentUser() const
Definition: SessionInfo.h:93
HOST DEVICE void set_type(SQLTypes t)
Definition: sqltypes.h:409
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
void set_order_by(std::list< OrderEntry > *o)
Definition: Analyzer.h:1473
std::string extractObjectNameFromHierName(const std::string &objectHierName, const std::string &objectType, const Catalog_Namespace::Catalog &cat)
void set_fixed_size()
Definition: sqltypes.h:418
static const int32_t ALTER_TABLE
Definition: DBObject.h:94
void set_result_col_list(const std::list< int > &col_list)
Definition: Analyzer.h:1467
void set_dimension(int d)
Definition: sqltypes.h:411
void setPrivileges(const AccessPrivileges &privs)
Definition: DBObject.h:202
void set_offset(int64_t o)
Definition: Analyzer.h:1485
std::vector< std::string > splitObjectHierName(const std::string &hierName)
WhichRow get_which_row() const
Definition: Analyzer.h:272
std::string get_session_id() const
Definition: SessionInfo.h:98
static const AccessPrivileges SELECT_FROM_TABLE
Definition: DBObject.h:153
void set_scale(int s)
Definition: sqltypes.h:414
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:327
void set_compression(EncodingType c)
Definition: sqltypes.h:419
int32_t intval
Definition: sqltypes.h:125
std::string to_string(char const *&&v)
void set_notnull(bool n)
Definition: sqltypes.h:416
void set_output_srid(int s)
Definition: sqltypes.h:415
static const AccessPrivileges ALL_VIEW
Definition: DBObject.h:170
int get_param2() const
Definition: ParserNode.h:69
EncodingType geo_coords_encoding
Definition: Importer.h:126
std::shared_ptr< StringDictionary > stringDict
int tableId
identifies the database into which the data is being inserted
Definition: Fragmenter.h:61
DBObjectType DBObjectTypeFromString(const std::string &type)
Definition: DBObject.cpp:88
Definition: sqldefs.h:71
AggregatedResult query(const Catalog_Namespace::SessionInfo &session, std::string &sql_query_string, bool validate_only)
std::string get_type_name() const
Definition: sqltypes.h:422
const ColumnDescriptor * getShardColumnMetadataForTable(const TableDescriptor *td) const
Definition: Catalog.cpp:2879
size_t numRows
a vector of column ids for the row(s) being inserted
Definition: Fragmenter.h:63
float floatval
Definition: sqltypes.h:127
std::numeric_limits< double > dbl
Datum get_constval() const
Definition: Analyzer.h:328
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:410
bool g_use_date_in_days_default_encoding
Definition: ParserNode.cpp:64
bool is_array() const
Definition: sqltypes.h:454
static SQLTypeInfo common_string_type(const SQLTypeInfo &type1, const SQLTypeInfo &type2)
Definition: Analyzer.cpp:379
void set_precision(int d)
Definition: sqltypes.h:412
RangeTableEntry * get_rte(int rte_idx) const
Definition: Analyzer.h:1479
bool is_integer() const
Definition: sqltypes.h:448
T v(const TargetValue &r)
Classes representing a parse tree.
std::string s3_access_key
Definition: Importer.h:115
const std::string * get_column_name() const
Definition: ParserNode.h:804
static const int32_t DROP_DATABASE
Definition: DBObject.h:80
#define NULL_TINYINT
Definition: sqltypes.h:172
const std::list< int > & get_result_col_list() const
Definition: Analyzer.h:1466
static const AccessPrivileges DROP_TABLE
Definition: DBObject.h:152
bool is_decimal() const
Definition: sqltypes.h:449
decltype(auto) get_shard_count_def(TableDescriptor &td, const NameValueAssign *p, const std::list< ColumnDescriptor > &columns)
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:320
Catalog & getCatalog() const
Definition: SessionInfo.h:90
boost::function< void(TableDescriptor &, const NameValueAssign *, const std::list< ColumnDescriptor > &columns)> TableDefFuncPtr
Definition: ParserNode.cpp:72
int64_t bigintval
Definition: sqltypes.h:126
#define NULL_FLOAT
Definition: sqltypes.h:176
std::list< ColumnDescriptor > getColumnDescriptors(AggregatedResult &result, bool for_create)
static const int32_t DELETE_FROM_TABLE
Definition: DBObject.h:92
int get_array_size() const
Definition: ParserNode.h:72
static SQLTypeInfo common_numeric_type(const SQLTypeInfo &type1, const SQLTypeInfo &type2)
Definition: Analyzer.cpp:419
Definition: sqldefs.h:37
Definition: sqldefs.h:71
static const int32_t TRUNCATE_TABLE
Definition: DBObject.h:93
#define DEFAULT_PAGE_SIZE
int16_t smallintval
Definition: sqltypes.h:124
const ColumnConstraintDef * get_column_constraint() const
Definition: ParserNode.h:807
static const AccessPrivileges DELETE_FROM_TABLE
Definition: DBObject.h:156
ImportHeaderRow has_header
Definition: Importer.h:101
specifies the content in-memory of a row in the column metadata table
SQLTypeInfoCore get_elem_type() const
Definition: sqltypes.h:628
#define NULL_INT
Definition: sqltypes.h:174
bool expr_is_null(const Analyzer::Expr *expr)
Definition: ParserNode.cpp:880
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:176
static const int32_t EDIT_DASHBOARD
Definition: DBObject.h:105
static const int32_t DELETE_DASHBOARD
Definition: DBObject.h:103
static const AccessPrivileges CREATE_TABLE
Definition: DBObject.h:151
static const int32_t INSERT_INTO_TABLE
Definition: DBObject.h:90
SQLTypeInfoCore< ArrayContextTypeSizer, ExecutorTypePackaging, DateTimeFacilities > SQLTypeInfo
Definition: sqltypes.h:819
std::string keyMetainfo
void set_num_aggs(int a)
Definition: Analyzer.h:1477
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1348
void set_group_by(std::list< std::shared_ptr< Analyzer::Expr >> &g)
Definition: Analyzer.h:1471
void set_next_query(Query *q)
Definition: Analyzer.h:1474
std::string * stringval
Definition: sqltypes.h:131
MapDHandler * get_mapdHandler() const
Definition: SessionInfo.h:89
static void set_geo_physical_import_buffer(const Catalog_Namespace::Catalog &catalog, const ColumnDescriptor *cd, std::vector< std::unique_ptr< TypedImportBuffer >> &import_buffers, size_t &col_idx, std::vector< double > &coords, std::vector< double > &bounds, std::vector< int > &ring_sizes, std::vector< int > &poly_rings, int render_group, const int64_t replicate_count=0)
Definition: Importer.cpp:1581
std::string s3_endpoint
Definition: Importer.h:118
void set_having_predicate(std::shared_ptr< Analyzer::Expr > p)
Definition: Analyzer.h:1472
void set_result_table_id(int id)
Definition: Analyzer.h:1468
bool is_boolean() const
Definition: sqltypes.h:453
void loadKey()
Definition: DBObject.cpp:165
bool get_is_null() const
Definition: Analyzer.h:327
void validate_shard_column_type(const ColumnDescriptor &cd)
const AccessPrivileges & getPrivileges() const
Definition: DBObject.h:201
Definition: sqltypes.h:54
Definition: sqltypes.h:55
void check_alter_table_privilege(const Catalog_Namespace::SessionInfo &session, const TableDescriptor *td)
HOST DEVICE int get_output_srid() const
Definition: sqltypes.h:325
Definition: sqldefs.h:69
#define TRANSIENT_DICT_ID
Definition: sqltypes.h:187
const std::string OMNISCI_ROOT_USER
Definition: SysCatalog.h:59
void set_is_distinct(bool d)
Definition: Analyzer.h:1469
static const AccessPrivileges SELECT_FROM_VIEW
Definition: DBObject.h:173
bool is_null(const T &v, const SQLTypeInfo &t)
std::vector< std::shared_ptr< TargetEntry > > & get_targetlist_nonconst()
Definition: Analyzer.h:1452
bool get_is_distinct() const
Definition: Analyzer.h:1447
Datum StringToDatum(const std::string &s, SQLTypeInfo &ti)
Definition: Datum.cpp:96
const std::string * get_encoding_name() const
Definition: ParserNode.h:788
std::string s3_region
Definition: Importer.h:117
SQLTypeInfoCore get_array_type() const
Definition: sqltypes.h:632
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1579
Fragmenter_Namespace::FragmenterType fragType
Definition: sqldefs.h:71
Data_Namespace::MemoryLevel persistenceLevel
int get_rte_idx(const std::string &range_var_name) const
Definition: Analyzer.cpp:1255
void set_is_unionall(bool u)
Definition: Analyzer.h:1475
static const int32_t CREATE_DATABASE
Definition: DBObject.h:79
Definition: Catalog.h:60
std::vector< std::string > split(const std::string &str, const std::string &delim)
void insertData(const Catalog_Namespace::SessionInfo &session_info, InsertData &insert_data)
decltype(auto) get_sort_column_def(TableDescriptor &td, const NameValueAssign *p, const std::list< ColumnDescriptor > &columns)
static const AccessPrivileges ALL_DASHBOARD
Definition: DBObject.h:162
static bool getGeoColumns(const std::string &wkt, SQLTypeInfo &ti, std::vector< double > &coords, std::vector< double > &bounds, std::vector< int > &ring_sizes, std::vector< int > &poly_rings, const bool promote_poly_to_mpoly=false)
Definition: geo_types.cpp:459
static const AccessPrivileges ACCESS
Definition: DBObject.h:146
std::string geo_layer_name
Definition: Importer.h:131
static const int32_t VIEW_DASHBOARD
Definition: DBObject.h:104
static std::pair< AccessPrivileges, DBObjectType > parseStringPrivs(const std::string &privs, const DBObjectType &objectType, const std::string &object_name)
Definition: sqltypes.h:43
bool checkDBAccessPrivileges(const DBObjectType &permissionType, const AccessPrivileges &privs, const std::string &objectName="") const
Definition: SessionInfo.cpp:24
virtual std::shared_ptr< Analyzer::Expr > add_cast(const SQLTypeInfo &new_type_info)
Definition: Analyzer.cpp:663
static const AccessPrivileges ALL_TABLE
Definition: DBObject.h:150
decltype(auto) get_frag_size_def(TableDescriptor &td, const NameValueAssign *p, const std::list< ColumnDescriptor > &columns)
#define DEFAULT_FRAGMENT_ROWS
int get_logical_size() const
Definition: sqltypes.h:330