OmniSciDB  1dac507f6e
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
ParserNode.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
25 #include "ParserNode.h"
26 
27 #include "../Analyzer/RangeTableEntry.h"
28 #include "../Catalog/Catalog.h"
29 #include "../Catalog/SharedDictionaryValidator.h"
30 #include "../Fragmenter/InsertOrderFragmenter.h"
31 #include "../Fragmenter/TargetValueConvertersFactories.h"
32 #include "../Import/Importer.h"
33 #include "../Planner/Planner.h"
34 #include "../QueryEngine/CalciteAdapter.h"
35 #include "../QueryEngine/Execute.h"
36 #include "../QueryEngine/ExtensionFunctionsWhitelist.h"
37 #include "../QueryEngine/RelAlgExecutor.h"
38 #include "../Shared/TimeGM.h"
39 #include "../Shared/geo_types.h"
40 #include "../Shared/mapd_glob.h"
41 #include "../Shared/measure.h"
42 #include "../Shared/shard_key.h"
43 #include "LockMgr/LockMgr.h"
44 #include "LockMgr/TableLockMgr.h"
45 #include "ReservedKeywords.h"
46 #include "gen-cpp/CalciteServer.h"
47 #include "parser.h"
48 
49 #include <boost/algorithm/string.hpp>
50 #include <boost/core/null_deleter.hpp>
51 #include <boost/filesystem.hpp>
52 #include <boost/function.hpp>
53 
54 #include <rapidjson/document.h>
55 #include <rapidjson/stringbuffer.h>
56 #include <rapidjson/writer.h>
57 
58 #include <cassert>
59 #include <cmath>
60 #include <limits>
61 #include <random>
62 #include <stdexcept>
63 #include <type_traits>
64 #include <typeinfo>
65 
66 size_t g_leaf_count{0};
69 
70 using namespace Lock_Namespace;
72 using namespace std::string_literals;
73 
74 using TableDefFuncPtr = boost::function<void(TableDescriptor&,
75  const NameValueAssign*,
76  const std::list<ColumnDescriptor>& columns)>;
77 
78 namespace Importer_NS {
79 std::vector<uint8_t> compress_coords(std::vector<double>& coords, const SQLTypeInfo& ti);
80 } // namespace Importer_NS
81 
82 namespace Parser {
83 std::shared_ptr<Analyzer::Expr> NullLiteral::analyze(
84  const Catalog_Namespace::Catalog& catalog,
85  Analyzer::Query& query,
86  TlistRefType allow_tlist_ref) const {
87  return makeExpr<Analyzer::Constant>(kNULLT, true);
88 }
89 
90 std::shared_ptr<Analyzer::Expr> StringLiteral::analyze(
91  const Catalog_Namespace::Catalog& catalog,
92  Analyzer::Query& query,
93  TlistRefType allow_tlist_ref) const {
94  return analyzeValue(*stringval);
95 }
96 
97 std::shared_ptr<Analyzer::Expr> StringLiteral::analyzeValue(
98  const std::string& stringval) {
99  SQLTypeInfo ti(kVARCHAR, stringval.length(), 0, true);
100  Datum d;
101  d.stringval = new std::string(stringval);
102  return makeExpr<Analyzer::Constant>(ti, false, d);
103 }
104 
105 std::shared_ptr<Analyzer::Expr> IntLiteral::analyze(
106  const Catalog_Namespace::Catalog& catalog,
107  Analyzer::Query& query,
108  TlistRefType allow_tlist_ref) const {
109  return analyzeValue(intval);
110 }
111 
112 std::shared_ptr<Analyzer::Expr> IntLiteral::analyzeValue(const int64_t intval) {
113  SQLTypes t;
114  Datum d;
115  if (intval >= INT16_MIN && intval <= INT16_MAX) {
116  t = kSMALLINT;
117  d.smallintval = (int16_t)intval;
118  } else if (intval >= INT32_MIN && intval <= INT32_MAX) {
119  t = kINT;
120  d.intval = (int32_t)intval;
121  } else {
122  t = kBIGINT;
123  d.bigintval = intval;
124  }
125  return makeExpr<Analyzer::Constant>(t, false, d);
126 }
127 
128 std::shared_ptr<Analyzer::Expr> FixedPtLiteral::analyze(
129  const Catalog_Namespace::Catalog& catalog,
130  Analyzer::Query& query,
131  TlistRefType allow_tlist_ref) const {
132  SQLTypeInfo ti(kNUMERIC, 0, 0, false);
133  Datum d = StringToDatum(*fixedptval, ti);
134  return makeExpr<Analyzer::Constant>(ti, false, d);
135 }
136 
137 std::shared_ptr<Analyzer::Expr> FixedPtLiteral::analyzeValue(const int64_t numericval,
138  const int scale,
139  const int precision) {
140  SQLTypeInfo ti(kNUMERIC, 0, 0, false);
141  ti.set_scale(scale);
142  ti.set_precision(precision);
143  Datum d;
144  d.bigintval = numericval;
145  return makeExpr<Analyzer::Constant>(ti, false, d);
146 }
147 
148 std::shared_ptr<Analyzer::Expr> FloatLiteral::analyze(
149  const Catalog_Namespace::Catalog& catalog,
150  Analyzer::Query& query,
151  TlistRefType allow_tlist_ref) const {
152  Datum d;
153  d.floatval = floatval;
154  return makeExpr<Analyzer::Constant>(kFLOAT, false, d);
155 }
156 
157 std::shared_ptr<Analyzer::Expr> DoubleLiteral::analyze(
158  const Catalog_Namespace::Catalog& catalog,
159  Analyzer::Query& query,
160  TlistRefType allow_tlist_ref) const {
161  Datum d;
162  d.doubleval = doubleval;
163  return makeExpr<Analyzer::Constant>(kDOUBLE, false, d);
164 }
165 
166 std::shared_ptr<Analyzer::Expr> TimestampLiteral::analyze(
167  const Catalog_Namespace::Catalog& catalog,
168  Analyzer::Query& query,
169  TlistRefType allow_tlist_ref) const {
170  return get(timestampval_);
171 }
172 
173 std::shared_ptr<Analyzer::Expr> TimestampLiteral::get(const int64_t timestampval) {
174  Datum d;
175  d.bigintval = timestampval;
176  return makeExpr<Analyzer::Constant>(kTIMESTAMP, false, d);
177 }
178 
179 std::shared_ptr<Analyzer::Expr> UserLiteral::analyze(
180  const Catalog_Namespace::Catalog& catalog,
181  Analyzer::Query& query,
182  TlistRefType allow_tlist_ref) const {
183  throw std::runtime_error("USER literal not supported yet.");
184  return nullptr;
185 }
186 
187 std::shared_ptr<Analyzer::Expr> ArrayLiteral::analyze(
188  const Catalog_Namespace::Catalog& catalog,
189  Analyzer::Query& query,
190  TlistRefType allow_tlist_ref) const {
191  SQLTypeInfo ti = SQLTypeInfo(kARRAY, true);
192  bool set_subtype = true;
193  std::list<std::shared_ptr<Analyzer::Expr>> value_exprs;
194  for (auto& p : value_list) {
195  auto e = p->analyze(catalog, query, allow_tlist_ref);
196  CHECK(e);
197  auto c = std::dynamic_pointer_cast<Analyzer::Constant>(e);
198  if (c != nullptr && c->get_is_null()) {
199  value_exprs.push_back(c);
200  continue;
201  }
202  auto subtype = e->get_type_info().get_type();
203  if (subtype == kNULLT) {
204  // NULL element
205  } else if (set_subtype) {
206  ti.set_subtype(subtype);
207  set_subtype = false;
208  } else {
209  if (ti.get_subtype() != subtype) {
210  throw std::runtime_error("ARRAY element literals should be of the same type.");
211  }
212  }
213  value_exprs.push_back(e);
214  }
215  std::shared_ptr<Analyzer::Expr> result =
216  makeExpr<Analyzer::Constant>(ti, false, value_exprs);
217  return result;
218 }
219 
220 std::string ArrayLiteral::to_string() const {
221  std::string str = "{";
222  bool notfirst = false;
223  for (auto& p : value_list) {
224  if (notfirst) {
225  str += ", ";
226  } else {
227  notfirst = true;
228  }
229  str += p->to_string();
230  }
231  str += "}";
232  return str;
233 }
234 
235 std::shared_ptr<Analyzer::Expr> OperExpr::analyze(
236  const Catalog_Namespace::Catalog& catalog,
237  Analyzer::Query& query,
238  TlistRefType allow_tlist_ref) const {
239  auto left_expr = left->analyze(catalog, query, allow_tlist_ref);
240  const auto& left_type = left_expr->get_type_info();
241  if (right == nullptr) {
242  return makeExpr<Analyzer::UOper>(
243  left_type, left_expr->get_contains_agg(), optype, left_expr->decompress());
244  }
245  if (optype == kARRAY_AT) {
246  if (left_type.get_type() != kARRAY) {
247  throw std::runtime_error(left->to_string() + " is not of array type.");
248  }
249  auto right_expr = right->analyze(catalog, query, allow_tlist_ref);
250  const auto& right_type = right_expr->get_type_info();
251  if (!right_type.is_integer()) {
252  throw std::runtime_error(right->to_string() + " is not of integer type.");
253  }
254  return makeExpr<Analyzer::BinOper>(
255  left_type.get_elem_type(), false, kARRAY_AT, kONE, left_expr, right_expr);
256  }
257  auto right_expr = right->analyze(catalog, query, allow_tlist_ref);
258  return normalize(optype, opqualifier, left_expr, right_expr);
259 }
260 
261 std::shared_ptr<Analyzer::Expr> OperExpr::normalize(
262  const SQLOps optype,
263  const SQLQualifier qual,
264  std::shared_ptr<Analyzer::Expr> left_expr,
265  std::shared_ptr<Analyzer::Expr> right_expr) {
266  if (left_expr->get_type_info().is_date_in_days() ||
267  right_expr->get_type_info().is_date_in_days()) {
268  // Do not propogate encoding
269  left_expr = left_expr->decompress();
270  right_expr = right_expr->decompress();
271  }
272  const auto& left_type = left_expr->get_type_info();
273  auto right_type = right_expr->get_type_info();
274  if (qual != kONE) {
275  // subquery not supported yet.
276  CHECK(!std::dynamic_pointer_cast<Analyzer::Subquery>(right_expr));
277  if (right_type.get_type() != kARRAY) {
278  throw std::runtime_error(
279  "Existential or universal qualifiers can only be used in front of a subquery "
280  "or an "
281  "expression of array type.");
282  }
283  right_type = right_type.get_elem_type();
284  }
285  SQLTypeInfo new_left_type;
286  SQLTypeInfo new_right_type;
287  auto result_type = Analyzer::BinOper::analyze_type_info(
288  optype, left_type, right_type, &new_left_type, &new_right_type);
289  if (result_type.is_timeinterval()) {
290  return makeExpr<Analyzer::BinOper>(
291  result_type, false, optype, qual, left_expr, right_expr);
292  }
293  if (left_type != new_left_type) {
294  left_expr = left_expr->add_cast(new_left_type);
295  }
296  if (right_type != new_right_type) {
297  if (qual == kONE) {
298  right_expr = right_expr->add_cast(new_right_type);
299  } else {
300  right_expr = right_expr->add_cast(new_right_type.get_array_type());
301  }
302  }
303  auto check_compression = IS_COMPARISON(optype);
304  if (check_compression) {
305  if (new_left_type.get_compression() == kENCODING_DICT &&
306  new_right_type.get_compression() == kENCODING_DICT &&
307  new_left_type.get_comp_param() == new_right_type.get_comp_param()) {
308  // do nothing
309  } else if (new_left_type.get_compression() == kENCODING_DICT &&
310  new_right_type.get_compression() == kENCODING_NONE) {
311  SQLTypeInfo ti(new_right_type);
312  ti.set_compression(new_left_type.get_compression());
313  ti.set_comp_param(new_left_type.get_comp_param());
314  ti.set_fixed_size();
315  right_expr = right_expr->add_cast(ti);
316  } else if (new_right_type.get_compression() == kENCODING_DICT &&
317  new_left_type.get_compression() == kENCODING_NONE) {
318  SQLTypeInfo ti(new_left_type);
319  ti.set_compression(new_right_type.get_compression());
320  ti.set_comp_param(new_right_type.get_comp_param());
321  ti.set_fixed_size();
322  left_expr = left_expr->add_cast(ti);
323  } else {
324  left_expr = left_expr->decompress();
325  right_expr = right_expr->decompress();
326  }
327  } else {
328  left_expr = left_expr->decompress();
329  right_expr = right_expr->decompress();
330  }
331  bool has_agg = (left_expr->get_contains_agg() || right_expr->get_contains_agg());
332  return makeExpr<Analyzer::BinOper>(
333  result_type, has_agg, optype, qual, left_expr, right_expr);
334 }
335 
336 std::shared_ptr<Analyzer::Expr> SubqueryExpr::analyze(
337  const Catalog_Namespace::Catalog& catalog,
338  Analyzer::Query& query,
339  TlistRefType allow_tlist_ref) const {
340  throw std::runtime_error("Subqueries are not supported yet.");
341  return nullptr;
342 }
343 
344 std::shared_ptr<Analyzer::Expr> IsNullExpr::analyze(
345  const Catalog_Namespace::Catalog& catalog,
346  Analyzer::Query& query,
347  TlistRefType allow_tlist_ref) const {
348  auto arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
349  auto result = makeExpr<Analyzer::UOper>(kBOOLEAN, kISNULL, arg_expr);
350  if (is_not) {
351  result = makeExpr<Analyzer::UOper>(kBOOLEAN, kNOT, result);
352  }
353  return result;
354 }
355 
356 std::shared_ptr<Analyzer::Expr> InSubquery::analyze(
357  const Catalog_Namespace::Catalog& catalog,
358  Analyzer::Query& query,
359  TlistRefType allow_tlist_ref) const {
360  throw std::runtime_error("Subqueries are not supported yet.");
361  return nullptr;
362 }
363 
364 std::shared_ptr<Analyzer::Expr> InValues::analyze(
365  const Catalog_Namespace::Catalog& catalog,
366  Analyzer::Query& query,
367  TlistRefType allow_tlist_ref) const {
368  auto arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
369  SQLTypeInfo ti = arg_expr->get_type_info();
370  bool dict_comp = ti.get_compression() == kENCODING_DICT;
371  std::list<std::shared_ptr<Analyzer::Expr>> value_exprs;
372  for (auto& p : value_list) {
373  auto e = p->analyze(catalog, query, allow_tlist_ref);
374  if (ti != e->get_type_info()) {
375  if (ti.is_string() && e->get_type_info().is_string()) {
376  ti = Analyzer::BinOper::common_string_type(ti, e->get_type_info());
377  } else if (ti.is_number() && e->get_type_info().is_number()) {
378  ti = Analyzer::BinOper::common_numeric_type(ti, e->get_type_info());
379  } else {
380  throw std::runtime_error("IN expressions must contain compatible types.");
381  }
382  }
383  if (dict_comp) {
384  value_exprs.push_back(e->add_cast(arg_expr->get_type_info()));
385  } else {
386  value_exprs.push_back(e);
387  }
388  }
389  if (!dict_comp) {
390  arg_expr = arg_expr->decompress();
391  arg_expr = arg_expr->add_cast(ti);
392  std::list<std::shared_ptr<Analyzer::Expr>> cast_vals;
393  for (auto p : value_exprs) {
394  cast_vals.push_back(p->add_cast(ti));
395  }
396  value_exprs.swap(cast_vals);
397  }
398  std::shared_ptr<Analyzer::Expr> result =
399  makeExpr<Analyzer::InValues>(arg_expr, value_exprs);
400  if (is_not) {
401  result = makeExpr<Analyzer::UOper>(kBOOLEAN, kNOT, result);
402  }
403  return result;
404 }
405 
406 std::shared_ptr<Analyzer::Expr> BetweenExpr::analyze(
407  const Catalog_Namespace::Catalog& catalog,
408  Analyzer::Query& query,
409  TlistRefType allow_tlist_ref) const {
410  auto arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
411  auto lower_expr = lower->analyze(catalog, query, allow_tlist_ref);
412  auto upper_expr = upper->analyze(catalog, query, allow_tlist_ref);
413  SQLTypeInfo new_left_type, new_right_type;
415  arg_expr->get_type_info(),
416  lower_expr->get_type_info(),
417  &new_left_type,
418  &new_right_type);
419  auto lower_pred =
420  makeExpr<Analyzer::BinOper>(kBOOLEAN,
421  kGE,
422  kONE,
423  arg_expr->add_cast(new_left_type)->decompress(),
424  lower_expr->add_cast(new_right_type)->decompress());
426  arg_expr->get_type_info(),
427  lower_expr->get_type_info(),
428  &new_left_type,
429  &new_right_type);
430  auto upper_pred = makeExpr<Analyzer::BinOper>(
431  kBOOLEAN,
432  kLE,
433  kONE,
434  arg_expr->deep_copy()->add_cast(new_left_type)->decompress(),
435  upper_expr->add_cast(new_right_type)->decompress());
436  std::shared_ptr<Analyzer::Expr> result =
437  makeExpr<Analyzer::BinOper>(kBOOLEAN, kAND, kONE, lower_pred, upper_pred);
438  if (is_not) {
439  result = makeExpr<Analyzer::UOper>(kBOOLEAN, kNOT, result);
440  }
441  return result;
442 }
443 
444 std::shared_ptr<Analyzer::Expr> CharLengthExpr::analyze(
445  const Catalog_Namespace::Catalog& catalog,
446  Analyzer::Query& query,
447  TlistRefType allow_tlist_ref) const {
448  auto arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
449  if (!arg_expr->get_type_info().is_string()) {
450  throw std::runtime_error(
451  "expression in char_length clause must be of a string type.");
452  }
453  std::shared_ptr<Analyzer::Expr> result =
454  makeExpr<Analyzer::CharLengthExpr>(arg_expr->decompress(), calc_encoded_length);
455  return result;
456 }
457 
458 std::shared_ptr<Analyzer::Expr> CardinalityExpr::analyze(
459  const Catalog_Namespace::Catalog& catalog,
460  Analyzer::Query& query,
461  TlistRefType allow_tlist_ref) const {
462  auto arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
463  if (!arg_expr->get_type_info().is_array()) {
464  throw std::runtime_error(
465  "expression in cardinality clause must be of an array type.");
466  }
467  std::shared_ptr<Analyzer::Expr> result =
468  makeExpr<Analyzer::CardinalityExpr>(arg_expr->decompress());
469  return result;
470 }
471 
472 void LikeExpr::check_like_expr(const std::string& like_str, char escape_char) {
473  if (like_str.back() == escape_char) {
474  throw std::runtime_error("LIKE pattern must not end with escape character.");
475  }
476 }
477 
478 bool LikeExpr::test_is_simple_expr(const std::string& like_str, char escape_char) {
479  // if not bounded by '%' then not a simple string
480  if (like_str.size() < 2 || like_str[0] != '%' || like_str[like_str.size() - 1] != '%') {
481  return false;
482  }
483  // if the last '%' is escaped then not a simple string
484  if (like_str[like_str.size() - 2] == escape_char &&
485  like_str[like_str.size() - 3] != escape_char) {
486  return false;
487  }
488  for (size_t i = 1; i < like_str.size() - 1; i++) {
489  if (like_str[i] == '%' || like_str[i] == '_' || like_str[i] == '[' ||
490  like_str[i] == ']') {
491  if (like_str[i - 1] != escape_char) {
492  return false;
493  }
494  }
495  }
496  return true;
497 }
498 
499 void LikeExpr::erase_cntl_chars(std::string& like_str, char escape_char) {
500  char prev_char = '\0';
501  // easier to create new string of allowable chars
502  // rather than erase chars from
503  // existing string
504  std::string new_str;
505  for (char& cur_char : like_str) {
506  if (cur_char == '%' || cur_char == escape_char) {
507  if (prev_char != escape_char) {
508  prev_char = cur_char;
509  continue;
510  }
511  }
512  new_str.push_back(cur_char);
513  prev_char = cur_char;
514  }
515  like_str = new_str;
516 }
517 
518 std::shared_ptr<Analyzer::Expr> LikeExpr::analyze(
519  const Catalog_Namespace::Catalog& catalog,
520  Analyzer::Query& query,
521  TlistRefType allow_tlist_ref) const {
522  auto arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
523  auto like_expr = like_string->analyze(catalog, query, allow_tlist_ref);
524  auto escape_expr = escape_string == nullptr
525  ? nullptr
526  : escape_string->analyze(catalog, query, allow_tlist_ref);
527  return LikeExpr::get(arg_expr, like_expr, escape_expr, is_ilike, is_not);
528 }
529 
530 std::shared_ptr<Analyzer::Expr> LikeExpr::get(std::shared_ptr<Analyzer::Expr> arg_expr,
531  std::shared_ptr<Analyzer::Expr> like_expr,
532  std::shared_ptr<Analyzer::Expr> escape_expr,
533  const bool is_ilike,
534  const bool is_not) {
535  if (!arg_expr->get_type_info().is_string()) {
536  throw std::runtime_error("expression before LIKE must be of a string type.");
537  }
538  if (!like_expr->get_type_info().is_string()) {
539  throw std::runtime_error("expression after LIKE must be of a string type.");
540  }
541  char escape_char = '\\';
542  if (escape_expr != nullptr) {
543  if (!escape_expr->get_type_info().is_string()) {
544  throw std::runtime_error("expression after ESCAPE must be of a string type.");
545  }
546  if (!escape_expr->get_type_info().is_string()) {
547  throw std::runtime_error("expression after ESCAPE must be of a string type.");
548  }
549  auto c = std::dynamic_pointer_cast<Analyzer::Constant>(escape_expr);
550  if (c != nullptr && c->get_constval().stringval->length() > 1) {
551  throw std::runtime_error("String after ESCAPE must have a single character.");
552  }
553  escape_char = (*c->get_constval().stringval)[0];
554  }
555  auto c = std::dynamic_pointer_cast<Analyzer::Constant>(like_expr);
556  bool is_simple = false;
557  if (c != nullptr) {
558  std::string& pattern = *c->get_constval().stringval;
559  if (is_ilike) {
560  std::transform(pattern.begin(), pattern.end(), pattern.begin(), ::tolower);
561  }
562  check_like_expr(pattern, escape_char);
563  is_simple = test_is_simple_expr(pattern, escape_char);
564  if (is_simple) {
565  erase_cntl_chars(pattern, escape_char);
566  }
567  }
568  std::shared_ptr<Analyzer::Expr> result = makeExpr<Analyzer::LikeExpr>(
569  arg_expr->decompress(), like_expr, escape_expr, is_ilike, is_simple);
570  if (is_not) {
571  result = makeExpr<Analyzer::UOper>(kBOOLEAN, kNOT, result);
572  }
573  return result;
574 }
575 
576 void RegexpExpr::check_pattern_expr(const std::string& pattern_str, char escape_char) {
577  if (pattern_str.back() == escape_char) {
578  throw std::runtime_error("REGEXP pattern must not end with escape character.");
579  }
580 }
581 
582 bool RegexpExpr::translate_to_like_pattern(std::string& pattern_str, char escape_char) {
583  char prev_char = '\0';
584  char prev_prev_char = '\0';
585  std::string like_str;
586  for (char& cur_char : pattern_str) {
587  if (prev_char == escape_char || isalnum(cur_char) || cur_char == ' ' ||
588  cur_char == '.') {
589  like_str.push_back((cur_char == '.') ? '_' : cur_char);
590  prev_prev_char = prev_char;
591  prev_char = cur_char;
592  continue;
593  }
594  if (prev_char == '.' && prev_prev_char != escape_char) {
595  if (cur_char == '*' || cur_char == '+') {
596  if (cur_char == '*') {
597  like_str.pop_back();
598  }
599  // .* --> %
600  // .+ --> _%
601  like_str.push_back('%');
602  prev_prev_char = prev_char;
603  prev_char = cur_char;
604  continue;
605  }
606  }
607  return false;
608  }
609  pattern_str = like_str;
610  return true;
611 }
612 
613 std::shared_ptr<Analyzer::Expr> RegexpExpr::analyze(
614  const Catalog_Namespace::Catalog& catalog,
615  Analyzer::Query& query,
616  TlistRefType allow_tlist_ref) const {
617  auto arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
618  auto pattern_expr = pattern_string->analyze(catalog, query, allow_tlist_ref);
619  auto escape_expr = escape_string == nullptr
620  ? nullptr
621  : escape_string->analyze(catalog, query, allow_tlist_ref);
622  return RegexpExpr::get(arg_expr, pattern_expr, escape_expr, is_not);
623 }
624 
625 std::shared_ptr<Analyzer::Expr> RegexpExpr::get(
626  std::shared_ptr<Analyzer::Expr> arg_expr,
627  std::shared_ptr<Analyzer::Expr> pattern_expr,
628  std::shared_ptr<Analyzer::Expr> escape_expr,
629  const bool is_not) {
630  if (!arg_expr->get_type_info().is_string()) {
631  throw std::runtime_error("expression before REGEXP must be of a string type.");
632  }
633  if (!pattern_expr->get_type_info().is_string()) {
634  throw std::runtime_error("expression after REGEXP must be of a string type.");
635  }
636  char escape_char = '\\';
637  if (escape_expr != nullptr) {
638  if (!escape_expr->get_type_info().is_string()) {
639  throw std::runtime_error("expression after ESCAPE must be of a string type.");
640  }
641  if (!escape_expr->get_type_info().is_string()) {
642  throw std::runtime_error("expression after ESCAPE must be of a string type.");
643  }
644  auto c = std::dynamic_pointer_cast<Analyzer::Constant>(escape_expr);
645  if (c != nullptr && c->get_constval().stringval->length() > 1) {
646  throw std::runtime_error("String after ESCAPE must have a single character.");
647  }
648  escape_char = (*c->get_constval().stringval)[0];
649  if (escape_char != '\\') {
650  throw std::runtime_error("Only supporting '\\' escape character.");
651  }
652  }
653  auto c = std::dynamic_pointer_cast<Analyzer::Constant>(pattern_expr);
654  if (c != nullptr) {
655  std::string& pattern = *c->get_constval().stringval;
656  if (translate_to_like_pattern(pattern, escape_char)) {
657  return LikeExpr::get(arg_expr, pattern_expr, escape_expr, false, is_not);
658  }
659  }
660  std::shared_ptr<Analyzer::Expr> result =
661  makeExpr<Analyzer::RegexpExpr>(arg_expr->decompress(), pattern_expr, escape_expr);
662  if (is_not) {
663  result = makeExpr<Analyzer::UOper>(kBOOLEAN, kNOT, result);
664  }
665  return result;
666 }
667 
668 std::shared_ptr<Analyzer::Expr> LikelihoodExpr::analyze(
669  const Catalog_Namespace::Catalog& catalog,
670  Analyzer::Query& query,
671  TlistRefType allow_tlist_ref) const {
672  auto arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
673  return LikelihoodExpr::get(arg_expr, likelihood, is_not);
674 }
675 
676 std::shared_ptr<Analyzer::Expr> LikelihoodExpr::get(
677  std::shared_ptr<Analyzer::Expr> arg_expr,
678  float likelihood,
679  const bool is_not) {
680  if (!arg_expr->get_type_info().is_boolean()) {
681  throw std::runtime_error("likelihood expression expects boolean type.");
682  }
683  std::shared_ptr<Analyzer::Expr> result = makeExpr<Analyzer::LikelihoodExpr>(
684  arg_expr->decompress(), is_not ? 1 - likelihood : likelihood);
685  return result;
686 }
687 
688 std::shared_ptr<Analyzer::Expr> ExistsExpr::analyze(
689  const Catalog_Namespace::Catalog& catalog,
690  Analyzer::Query& query,
691  TlistRefType allow_tlist_ref) const {
692  throw std::runtime_error("Subqueries are not supported yet.");
693  return nullptr;
694 }
695 
696 std::shared_ptr<Analyzer::Expr> ColumnRef::analyze(
697  const Catalog_Namespace::Catalog& catalog,
698  Analyzer::Query& query,
699  TlistRefType allow_tlist_ref) const {
700  int table_id{0};
701  int rte_idx{0};
702  const ColumnDescriptor* cd{nullptr};
703  if (column == nullptr) {
704  throw std::runtime_error("invalid column name *.");
705  }
706  if (table != nullptr) {
707  rte_idx = query.get_rte_idx(*table);
708  if (rte_idx < 0) {
709  throw std::runtime_error("range variable or table name " + *table +
710  " does not exist.");
711  }
712  Analyzer::RangeTableEntry* rte = query.get_rte(rte_idx);
713  cd = rte->get_column_desc(catalog, *column);
714  if (cd == nullptr) {
715  throw std::runtime_error("Column name " + *column + " does not exist.");
716  }
717  table_id = rte->get_table_id();
718  } else {
719  bool found = false;
720  int i = 0;
721  for (auto rte : query.get_rangetable()) {
722  cd = rte->get_column_desc(catalog, *column);
723  if (cd != nullptr && !found) {
724  found = true;
725  rte_idx = i;
726  table_id = rte->get_table_id();
727  } else if (cd != nullptr && found) {
728  throw std::runtime_error("Column name " + *column + " is ambiguous.");
729  }
730  i++;
731  }
732  if (cd == nullptr && allow_tlist_ref != TlistRefType::TLIST_NONE) {
733  // check if this is a reference to a targetlist entry
734  bool found = false;
735  int varno = -1;
736  int i = 1;
737  std::shared_ptr<Analyzer::TargetEntry> tle;
738  for (auto p : query.get_targetlist()) {
739  if (*column == p->get_resname() && !found) {
740  found = true;
741  varno = i;
742  tle = p;
743  } else if (*column == p->get_resname() && found) {
744  throw std::runtime_error("Output alias " + *column + " is ambiguous.");
745  }
746  i++;
747  }
748  if (found) {
749  if (dynamic_cast<Analyzer::Var*>(tle->get_expr())) {
750  Analyzer::Var* v = static_cast<Analyzer::Var*>(tle->get_expr());
752  return v->deep_copy();
753  }
754  }
755  if (allow_tlist_ref == TlistRefType::TLIST_COPY) {
756  return tle->get_expr()->deep_copy();
757  } else {
758  return makeExpr<Analyzer::Var>(
759  tle->get_expr()->get_type_info(), Analyzer::Var::kOUTPUT, varno);
760  }
761  }
762  }
763  if (cd == nullptr) {
764  throw std::runtime_error("Column name " + *column + " does not exist.");
765  }
766  }
767  return makeExpr<Analyzer::ColumnVar>(cd->columnType, table_id, cd->columnId, rte_idx);
768 }
769 
770 std::shared_ptr<Analyzer::Expr> FunctionRef::analyze(
771  const Catalog_Namespace::Catalog& catalog,
772  Analyzer::Query& query,
773  TlistRefType allow_tlist_ref) const {
774  SQLTypeInfo result_type;
775  SQLAgg agg_type;
776  std::shared_ptr<Analyzer::Expr> arg_expr;
777  bool is_distinct = false;
778  if (boost::iequals(*name, "count")) {
779  result_type = SQLTypeInfo(kBIGINT, false);
780  agg_type = kCOUNT;
781  if (arg) {
782  arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
783  const SQLTypeInfo& ti = arg_expr->get_type_info();
784  if (ti.is_string() && (ti.get_compression() != kENCODING_DICT || !distinct)) {
785  throw std::runtime_error(
786  "Strings must be dictionary-encoded in COUNT(DISTINCT).");
787  }
788  if (ti.get_type() == kARRAY && !distinct) {
789  throw std::runtime_error("Only COUNT(DISTINCT) is supported on arrays.");
790  }
791  }
792  is_distinct = distinct;
793  } else {
794  if (!arg) {
795  throw std::runtime_error("Cannot compute " + *name + " with argument '*'.");
796  }
797  if (boost::iequals(*name, "min")) {
798  agg_type = kMIN;
799  arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
800  arg_expr = arg_expr->decompress();
801  result_type = arg_expr->get_type_info();
802  } else if (boost::iequals(*name, "max")) {
803  agg_type = kMAX;
804  arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
805  arg_expr = arg_expr->decompress();
806  result_type = arg_expr->get_type_info();
807  } else if (boost::iequals(*name, "avg")) {
808  agg_type = kAVG;
809  arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
810  if (!arg_expr->get_type_info().is_number()) {
811  throw std::runtime_error("Cannot compute AVG on non-number-type arguments.");
812  }
813  arg_expr = arg_expr->decompress();
814  result_type = SQLTypeInfo(kDOUBLE, false);
815  } else if (boost::iequals(*name, "sum")) {
816  agg_type = kSUM;
817  arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
818  if (!arg_expr->get_type_info().is_number()) {
819  throw std::runtime_error("Cannot compute SUM on non-number-type arguments.");
820  }
821  arg_expr = arg_expr->decompress();
822  result_type = arg_expr->get_type_info().is_integer() ? SQLTypeInfo(kBIGINT, false)
823  : arg_expr->get_type_info();
824  } else if (boost::iequals(*name, "unnest")) {
825  arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
826  const SQLTypeInfo& arg_ti = arg_expr->get_type_info();
827  if (arg_ti.get_type() != kARRAY) {
828  throw std::runtime_error(arg->to_string() + " is not of array type.");
829  }
830  return makeExpr<Analyzer::UOper>(arg_ti.get_elem_type(), false, kUNNEST, arg_expr);
831  } else {
832  throw std::runtime_error("invalid function name: " + *name);
833  }
834  if (arg_expr->get_type_info().is_string() ||
835  arg_expr->get_type_info().get_type() == kARRAY) {
836  throw std::runtime_error(
837  "Only COUNT(DISTINCT ) aggregate is supported on strings and arrays.");
838  }
839  }
840  int naggs = query.get_num_aggs();
841  query.set_num_aggs(naggs + 1);
842  return makeExpr<Analyzer::AggExpr>(
843  result_type, agg_type, arg_expr, is_distinct, nullptr);
844 }
845 
846 std::shared_ptr<Analyzer::Expr> CastExpr::analyze(
847  const Catalog_Namespace::Catalog& catalog,
848  Analyzer::Query& query,
849  TlistRefType allow_tlist_ref) const {
850  target_type->check_type();
851  auto arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
852  SQLTypeInfo ti(target_type->get_type(),
853  target_type->get_param1(),
854  target_type->get_param2(),
855  arg_expr->get_type_info().get_notnull());
856  if (arg_expr->get_type_info().get_type() != target_type->get_type() &&
857  arg_expr->get_type_info().get_compression() != kENCODING_NONE) {
858  arg_expr->decompress();
859  }
860  return arg_expr->add_cast(ti);
861 }
862 
863 std::shared_ptr<Analyzer::Expr> CaseExpr::analyze(
864  const Catalog_Namespace::Catalog& catalog,
865  Analyzer::Query& query,
866  TlistRefType allow_tlist_ref) const {
867  SQLTypeInfo ti;
868  std::list<std::pair<std::shared_ptr<Analyzer::Expr>, std::shared_ptr<Analyzer::Expr>>>
869  expr_pair_list;
870  for (auto& p : when_then_list) {
871  auto e1 = p->get_expr1()->analyze(catalog, query, allow_tlist_ref);
872  if (e1->get_type_info().get_type() != kBOOLEAN) {
873  throw std::runtime_error("Only boolean expressions can be used after WHEN.");
874  }
875  auto e2 = p->get_expr2()->analyze(catalog, query, allow_tlist_ref);
876  expr_pair_list.emplace_back(e1, e2);
877  }
878  auto else_e = else_expr ? else_expr->analyze(catalog, query, allow_tlist_ref) : nullptr;
879  return normalize(expr_pair_list, else_e);
880 }
881 
882 namespace {
883 
884 bool expr_is_null(const Analyzer::Expr* expr) {
885  if (expr->get_type_info().get_type() == kNULLT) {
886  return true;
887  }
888  const auto const_expr = dynamic_cast<const Analyzer::Constant*>(expr);
889  return const_expr && const_expr->get_is_null();
890 }
891 
893  const std::string* s = str_literal->get_stringval();
894  if (*s == "t" || *s == "true" || *s == "T" || *s == "True") {
895  return true;
896  } else if (*s == "f" || *s == "false" || *s == "F" || *s == "False") {
897  return false;
898  } else {
899  throw std::runtime_error("Invalid string for boolean " + *s);
900  }
901 }
902 
903 } // namespace
904 
905 std::shared_ptr<Analyzer::Expr> CaseExpr::normalize(
906  const std::list<std::pair<std::shared_ptr<Analyzer::Expr>,
907  std::shared_ptr<Analyzer::Expr>>>& expr_pair_list,
908  const std::shared_ptr<Analyzer::Expr> else_e_in) {
909  SQLTypeInfo ti;
910  bool has_agg = false;
911  for (auto& p : expr_pair_list) {
912  auto e1 = p.first;
913  CHECK(e1->get_type_info().is_boolean());
914  auto e2 = p.second;
915  if (ti.get_type() == kNULLT) {
916  ti = e2->get_type_info();
917  } else if (e2->get_type_info().get_type() == kNULLT) {
918  ti.set_notnull(false);
919  e2->set_type_info(ti);
920  } else if (ti != e2->get_type_info()) {
921  if (ti.is_string() && e2->get_type_info().is_string()) {
922  ti = Analyzer::BinOper::common_string_type(ti, e2->get_type_info());
923  } else if (ti.is_number() && e2->get_type_info().is_number()) {
924  ti = Analyzer::BinOper::common_numeric_type(ti, e2->get_type_info());
925  } else if (ti.is_boolean() && e2->get_type_info().is_boolean()) {
926  ti = Analyzer::BinOper::common_numeric_type(ti, e2->get_type_info());
927  } else {
928  throw std::runtime_error(
929  "expressions in THEN clause must be of the same or compatible types.");
930  }
931  }
932  if (e2->get_contains_agg()) {
933  has_agg = true;
934  }
935  }
936  auto else_e = else_e_in;
937  if (else_e) {
938  if (else_e->get_contains_agg()) {
939  has_agg = true;
940  }
941  if (expr_is_null(else_e.get())) {
942  ti.set_notnull(false);
943  else_e->set_type_info(ti);
944  } else if (ti != else_e->get_type_info()) {
945  ti.set_notnull(false);
946  if (ti.is_string() && else_e->get_type_info().is_string()) {
947  ti = Analyzer::BinOper::common_string_type(ti, else_e->get_type_info());
948  } else if (ti.is_number() && else_e->get_type_info().is_number()) {
949  ti = Analyzer::BinOper::common_numeric_type(ti, else_e->get_type_info());
950  } else if (ti.is_boolean() && else_e->get_type_info().is_boolean()) {
951  ti = Analyzer::BinOper::common_numeric_type(ti, else_e->get_type_info());
952  } else {
953  throw std::runtime_error(
954  "expressions in ELSE clause must be of the same or compatible types as those "
955  "in the THEN clauses.");
956  }
957  }
958  }
959  std::list<std::pair<std::shared_ptr<Analyzer::Expr>, std::shared_ptr<Analyzer::Expr>>>
960  cast_expr_pair_list;
961  for (auto p : expr_pair_list) {
962  ti.set_notnull(false);
963  cast_expr_pair_list.emplace_back(p.first, p.second->add_cast(ti));
964  }
965  if (else_e != nullptr) {
966  else_e = else_e->add_cast(ti);
967  } else {
968  Datum d;
969  // always create an else expr so that executor doesn't need to worry about it
970  ti.set_notnull(false);
971  else_e = makeExpr<Analyzer::Constant>(ti, true, d);
972  }
973  if (ti.get_type() == kNULLT) {
974  throw std::runtime_error(
975  "Can't deduce the type for case expressions, all branches null");
976  }
977  return makeExpr<Analyzer::CaseExpr>(ti, has_agg, cast_expr_pair_list, else_e);
978 }
979 
980 std::string CaseExpr::to_string() const {
981  std::string str("CASE ");
982  for (auto& p : when_then_list) {
983  str += "WHEN " + p->get_expr1()->to_string() + " THEN " +
984  p->get_expr2()->to_string() + " ";
985  }
986  if (else_expr != nullptr) {
987  str += "ELSE " + else_expr->to_string();
988  }
989  str += " END";
990  return str;
991 }
992 
993 void UnionQuery::analyze(const Catalog_Namespace::Catalog& catalog,
994  Analyzer::Query& query) const {
995  left->analyze(catalog, query);
996  Analyzer::Query* right_query = new Analyzer::Query();
997  right->analyze(catalog, *right_query);
998  query.set_next_query(right_query);
999  query.set_is_unionall(is_unionall);
1000 }
1001 
1002 void QuerySpec::analyze_having_clause(const Catalog_Namespace::Catalog& catalog,
1003  Analyzer::Query& query) const {
1004  std::shared_ptr<Analyzer::Expr> p;
1005  if (having_clause != nullptr) {
1006  p = having_clause->analyze(catalog, query, Expr::TlistRefType::TLIST_COPY);
1007  if (p->get_type_info().get_type() != kBOOLEAN) {
1008  throw std::runtime_error("Only boolean expressions can be in HAVING clause.");
1009  }
1010  p->check_group_by(query.get_group_by());
1011  }
1012  query.set_having_predicate(p);
1013 }
1014 
1015 void QuerySpec::analyze_group_by(const Catalog_Namespace::Catalog& catalog,
1016  Analyzer::Query& query) const {
1017  std::list<std::shared_ptr<Analyzer::Expr>> groupby;
1018  if (!groupby_clause.empty()) {
1019  int gexpr_no = 1;
1020  std::shared_ptr<Analyzer::Expr> gexpr;
1021  const std::vector<std::shared_ptr<Analyzer::TargetEntry>>& tlist =
1022  query.get_targetlist();
1023  for (auto& c : groupby_clause) {
1024  // special-case ordinal numbers in GROUP BY
1025  if (dynamic_cast<Literal*>(c.get())) {
1026  IntLiteral* i = dynamic_cast<IntLiteral*>(c.get());
1027  if (!i) {
1028  throw std::runtime_error("Invalid literal in GROUP BY clause.");
1029  }
1030  int varno = (int)i->get_intval();
1031  if (varno <= 0 || varno > static_cast<int>(tlist.size())) {
1032  throw std::runtime_error("Invalid ordinal number in GROUP BY clause.");
1033  }
1034  if (tlist[varno - 1]->get_expr()->get_contains_agg()) {
1035  throw std::runtime_error(
1036  "Ordinal number in GROUP BY cannot reference an expression containing "
1037  "aggregate "
1038  "functions.");
1039  }
1040  gexpr = makeExpr<Analyzer::Var>(
1041  tlist[varno - 1]->get_expr()->get_type_info(), Analyzer::Var::kOUTPUT, varno);
1042  } else {
1043  gexpr = c->analyze(catalog, query, Expr::TlistRefType::TLIST_REF);
1044  }
1045  const SQLTypeInfo gti = gexpr->get_type_info();
1046  bool set_new_type = false;
1047  SQLTypeInfo ti(gti);
1048  if (gti.is_string() && gti.get_compression() == kENCODING_NONE) {
1049  set_new_type = true;
1052  ti.set_fixed_size();
1053  }
1054  std::shared_ptr<Analyzer::Var> v;
1055  if (std::dynamic_pointer_cast<Analyzer::Var>(gexpr)) {
1056  v = std::static_pointer_cast<Analyzer::Var>(gexpr);
1057  int n = v->get_varno();
1058  gexpr = tlist[n - 1]->get_own_expr();
1059  auto cv = std::dynamic_pointer_cast<Analyzer::ColumnVar>(gexpr);
1060  if (cv != nullptr) {
1061  // inherit all ColumnVar info for lineage.
1062  *std::static_pointer_cast<Analyzer::ColumnVar>(v) = *cv;
1063  }
1064  v->set_which_row(Analyzer::Var::kGROUPBY);
1065  v->set_varno(gexpr_no);
1066  tlist[n - 1]->set_expr(v);
1067  }
1068  if (set_new_type) {
1069  auto new_e = gexpr->add_cast(ti);
1070  groupby.push_back(new_e);
1071  if (v != nullptr) {
1072  v->set_type_info(new_e->get_type_info());
1073  }
1074  } else {
1075  groupby.push_back(gexpr);
1076  }
1077  gexpr_no++;
1078  }
1079  }
1080  if (query.get_num_aggs() > 0 || !groupby.empty()) {
1081  for (auto t : query.get_targetlist()) {
1082  auto e = t->get_expr();
1083  e->check_group_by(groupby);
1084  }
1085  }
1086  query.set_group_by(groupby);
1087 }
1088 
1089 void QuerySpec::analyze_where_clause(const Catalog_Namespace::Catalog& catalog,
1090  Analyzer::Query& query) const {
1091  if (where_clause == nullptr) {
1092  query.set_where_predicate(nullptr);
1093  return;
1094  }
1095  auto p = where_clause->analyze(catalog, query, Expr::TlistRefType::TLIST_COPY);
1096  if (p->get_type_info().get_type() != kBOOLEAN) {
1097  throw std::runtime_error("Only boolean expressions can be in WHERE clause.");
1098  }
1099  query.set_where_predicate(p);
1100 }
1101 
1102 void QuerySpec::analyze_select_clause(const Catalog_Namespace::Catalog& catalog,
1103  Analyzer::Query& query) const {
1104  std::vector<std::shared_ptr<Analyzer::TargetEntry>>& tlist =
1105  query.get_targetlist_nonconst();
1106  if (select_clause.empty()) {
1107  // this means SELECT *
1108  int rte_idx = 0;
1109  for (auto rte : query.get_rangetable()) {
1110  rte->expand_star_in_targetlist(catalog, tlist, rte_idx++);
1111  }
1112  } else {
1113  for (auto& p : select_clause) {
1114  const Parser::Expr* select_expr = p->get_select_expr();
1115  // look for the case of range_var.*
1116  if (typeid(*select_expr) == typeid(ColumnRef) &&
1117  dynamic_cast<const ColumnRef*>(select_expr)->get_column() == nullptr) {
1118  const std::string* range_var_name =
1119  dynamic_cast<const ColumnRef*>(select_expr)->get_table();
1120  int rte_idx = query.get_rte_idx(*range_var_name);
1121  if (rte_idx < 0) {
1122  throw std::runtime_error("invalid range variable name: " + *range_var_name);
1123  }
1124  Analyzer::RangeTableEntry* rte = query.get_rte(rte_idx);
1125  rte->expand_star_in_targetlist(catalog, tlist, rte_idx);
1126  } else {
1127  auto e = select_expr->analyze(catalog, query);
1128  std::string resname;
1129 
1130  if (p->get_alias() != nullptr) {
1131  resname = *p->get_alias();
1132  } else if (std::dynamic_pointer_cast<Analyzer::ColumnVar>(e) &&
1133  !std::dynamic_pointer_cast<Analyzer::Var>(e)) {
1134  auto colvar = std::static_pointer_cast<Analyzer::ColumnVar>(e);
1135  const ColumnDescriptor* col_desc = catalog.getMetadataForColumn(
1136  colvar->get_table_id(), colvar->get_column_id());
1137  resname = col_desc->columnName;
1138  }
1139  if (e->get_type_info().get_type() == kNULLT) {
1140  throw std::runtime_error(
1141  "Untyped NULL in SELECT clause. Use CAST to specify a type.");
1142  }
1143  auto o = std::static_pointer_cast<Analyzer::UOper>(e);
1144  bool unnest = (o != nullptr && o->get_optype() == kUNNEST);
1145  auto tle = std::make_shared<Analyzer::TargetEntry>(resname, e, unnest);
1146  tlist.push_back(tle);
1147  }
1148  }
1149  }
1150 }
1151 
1152 void QuerySpec::analyze_from_clause(const Catalog_Namespace::Catalog& catalog,
1153  Analyzer::Query& query) const {
1155  for (auto& p : from_clause) {
1156  const TableDescriptor* table_desc;
1157  table_desc = catalog.getMetadataForTable(*p->get_table_name());
1158  if (table_desc == nullptr) {
1159  throw std::runtime_error("Table " + *p->get_table_name() + " does not exist.");
1160  }
1161  std::string range_var;
1162  if (p->get_range_var() == nullptr) {
1163  range_var = *p->get_table_name();
1164  } else {
1165  range_var = *p->get_range_var();
1166  }
1167  rte = new Analyzer::RangeTableEntry(range_var, table_desc, nullptr);
1168  query.add_rte(rte);
1169  }
1170 }
1171 
1172 void QuerySpec::analyze(const Catalog_Namespace::Catalog& catalog,
1173  Analyzer::Query& query) const {
1175  analyze_from_clause(catalog, query);
1176  analyze_select_clause(catalog, query);
1177  analyze_where_clause(catalog, query);
1178  analyze_group_by(catalog, query);
1179  analyze_having_clause(catalog, query);
1180 }
1181 
1182 void SelectStmt::analyze(const Catalog_Namespace::Catalog& catalog,
1183  Analyzer::Query& query) const {
1184  query.set_stmt_type(kSELECT);
1185  query.set_limit(limit);
1186  if (offset < 0) {
1187  throw std::runtime_error("OFFSET cannot be negative.");
1188  }
1189  query.set_offset(offset);
1190  query_expr->analyze(catalog, query);
1191  if (orderby_clause.empty() && !query.get_is_distinct()) {
1192  query.set_order_by(nullptr);
1193  return;
1194  }
1195  const std::vector<std::shared_ptr<Analyzer::TargetEntry>>& tlist =
1196  query.get_targetlist();
1197  std::list<Analyzer::OrderEntry>* order_by = new std::list<Analyzer::OrderEntry>();
1198  if (!orderby_clause.empty()) {
1199  for (auto& p : orderby_clause) {
1200  int tle_no = p->get_colno();
1201  if (tle_no == 0) {
1202  // use column name
1203  // search through targetlist for matching name
1204  const std::string* name = p->get_column()->get_column();
1205  tle_no = 1;
1206  bool found = false;
1207  for (auto tle : tlist) {
1208  if (tle->get_resname() == *name) {
1209  found = true;
1210  break;
1211  }
1212  tle_no++;
1213  }
1214  if (!found) {
1215  throw std::runtime_error("invalid name in order by: " + *name);
1216  }
1217  }
1218  order_by->push_back(
1219  Analyzer::OrderEntry(tle_no, p->get_is_desc(), p->get_nulls_first()));
1220  }
1221  }
1222  if (query.get_is_distinct()) {
1223  // extend order_by to include all targetlist entries.
1224  for (int i = 1; i <= static_cast<int>(tlist.size()); i++) {
1225  bool in_orderby = false;
1226  std::for_each(order_by->begin(),
1227  order_by->end(),
1228  [&in_orderby, i](const Analyzer::OrderEntry& oe) {
1229  in_orderby = in_orderby || (i == oe.tle_no);
1230  });
1231  if (!in_orderby) {
1232  order_by->push_back(Analyzer::OrderEntry(i, false, false));
1233  }
1234  }
1235  }
1236  query.set_order_by(order_by);
1237 }
1238 
1239 std::string SQLType::to_string() const {
1240  std::string str;
1241  switch (type) {
1242  case kBOOLEAN:
1243  str = "BOOLEAN";
1244  break;
1245  case kCHAR:
1246  str = "CHAR(" + boost::lexical_cast<std::string>(param1) + ")";
1247  break;
1248  case kVARCHAR:
1249  str = "VARCHAR(" + boost::lexical_cast<std::string>(param1) + ")";
1250  break;
1251  case kTEXT:
1252  str = "TEXT";
1253  break;
1254  case kNUMERIC:
1255  str = "NUMERIC(" + boost::lexical_cast<std::string>(param1);
1256  if (param2 > 0) {
1257  str += ", " + boost::lexical_cast<std::string>(param2);
1258  }
1259  str += ")";
1260  break;
1261  case kDECIMAL:
1262  str = "DECIMAL(" + boost::lexical_cast<std::string>(param1);
1263  if (param2 > 0) {
1264  str += ", " + boost::lexical_cast<std::string>(param2);
1265  }
1266  str += ")";
1267  break;
1268  case kBIGINT:
1269  str = "BIGINT";
1270  break;
1271  case kINT:
1272  str = "INT";
1273  break;
1274  case kTINYINT:
1275  str = "TINYINT";
1276  break;
1277  case kSMALLINT:
1278  str = "SMALLINT";
1279  break;
1280  case kFLOAT:
1281  str = "FLOAT";
1282  break;
1283  case kDOUBLE:
1284  str = "DOUBLE";
1285  break;
1286  case kTIME:
1287  str = "TIME";
1288  if (param1 < 6) {
1289  str += "(" + boost::lexical_cast<std::string>(param1) + ")";
1290  }
1291  break;
1292  case kTIMESTAMP:
1293  str = "TIMESTAMP";
1294  if (param1 <= 9) {
1295  str += "(" + boost::lexical_cast<std::string>(param1) + ")";
1296  }
1297  break;
1298  case kDATE:
1299  str = "DATE";
1300  break;
1301  default:
1302  assert(false);
1303  break;
1304  }
1305  if (is_array) {
1306  str += "[";
1307  if (array_size > 0) {
1308  str += boost::lexical_cast<std::string>(array_size);
1309  }
1310  str += "]";
1311  }
1312  return str;
1313 }
1314 
1315 std::string SelectEntry::to_string() const {
1316  std::string str = select_expr->to_string();
1317  if (alias != nullptr) {
1318  str += " AS " + *alias;
1319  }
1320  return str;
1321 }
1322 
1323 std::string TableRef::to_string() const {
1324  std::string str = *table_name;
1325  if (range_var != nullptr) {
1326  str += " " + *range_var;
1327  }
1328  return str;
1329 }
1330 
1331 std::string ColumnRef::to_string() const {
1332  std::string str;
1333  if (table == nullptr) {
1334  str = *column;
1335  } else if (column == nullptr) {
1336  str = *table + ".*";
1337  } else {
1338  str = *table + "." + *column;
1339  }
1340  return str;
1341 }
1342 
1343 std::string OperExpr::to_string() const {
1344  std::string op_str[] = {
1345  "=", "===", "<>", "<", ">", "<=", ">=", " AND ", " OR ", "NOT", "-", "+", "*", "/"};
1346  std::string str;
1347  if (optype == kUMINUS) {
1348  str = "-(" + left->to_string() + ")";
1349  } else if (optype == kNOT) {
1350  str = "NOT (" + left->to_string() + ")";
1351  } else if (optype == kARRAY_AT) {
1352  str = left->to_string() + "[" + right->to_string() + "]";
1353  } else if (optype == kUNNEST) {
1354  str = "UNNEST(" + left->to_string() + ")";
1355  } else if (optype == kIN) {
1356  str = "(" + left->to_string() + " IN " + right->to_string() + ")";
1357  } else {
1358  str = "(" + left->to_string() + op_str[optype] + right->to_string() + ")";
1359  }
1360  return str;
1361 }
1362 
1363 std::string InExpr::to_string() const {
1364  std::string str = arg->to_string();
1365  if (is_not) {
1366  str += " NOT IN ";
1367  } else {
1368  str += " IN ";
1369  }
1370  return str;
1371 }
1372 
1373 std::string ExistsExpr::to_string() const {
1374  return "EXISTS (" + query->to_string() + ")";
1375 }
1376 
1377 std::string SubqueryExpr::to_string() const {
1378  std::string str;
1379  str = "(";
1380  str += query->to_string();
1381  str += ")";
1382  return str;
1383 }
1384 
1385 std::string IsNullExpr::to_string() const {
1386  std::string str = arg->to_string();
1387  if (is_not) {
1388  str += " IS NOT NULL";
1389  } else {
1390  str += " IS NULL";
1391  }
1392  return str;
1393 }
1394 
1395 std::string InSubquery::to_string() const {
1396  std::string str = InExpr::to_string();
1397  str += subquery->to_string();
1398  return str;
1399 }
1400 
1401 std::string InValues::to_string() const {
1402  std::string str = InExpr::to_string() + "(";
1403  bool notfirst = false;
1404  for (auto& p : value_list) {
1405  if (notfirst) {
1406  str += ", ";
1407  } else {
1408  notfirst = true;
1409  }
1410  str += p->to_string();
1411  }
1412  str += ")";
1413  return str;
1414 }
1415 
1416 std::string BetweenExpr::to_string() const {
1417  std::string str = arg->to_string();
1418  if (is_not) {
1419  str += " NOT BETWEEN ";
1420  } else {
1421  str += " BETWEEN ";
1422  }
1423  str += lower->to_string() + " AND " + upper->to_string();
1424  return str;
1425 }
1426 
1427 std::string CharLengthExpr::to_string() const {
1428  std::string str;
1429  if (calc_encoded_length) {
1430  str = "CHAR_LENGTH (" + arg->to_string() + ")";
1431  } else {
1432  str = "LENGTH (" + arg->to_string() + ")";
1433  }
1434  return str;
1435 }
1436 
1437 std::string CardinalityExpr::to_string() const {
1438  std::string str = "CARDINALITY(" + arg->to_string() + ")";
1439  return str;
1440 }
1441 
1442 std::string LikeExpr::to_string() const {
1443  std::string str = arg->to_string();
1444  if (is_not) {
1445  str += " NOT LIKE ";
1446  } else {
1447  str += " LIKE ";
1448  }
1449  str += like_string->to_string();
1450  if (escape_string != nullptr) {
1451  str += " ESCAPE " + escape_string->to_string();
1452  }
1453  return str;
1454 }
1455 
1456 std::string RegexpExpr::to_string() const {
1457  std::string str = arg->to_string();
1458  if (is_not) {
1459  str += " NOT REGEXP ";
1460  } else {
1461  str += " REGEXP ";
1462  }
1463  str += pattern_string->to_string();
1464  if (escape_string != nullptr) {
1465  str += " ESCAPE " + escape_string->to_string();
1466  }
1467  return str;
1468 }
1469 
1470 std::string LikelihoodExpr::to_string() const {
1471  std::string str = " LIKELIHOOD ";
1472  str += arg->to_string();
1473  str += " ";
1474  str += boost::lexical_cast<std::string>(is_not ? 1.0 - likelihood : likelihood);
1475  return str;
1476 }
1477 
1478 std::string FunctionRef::to_string() const {
1479  std::string str = *name + "(";
1480  if (distinct) {
1481  str += "DISTINCT ";
1482  }
1483  if (arg == nullptr) {
1484  str += "*)";
1485  } else {
1486  str += arg->to_string() + ")";
1487  }
1488  return str;
1489 }
1490 
1491 std::string QuerySpec::to_string() const {
1492  std::string query_str = "SELECT ";
1493  if (is_distinct) {
1494  query_str += "DISTINCT ";
1495  }
1496  if (select_clause.empty()) {
1497  query_str += "* ";
1498  } else {
1499  bool notfirst = false;
1500  for (auto& p : select_clause) {
1501  if (notfirst) {
1502  query_str += ", ";
1503  } else {
1504  notfirst = true;
1505  }
1506  query_str += p->to_string();
1507  }
1508  }
1509  query_str += " FROM ";
1510  bool notfirst = false;
1511  for (auto& p : from_clause) {
1512  if (notfirst) {
1513  query_str += ", ";
1514  } else {
1515  notfirst = true;
1516  }
1517  query_str += p->to_string();
1518  }
1519  if (where_clause) {
1520  query_str += " WHERE " + where_clause->to_string();
1521  }
1522  if (!groupby_clause.empty()) {
1523  query_str += " GROUP BY ";
1524  bool notfirst = false;
1525  for (auto& p : groupby_clause) {
1526  if (notfirst) {
1527  query_str += ", ";
1528  } else {
1529  notfirst = true;
1530  }
1531  query_str += p->to_string();
1532  }
1533  }
1534  if (having_clause) {
1535  query_str += " HAVING " + having_clause->to_string();
1536  }
1537  query_str += ";";
1538  return query_str;
1539 }
1540 
1541 void InsertStmt::analyze(const Catalog_Namespace::Catalog& catalog,
1542  Analyzer::Query& query) const {
1543  query.set_stmt_type(kINSERT);
1544  const TableDescriptor* td = catalog.getMetadataForTable(*table);
1545  if (td == nullptr) {
1546  throw std::runtime_error("Table " + *table + " does not exist.");
1547  }
1548  if (td->isView) {
1549  throw std::runtime_error("Insert to views is not supported yet.");
1550  }
1551  query.set_result_table_id(td->tableId);
1552  std::list<int> result_col_list;
1553  if (column_list.empty()) {
1554  const std::list<const ColumnDescriptor*> all_cols =
1555  catalog.getAllColumnMetadataForTable(td->tableId, false, false, true);
1556  for (auto cd : all_cols) {
1557  result_col_list.push_back(cd->columnId);
1558  }
1559  } else {
1560  for (auto& c : column_list) {
1561  const ColumnDescriptor* cd = catalog.getMetadataForColumn(td->tableId, *c);
1562  if (cd == nullptr) {
1563  throw std::runtime_error("Column " + *c + " does not exist.");
1564  }
1565  result_col_list.push_back(cd->columnId);
1566  const auto& col_ti = cd->columnType;
1567  if (col_ti.get_physical_cols() > 0) {
1568  CHECK(cd->columnType.is_geometry());
1569  for (auto i = 1; i <= col_ti.get_physical_cols(); i++) {
1570  const ColumnDescriptor* pcd =
1571  catalog.getMetadataForColumn(td->tableId, cd->columnId + i);
1572  if (pcd == nullptr) {
1573  throw std::runtime_error("Column " + *c + "'s metadata is incomplete.");
1574  }
1575  result_col_list.push_back(pcd->columnId);
1576  }
1577  }
1578  }
1579  if (catalog.getAllColumnMetadataForTable(td->tableId, false, false, true).size() !=
1580  result_col_list.size()) {
1581  throw std::runtime_error("Insert into a subset of columns is not supported yet.");
1582  }
1583  }
1584  query.set_result_col_list(result_col_list);
1585 }
1586 
1587 size_t InsertValuesStmt::determineLeafIndex(const Catalog_Namespace::Catalog& catalog,
1588  size_t num_leafs) {
1589  const TableDescriptor* td = catalog.getMetadataForTable(*table);
1590  if (td == nullptr) {
1591  throw std::runtime_error("Table " + *table + " does not exist.");
1592  }
1593  if (td->isView) {
1594  throw std::runtime_error("Insert to views is not supported yet.");
1595  }
1596 
1597  if (td->partitions == "REPLICATED") {
1598  throw std::runtime_error("Cannot determine leaf on replicated table.");
1599  }
1600 
1601  if (0 == td->nShards) {
1602  std::random_device rd;
1603  std::mt19937_64 gen(rd());
1604  std::uniform_int_distribution<size_t> dis;
1605  const auto leaf_idx = dis(gen) % num_leafs;
1606  return leaf_idx;
1607  }
1608 
1609  size_t indexOfShardColumn = 0;
1610  const ColumnDescriptor* shardColumn = catalog.getShardColumnMetadataForTable(td);
1611  CHECK(shardColumn);
1612 
1613  if (column_list.empty()) {
1614  auto all_cols =
1615  catalog.getAllColumnMetadataForTable(td->tableId, false, false, false);
1616  auto iter = std::find(all_cols.begin(), all_cols.end(), shardColumn);
1617  CHECK(iter != all_cols.end());
1618  indexOfShardColumn = std::distance(all_cols.begin(), iter);
1619  } else {
1620  for (auto& c : column_list) {
1621  if (*c == shardColumn->columnName) {
1622  break;
1623  }
1624  indexOfShardColumn++;
1625  }
1626 
1627  if (indexOfShardColumn == column_list.size()) {
1628  throw std::runtime_error("No value defined for shard column.");
1629  }
1630  }
1631 
1632  if (indexOfShardColumn >= value_list.size()) {
1633  throw std::runtime_error("No value defined for shard column.");
1634  }
1635 
1636  auto& shardColumnValueExpr = *(std::next(value_list.begin(), indexOfShardColumn));
1637 
1638  Analyzer::Query query;
1639  auto e = shardColumnValueExpr->analyze(catalog, query);
1640  e = e->add_cast(shardColumn->columnType);
1641  const Analyzer::Constant* con = dynamic_cast<Analyzer::Constant*>(e.get());
1642  if (!con) {
1643  auto col_cast = dynamic_cast<const Analyzer::UOper*>(e.get());
1644  CHECK(col_cast);
1645  CHECK_EQ(kCAST, col_cast->get_optype());
1646  con = dynamic_cast<const Analyzer::Constant*>(col_cast->get_operand());
1647  }
1648  CHECK(con);
1649 
1650  Datum d = con->get_constval();
1651 
1652  auto shard_count = td->nShards * num_leafs;
1653  int64_t shardId = 0;
1654 
1655  if (con->get_is_null()) {
1657  shard_count);
1658  } else if (shardColumn->columnType.is_string()) {
1659  auto dictDesc =
1660  catalog.getMetadataForDict(shardColumn->columnType.get_comp_param(), true);
1661  auto str_id = dictDesc->stringDict->getOrAdd(*d.stringval);
1662  bool invalid = false;
1663 
1664  if (4 == shardColumn->columnType.get_size()) {
1665  invalid = str_id > max_valid_int_value<int32_t>();
1666  } else if (2 == shardColumn->columnType.get_size()) {
1667  invalid = str_id > max_valid_int_value<uint16_t>();
1668  } else if (1 == shardColumn->columnType.get_size()) {
1669  invalid = str_id > max_valid_int_value<uint8_t>();
1670  }
1671 
1672  if (invalid || str_id == inline_int_null_value<int32_t>()) {
1673  str_id = inline_fixed_encoding_null_val(shardColumn->columnType);
1674  }
1675  shardId = SHARD_FOR_KEY(str_id, shard_count);
1676  } else {
1677  switch (shardColumn->columnType.get_logical_size()) {
1678  case 8:
1679  shardId = SHARD_FOR_KEY(d.bigintval, shard_count);
1680  break;
1681  case 4:
1682  shardId = SHARD_FOR_KEY(d.intval, shard_count);
1683  break;
1684  case 2:
1685  shardId = SHARD_FOR_KEY(d.smallintval, shard_count);
1686  break;
1687  case 1:
1688  shardId = SHARD_FOR_KEY(d.tinyintval, shard_count);
1689  break;
1690  default:
1691  CHECK(false);
1692  }
1693  }
1694 
1695  return shardId / td->nShards;
1696 }
1697 
1698 void InsertValuesStmt::analyze(const Catalog_Namespace::Catalog& catalog,
1699  Analyzer::Query& query) const {
1700  InsertStmt::analyze(catalog, query);
1701  std::vector<std::shared_ptr<Analyzer::TargetEntry>>& tlist =
1702  query.get_targetlist_nonconst();
1703  const auto tableId = query.get_result_table_id();
1704  const std::list<const ColumnDescriptor*> non_phys_cols =
1705  catalog.getAllColumnMetadataForTable(tableId, false, false, false);
1706  if (non_phys_cols.size() != value_list.size()) {
1707  throw std::runtime_error("Insert has more target columns than expressions.");
1708  }
1709  std::list<int>::const_iterator it = query.get_result_col_list().begin();
1710  for (auto& v : value_list) {
1711  auto e = v->analyze(catalog, query);
1712  const ColumnDescriptor* cd =
1713  catalog.getMetadataForColumn(query.get_result_table_id(), *it);
1714  CHECK(cd);
1715  if (cd->columnType.get_notnull()) {
1716  auto c = std::dynamic_pointer_cast<Analyzer::Constant>(e);
1717  if (c != nullptr && c->get_is_null()) {
1718  throw std::runtime_error("Cannot insert NULL into column " + cd->columnName);
1719  }
1720  }
1721  e = e->add_cast(cd->columnType);
1722  tlist.emplace_back(new Analyzer::TargetEntry("", e, false));
1723  ++it;
1724 
1725  const auto& col_ti = cd->columnType;
1726  if (col_ti.get_physical_cols() > 0) {
1727  CHECK(cd->columnType.is_geometry());
1728  std::string* wkt{nullptr};
1729  auto c = std::dynamic_pointer_cast<Analyzer::Constant>(e);
1730  if (c) {
1731  wkt = c->get_constval().stringval;
1732  } else {
1733  auto uoper = std::dynamic_pointer_cast<Analyzer::UOper>(e);
1734  if (uoper && uoper->get_optype() == kCAST) {
1735  auto c = dynamic_cast<const Analyzer::Constant*>(uoper->get_operand());
1736  if (c) {
1737  wkt = c->get_constval().stringval;
1738  }
1739  }
1740  }
1741  if (!wkt) {
1742  throw std::runtime_error("Expecting a WKT string for column " + cd->columnName);
1743  }
1744  std::vector<double> coords;
1745  std::vector<double> bounds;
1746  std::vector<int> ring_sizes;
1747  std::vector<int> poly_rings;
1748  int render_group =
1749  0; // @TODO simon.eves where to get render_group from in this context?!
1750  SQLTypeInfo import_ti;
1752  *wkt, import_ti, coords, bounds, ring_sizes, poly_rings)) {
1753  throw std::runtime_error("Cannot read geometry to insert into column " +
1754  cd->columnName);
1755  }
1756  if (cd->columnType.get_type() != import_ti.get_type()) {
1757  // allow POLYGON to be inserted into MULTIPOLYGON column
1758  if (!(import_ti.get_type() == SQLTypes::kPOLYGON &&
1760  throw std::runtime_error("Imported geometry doesn't match the type of column " +
1761  cd->columnName);
1762  }
1763  }
1764  // TODO: check if import SRID matches columns SRID, may need to transform before
1765  // inserting
1766 
1767  int nextColumnOffset = 1;
1768 
1769  const ColumnDescriptor* cd_coords = catalog.getMetadataForColumn(
1770  query.get_result_table_id(), cd->columnId + nextColumnOffset);
1771  CHECK(cd_coords);
1772  CHECK_EQ(cd_coords->columnType.get_type(), kARRAY);
1773  CHECK_EQ(cd_coords->columnType.get_subtype(), kTINYINT);
1774  auto compressed_coords = Importer_NS::compress_coords(coords, col_ti);
1775  std::list<std::shared_ptr<Analyzer::Expr>> value_exprs;
1776  for (auto cc : compressed_coords) {
1777  Datum d;
1778  d.tinyintval = cc;
1779  auto e = makeExpr<Analyzer::Constant>(kTINYINT, false, d);
1780  value_exprs.push_back(e);
1781  }
1782  tlist.emplace_back(new Analyzer::TargetEntry(
1783  "",
1784  makeExpr<Analyzer::Constant>(cd_coords->columnType, false, value_exprs),
1785  false));
1786  ++it;
1787  nextColumnOffset++;
1788 
1789  if (cd->columnType.get_type() == kPOLYGON ||
1790  cd->columnType.get_type() == kMULTIPOLYGON) {
1791  // Put ring sizes array into separate physical column
1792  const ColumnDescriptor* cd_ring_sizes = catalog.getMetadataForColumn(
1793  query.get_result_table_id(), cd->columnId + nextColumnOffset);
1794  CHECK(cd_ring_sizes);
1795  CHECK_EQ(cd_ring_sizes->columnType.get_type(), kARRAY);
1796  CHECK_EQ(cd_ring_sizes->columnType.get_subtype(), kINT);
1797  std::list<std::shared_ptr<Analyzer::Expr>> value_exprs;
1798  for (auto c : ring_sizes) {
1799  Datum d;
1800  d.intval = c;
1801  auto e = makeExpr<Analyzer::Constant>(kINT, false, d);
1802  value_exprs.push_back(e);
1803  }
1804  tlist.emplace_back(new Analyzer::TargetEntry(
1805  "",
1806  makeExpr<Analyzer::Constant>(cd_ring_sizes->columnType, false, value_exprs),
1807  false));
1808  ++it;
1809  nextColumnOffset++;
1810 
1811  if (cd->columnType.get_type() == kMULTIPOLYGON) {
1812  // Put poly_rings array into separate physical column
1813  const ColumnDescriptor* cd_poly_rings = catalog.getMetadataForColumn(
1814  query.get_result_table_id(), cd->columnId + nextColumnOffset);
1815  CHECK(cd_poly_rings);
1816  CHECK_EQ(cd_poly_rings->columnType.get_type(), kARRAY);
1817  CHECK_EQ(cd_poly_rings->columnType.get_subtype(), kINT);
1818  std::list<std::shared_ptr<Analyzer::Expr>> value_exprs;
1819  for (auto c : poly_rings) {
1820  Datum d;
1821  d.intval = c;
1822  auto e = makeExpr<Analyzer::Constant>(kINT, false, d);
1823  value_exprs.push_back(e);
1824  }
1825  tlist.emplace_back(new Analyzer::TargetEntry(
1826  "",
1827  makeExpr<Analyzer::Constant>(cd_poly_rings->columnType, false, value_exprs),
1828  false));
1829  ++it;
1830  nextColumnOffset++;
1831  }
1832  }
1833 
1834  if (cd->columnType.get_type() == kLINESTRING ||
1835  cd->columnType.get_type() == kPOLYGON ||
1836  cd->columnType.get_type() == kMULTIPOLYGON) {
1837  const ColumnDescriptor* cd_bounds = catalog.getMetadataForColumn(
1838  query.get_result_table_id(), cd->columnId + nextColumnOffset);
1839  CHECK(cd_bounds);
1840  CHECK_EQ(cd_bounds->columnType.get_type(), kARRAY);
1841  CHECK_EQ(cd_bounds->columnType.get_subtype(), kDOUBLE);
1842  std::list<std::shared_ptr<Analyzer::Expr>> value_exprs;
1843  for (auto b : bounds) {
1844  Datum d;
1845  d.doubleval = b;
1846  auto e = makeExpr<Analyzer::Constant>(kDOUBLE, false, d);
1847  value_exprs.push_back(e);
1848  }
1849  tlist.emplace_back(new Analyzer::TargetEntry(
1850  "",
1851  makeExpr<Analyzer::Constant>(cd_bounds->columnType, false, value_exprs),
1852  false));
1853  ++it;
1854  nextColumnOffset++;
1855  }
1856 
1857  if (cd->columnType.get_type() == kPOLYGON ||
1858  cd->columnType.get_type() == kMULTIPOLYGON) {
1859  // Put render group into separate physical column
1860  const ColumnDescriptor* cd_render_group = catalog.getMetadataForColumn(
1861  query.get_result_table_id(), cd->columnId + nextColumnOffset);
1862  CHECK(cd_render_group);
1863  CHECK_EQ(cd_render_group->columnType.get_type(), kINT);
1864  Datum d;
1865  d.intval = render_group;
1866  tlist.emplace_back(new Analyzer::TargetEntry(
1867  "",
1868  makeExpr<Analyzer::Constant>(cd_render_group->columnType, false, d),
1869  false));
1870  ++it;
1871  nextColumnOffset++;
1872  }
1873  }
1874  }
1875 }
1876 
1877 void InsertQueryStmt::analyze(const Catalog_Namespace::Catalog& catalog,
1878  Analyzer::Query& insert_query) const {
1879  InsertStmt::analyze(catalog, insert_query);
1880  query->analyze(catalog, insert_query);
1881 }
1882 
1883 void UpdateStmt::analyze(const Catalog_Namespace::Catalog& catalog,
1884  Analyzer::Query& query) const {
1885  throw std::runtime_error("UPDATE statement not supported yet.");
1886 }
1887 
1888 void DeleteStmt::analyze(const Catalog_Namespace::Catalog& catalog,
1889  Analyzer::Query& query) const {
1890  throw std::runtime_error("DELETE statement not supported yet.");
1891 }
1892 
1893 void SQLType::check_type() {
1894  switch (type) {
1895  case kCHAR:
1896  case kVARCHAR:
1897  if (param1 <= 0) {
1898  throw std::runtime_error("CHAR and VARCHAR must have a positive dimension.");
1899  }
1900  break;
1901  case kDECIMAL:
1902  case kNUMERIC:
1903  if (param1 <= 0) {
1904  throw std::runtime_error("DECIMAL and NUMERIC must have a positive precision.");
1905  } else if (param1 > 19) {
1906  throw std::runtime_error(
1907  "DECIMAL and NUMERIC precision cannot be larger than 19.");
1908  } else if (param1 <= param2) {
1909  throw std::runtime_error(
1910  "DECIMAL and NUMERIC must have precision larger than scale.");
1911  }
1912  break;
1913  case kTIMESTAMP:
1914  if (param1 == -1) {
1915  param1 = 0; // set default to 0
1916  } else if (param1 != 0 && param1 != 3 && param1 != 6 &&
1917  param1 != 9) { // support ms, us, ns
1918  throw std::runtime_error(
1919  "Only TIMESTAMP(n) where n = (0,3,6,9) are supported now.");
1920  }
1921  break;
1922  case kTIME:
1923  if (param1 == -1) {
1924  param1 = 0; // default precision is 0
1925  }
1926  if (param1 > 0) { // @TODO(wei) support sub-second precision later.
1927  throw std::runtime_error("Only TIME(0) is supported now.");
1928  }
1929  break;
1930  case kPOINT:
1931  case kLINESTRING:
1932  case kPOLYGON:
1933  case kMULTIPOLYGON:
1934  // Storing SRID in param1
1935  break;
1936  default:
1937  param1 = 0;
1938  break;
1939  }
1940 }
1941 
1942 namespace {
1943 
1945  const auto& col_ti = cd.columnType;
1946  if (col_ti.is_integer() ||
1947  (col_ti.is_string() && col_ti.get_compression() == kENCODING_DICT) ||
1948  col_ti.is_time()) {
1949  return;
1950  }
1951  throw std::runtime_error("Cannot shard on type " + col_ti.get_type_name() +
1952  ", encoding " + col_ti.get_compression_name());
1953 }
1954 
1955 size_t shard_column_index(const std::string& name,
1956  const std::list<ColumnDescriptor>& columns) {
1957  size_t index = 1;
1958  for (const auto& cd : columns) {
1959  if (cd.columnName == name) {
1961  return index;
1962  }
1963  ++index;
1964  if (cd.columnType.is_geometry()) {
1965  index += cd.columnType.get_physical_cols();
1966  }
1967  }
1968  // Not found, return 0
1969  return 0;
1970 }
1971 
1972 size_t sort_column_index(const std::string& name,
1973  const std::list<ColumnDescriptor>& columns) {
1974  size_t index = 1;
1975  for (const auto& cd : columns) {
1976  if (boost::to_upper_copy<std::string>(cd.columnName) == name) {
1977  return index;
1978  }
1979  ++index;
1980  if (cd.columnType.is_geometry()) {
1981  index += cd.columnType.get_physical_cols();
1982  }
1983  }
1984  // Not found, return 0
1985  return 0;
1986 }
1987 
1988 void set_string_field(rapidjson::Value& obj,
1989  const std::string& field_name,
1990  const std::string& field_value,
1991  rapidjson::Document& document) {
1992  rapidjson::Value field_name_json_str;
1993  field_name_json_str.SetString(
1994  field_name.c_str(), field_name.size(), document.GetAllocator());
1995  rapidjson::Value field_value_json_str;
1996  field_value_json_str.SetString(
1997  field_value.c_str(), field_value.size(), document.GetAllocator());
1998  obj.AddMember(field_name_json_str, field_value_json_str, document.GetAllocator());
1999 }
2000 
2002  const ShardKeyDef* shard_key_def,
2003  const std::vector<SharedDictionaryDef>& shared_dict_defs) {
2004  rapidjson::Document document;
2005  auto& allocator = document.GetAllocator();
2006  rapidjson::Value arr(rapidjson::kArrayType);
2007  if (shard_key_def) {
2008  rapidjson::Value shard_key_obj(rapidjson::kObjectType);
2009  set_string_field(shard_key_obj, "type", "SHARD KEY", document);
2010  set_string_field(shard_key_obj, "name", shard_key_def->get_column(), document);
2011  arr.PushBack(shard_key_obj, allocator);
2012  }
2013  for (const auto& shared_dict_def : shared_dict_defs) {
2014  rapidjson::Value shared_dict_obj(rapidjson::kObjectType);
2015  set_string_field(shared_dict_obj, "type", "SHARED DICTIONARY", document);
2016  set_string_field(shared_dict_obj, "name", shared_dict_def.get_column(), document);
2018  shared_dict_obj, "foreign_table", shared_dict_def.get_foreign_table(), document);
2019  set_string_field(shared_dict_obj,
2020  "foreign_column",
2021  shared_dict_def.get_foreign_column(),
2022  document);
2023  arr.PushBack(shared_dict_obj, allocator);
2024  }
2025  rapidjson::StringBuffer buffer;
2026  rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
2027  arr.Accept(writer);
2028  return buffer.GetString();
2029 }
2030 
2031 template <typename LITERAL_TYPE,
2032  typename ASSIGNMENT,
2033  typename VALIDATE = DefaultValidate<LITERAL_TYPE>>
2034 decltype(auto) get_property_value(const NameValueAssign* p,
2035  ASSIGNMENT op,
2036  VALIDATE validate = VALIDATE()) {
2037  const auto val = validate(p);
2038  return op(val);
2039 }
2040 
2042  const NameValueAssign* p,
2043  const std::list<ColumnDescriptor>& columns) {
2044  return get_property_value<IntLiteral>(p,
2045  [&td](const auto val) { td.maxFragRows = val; });
2046 }
2047 
2049  const NameValueAssign* p,
2050  const std::list<ColumnDescriptor>& columns) {
2051  return get_property_value<IntLiteral>(p,
2052  [&td](const auto val) { td.maxChunkSize = val; });
2053 }
2054 
2056  const NameValueAssign* p,
2057  const std::list<ColumnDescriptor>& columns) {
2058  return get_property_value<IntLiteral>(p,
2059  [&td](const auto val) { td.fragPageSize = val; });
2060 }
2062  const NameValueAssign* p,
2063  const std::list<ColumnDescriptor>& columns) {
2064  return get_property_value<IntLiteral>(p, [&td](const auto val) { td.maxRows = val; });
2065 }
2067  const NameValueAssign* p,
2068  const std::list<ColumnDescriptor>& columns) {
2069  return get_property_value<StringLiteral>(p, [&td](const auto partitions_uc) {
2070  if (partitions_uc != "SHARDED" && partitions_uc != "REPLICATED") {
2071  throw std::runtime_error("PARTITIONS must be SHARDED or REPLICATED");
2072  }
2073  if (td.shardedColumnId != 0 && partitions_uc == "REPLICATED") {
2074  throw std::runtime_error(
2075  "A table cannot be sharded and replicated at the same time");
2076  };
2077  td.partitions = partitions_uc;
2078  });
2079 }
2081  const NameValueAssign* p,
2082  const std::list<ColumnDescriptor>& columns) {
2083  if (!td.shardedColumnId) {
2084  throw std::runtime_error("SHARD KEY must be defined.");
2085  }
2086  return get_property_value<IntLiteral>(p, [&td](const auto shard_count) {
2087  if (g_leaf_count && shard_count % g_leaf_count) {
2088  throw std::runtime_error(
2089  "SHARD_COUNT must be a multiple of the number of leaves in the cluster.");
2090  }
2091  td.nShards = g_leaf_count ? shard_count / g_leaf_count : shard_count;
2092  if (!td.shardedColumnId && !td.nShards) {
2093  throw std::runtime_error(
2094  "Must specify the number of shards through the SHARD_COUNT option");
2095  };
2096  });
2097 }
2098 
2099 decltype(auto) get_vacuum_def(TableDescriptor& td,
2100  const NameValueAssign* p,
2101  const std::list<ColumnDescriptor>& columns) {
2102  return get_property_value<StringLiteral>(p, [&td](const auto vacuum_uc) {
2103  if (vacuum_uc != "IMMEDIATE" && vacuum_uc != "DELAYED") {
2104  throw std::runtime_error("VACUUM must be IMMEDIATE or DELAYED");
2105  }
2106  td.hasDeletedCol = boost::iequals(vacuum_uc, "IMMEDIATE") ? false : true;
2107  });
2108 }
2109 
2111  const NameValueAssign* p,
2112  const std::list<ColumnDescriptor>& columns) {
2113  return get_property_value<StringLiteral>(p, [&td, &columns](const auto sort_upper) {
2114  td.sortedColumnId = sort_column_index(sort_upper, columns);
2115  if (!td.sortedColumnId) {
2116  throw std::runtime_error("Specified sort column " + sort_upper + " doesn't exist");
2117  }
2118  });
2119 }
2120 
2121 static const std::map<const std::string, const TableDefFuncPtr> tableDefFuncMap = {
2122  {"fragment_size"s, get_frag_size_def},
2123  {"max_chunk_size"s, get_max_chunk_size_def},
2124  {"page_size"s, get_page_size_def},
2125  {"max_rows"s, get_max_rows_def},
2126  {"partitions"s, get_partions_def},
2127  {"shard_count"s, get_shard_count_def},
2128  {"vacuum"s, get_vacuum_def},
2129  {"sort_column"s, get_sort_column_def}};
2130 
2132  const std::unique_ptr<NameValueAssign>& p,
2133  const std::list<ColumnDescriptor>& columns) {
2134  const auto it = tableDefFuncMap.find(boost::to_lower_copy<std::string>(*p->get_name()));
2135  if (it == tableDefFuncMap.end()) {
2136  throw std::runtime_error("Invalid CREATE TABLE option " + *p->get_name() +
2137  ". Should be FRAGMENT_SIZE, PAGE_SIZE, MAX_ROWS, "
2138  "PARTITIONS, VACUUM, SORT_COLUMN or SHARD_COUNT.");
2139  }
2140  return it->second(td, p.get(), columns);
2141 }
2142 
2143 } // namespace
2144 
2145 void CreateTableStmt::executeDryRun(const Catalog_Namespace::SessionInfo& session,
2146  TableDescriptor& td,
2147  std::list<ColumnDescriptor>& columns,
2148  std::vector<SharedDictionaryDef>& shared_dict_defs) {
2149  std::unordered_set<std::string> uc_col_names;
2150  const auto& catalog = session.getCatalog();
2151  const ShardKeyDef* shard_key_def{nullptr};
2152  for (auto& e : table_element_list_) {
2153  if (dynamic_cast<SharedDictionaryDef*>(e.get())) {
2154  auto shared_dict_def = static_cast<SharedDictionaryDef*>(e.get());
2156  this, shared_dict_def, columns, shared_dict_defs, catalog);
2157  shared_dict_defs.push_back(*shared_dict_def);
2158  continue;
2159  }
2160  if (dynamic_cast<ShardKeyDef*>(e.get())) {
2161  if (shard_key_def) {
2162  throw std::runtime_error("Specified more than one shard key");
2163  }
2164  shard_key_def = static_cast<const ShardKeyDef*>(e.get());
2165  continue;
2166  }
2167  if (!dynamic_cast<ColumnDef*>(e.get())) {
2168  throw std::runtime_error("Table constraints are not supported yet.");
2169  }
2170  ColumnDef* coldef = static_cast<ColumnDef*>(e.get());
2171  ColumnDescriptor cd;
2172  cd.columnName = *coldef->get_column_name();
2173  const auto uc_col_name = boost::to_upper_copy<std::string>(cd.columnName);
2174  if (reserved_keywords.find(uc_col_name) != reserved_keywords.end()) {
2175  throw std::runtime_error("Cannot create column with reserved keyword '" +
2176  cd.columnName + "'");
2177  }
2178  const auto it_ok = uc_col_names.insert(uc_col_name);
2179  if (!it_ok.second) {
2180  throw std::runtime_error("Column '" + cd.columnName + "' defined more than once");
2181  }
2182 
2183  setColumnDescriptor(cd, coldef);
2184  columns.push_back(cd);
2185  }
2186 
2187  td.tableName = *table_;
2188  td.nColumns = columns.size();
2189  td.isView = false;
2190  td.fragmenter = nullptr;
2196  if (shard_key_def) {
2197  td.shardedColumnId = shard_column_index(shard_key_def->get_column(), columns);
2198  if (!td.shardedColumnId) {
2199  throw std::runtime_error("Specified shard column " + shard_key_def->get_column() +
2200  " doesn't exist");
2201  }
2202  }
2203  if (is_temporary_) {
2205  } else {
2207  }
2208  if (!storage_options_.empty()) {
2209  for (auto& p : storage_options_) {
2210  get_table_definitions(td, p, columns);
2211  }
2212  }
2213  if (td.shardedColumnId && !td.nShards) {
2214  throw std::runtime_error("SHARD_COUNT needs to be specified with SHARD_KEY.");
2215  }
2216  td.keyMetainfo = serialize_key_metainfo(shard_key_def, shared_dict_defs);
2217 }
2218 
2219 void CreateTableStmt::execute(const Catalog_Namespace::SessionInfo& session) {
2220  auto& catalog = session.getCatalog();
2221  // check access privileges
2224  throw std::runtime_error("Table " + *table_ +
2225  " will not be created. User has no create privileges.");
2226  }
2227 
2228  if (catalog.getMetadataForTable(*table_) != nullptr) {
2229  if (if_not_exists_) {
2230  return;
2231  }
2232  throw std::runtime_error("Table " + *table_ + " already exists.");
2233  }
2234  TableDescriptor td;
2235  std::list<ColumnDescriptor> columns;
2236  std::unordered_set<std::string> uc_col_names;
2237  std::vector<SharedDictionaryDef> shared_dict_defs;
2238 
2239  executeDryRun(session, td, columns, shared_dict_defs);
2240  td.userId = session.get_currentUser().userId;
2241 
2242  catalog.createShardedTable(td, columns, shared_dict_defs);
2243  // TODO (max): It's transactionally unsafe, should be fixed: we may create object w/o
2244  // privileges
2245  SysCatalog::instance().createDBObject(
2246  session.get_currentUser(), td.tableName, TableDBObjectType, catalog);
2247 }
2248 
2249 std::shared_ptr<ResultSet> getResultSet(QueryStateProxy query_state_proxy,
2250  const std::string select_stmt,
2251  std::vector<TargetMetaInfo>& targets,
2252  bool validate_only = false) {
2253  auto const session = query_state_proxy.getQueryState().getConstSessionInfo();
2254  auto& catalog = session->getCatalog();
2255 
2256  auto executor = Executor::getExecutor(catalog.getCurrentDB().dbId);
2257 
2258 #ifdef HAVE_CUDA
2259  const auto device_type = session->get_executor_device_type();
2260 #else
2261  const auto device_type = ExecutorDeviceType::CPU;
2262 #endif // HAVE_CUDA
2263  auto calcite_mgr = catalog.getCalciteMgr();
2264 
2265  // TODO MAT this should actually get the global or the session parameter for
2266  // view optimization
2267  const auto query_ra =
2268  calcite_mgr
2269  ->process(query_state_proxy, pg_shim(select_stmt), {}, true, false, false)
2270  .plan_result;
2271  CompilationOptions co = {
2272  device_type, true, ExecutorOptLevel::LoopStrengthReduction, false};
2273  // TODO(adb): Need a better method of dropping constants into this ExecutionOptions
2274  // struct
2275  ExecutionOptions eo = {false,
2276  true,
2277  false,
2278  true,
2279  false,
2280  false,
2281  validate_only,
2282  false,
2283  10000,
2284  false,
2285  false,
2286  0.9};
2287  RelAlgExecutor ra_executor(executor.get(), catalog);
2288  ExecutionResult result{std::make_shared<ResultSet>(std::vector<TargetInfo>{},
2291  nullptr,
2292  nullptr),
2293  {}};
2294  result = ra_executor.executeRelAlgQuery(query_ra, co, eo, nullptr);
2295  targets = result.getTargetsMeta();
2296 
2297  return result.getRows();
2298 }
2299 
2300 AggregatedResult InsertIntoTableAsSelectStmt::LocalConnector::query(
2301  QueryStateProxy query_state_proxy,
2302  std::string& sql_query_string,
2303  bool validate_only) {
2304  auto const session = query_state_proxy.getQueryState().getConstSessionInfo();
2305  std::string pg_shimmed_select_query = pg_shim(sql_query_string);
2306  const auto query_ra = parse_to_ra(query_state_proxy, pg_shimmed_select_query);
2307 
2308  // Note that the below command should *only* get read locks on the source table for
2309  // InsertIntoAsSelect. Write locks will be taken on the target table in
2310  // InsertOrderFragmenter::deleteFragments. This potentially leaves a small window where
2311  // the target table is not locked, and should be investigated.
2312  std::vector<TableLock> table_locks;
2313  TableLockMgr::getTableLocks(session->getCatalog(), query_ra, table_locks);
2314 
2315  std::vector<TargetMetaInfo> target_metainfos;
2316 
2317  auto result_rows =
2318  getResultSet(query_state_proxy, sql_query_string, target_metainfos, validate_only);
2319  AggregatedResult res = {result_rows, target_metainfos};
2320  return res;
2321 }
2322 
2323 AggregatedResult InsertIntoTableAsSelectStmt::LocalConnector::query(
2324  QueryStateProxy query_state_proxy,
2325  std::string& sql_query_string) {
2326  return query(query_state_proxy, sql_query_string, false);
2327 }
2328 
2329 void InsertIntoTableAsSelectStmt::LocalConnector::insertDataToLeaf(
2330  const Catalog_Namespace::SessionInfo& session,
2331  const size_t leaf_idx,
2332  Fragmenter_Namespace::InsertData& insert_data) {
2333  CHECK(leaf_idx == 0);
2334  auto& catalog = session.getCatalog();
2335  auto created_td = catalog.getMetadataForTable(insert_data.tableId);
2336  // get CheckpointLock on the table before trying to create
2337  // its 1st fragment
2338  ChunkKey chunkKey = {catalog.getCurrentDB().dbId, created_td->tableId};
2339  mapd_unique_lock<mapd_shared_mutex> chkptlLock(
2342  // [ TableWriteLock ] lock is deferred in
2343  // InsertOrderFragmenter::deleteFragments
2344  created_td->fragmenter->insertDataNoCheckpoint(insert_data);
2345 }
2346 
2347 void InsertIntoTableAsSelectStmt::LocalConnector::checkpoint(
2348  const Catalog_Namespace::SessionInfo& session,
2349  int tableId) {
2350  auto& catalog = session.getCatalog();
2351  auto dbId = catalog.getCurrentDB().dbId;
2352  catalog.getDataMgr().checkpoint(dbId, tableId);
2353 }
2354 
2355 void InsertIntoTableAsSelectStmt::LocalConnector::rollback(
2356  const Catalog_Namespace::SessionInfo& session,
2357  int tableId) {
2358  auto& catalog = session.getCatalog();
2359  auto dbId = catalog.getCurrentDB().dbId;
2360  catalog.getDataMgr().checkpoint(dbId, tableId);
2361 }
2362 
2363 std::list<ColumnDescriptor>
2364 InsertIntoTableAsSelectStmt::LocalConnector::getColumnDescriptors(
2366  bool for_create) {
2367  std::list<ColumnDescriptor> column_descriptors;
2368  std::list<ColumnDescriptor> column_descriptors_for_create;
2369 
2370  int rowid_suffix = 0;
2371  for (const auto& target_metainfo : result.targets_meta) {
2372  ColumnDescriptor cd;
2373  cd.columnName = target_metainfo.get_resname();
2374  if (cd.columnName == "rowid") {
2375  cd.columnName += std::to_string(rowid_suffix++);
2376  }
2377  cd.columnType = target_metainfo.get_physical_type_info();
2378 
2379  ColumnDescriptor cd_for_create = cd;
2380 
2382  // we need to reset the comp param (as this points to the actual dictionary)
2383  if (cd.columnType.is_array()) {
2384  // for dict encoded arrays, it is always 4 bytes
2385  cd_for_create.columnType.set_comp_param(32);
2386  } else {
2387  cd_for_create.columnType.set_comp_param(cd.columnType.get_size() * 8);
2388  }
2389  }
2390 
2391  column_descriptors_for_create.push_back(cd_for_create);
2392  column_descriptors.push_back(cd);
2393  }
2394 
2395  if (for_create) {
2396  return column_descriptors_for_create;
2397  }
2398 
2399  return column_descriptors;
2400 }
2401 
2402 void InsertIntoTableAsSelectStmt::populateData(QueryStateProxy query_state_proxy,
2403  bool is_temporary,
2404  bool validate_table) {
2405  auto const session = query_state_proxy.getQueryState().getConstSessionInfo();
2406  LocalConnector local_connector;
2407 
2408  bool populate_table = false;
2409 
2410  if (leafs_connector_) {
2411  populate_table = true;
2412  } else {
2413  leafs_connector_ = &local_connector;
2414  if (!g_cluster) {
2415  populate_table = true;
2416  }
2417  }
2418 
2419  auto& catalog = session->getCatalog();
2420  auto get_target_column_descriptors = [this, &catalog](const TableDescriptor* td) {
2421  std::vector<const ColumnDescriptor*> target_column_descriptors;
2422  if (column_list_.empty()) {
2423  auto list = catalog.getAllColumnMetadataForTable(td->tableId, false, false, false);
2424  target_column_descriptors = {std::begin(list), std::end(list)};
2425 
2426  } else {
2427  for (auto& c : column_list_) {
2428  const ColumnDescriptor* cd = catalog.getMetadataForColumn(td->tableId, *c);
2429  if (cd == nullptr) {
2430  throw std::runtime_error("Column " + *c + " does not exist.");
2431  }
2432  target_column_descriptors.push_back(cd);
2433  }
2434  }
2435 
2436  return target_column_descriptors;
2437  };
2438 
2439  if (validate_table) {
2440  // check access privileges
2441  const TableDescriptor* td = catalog.getMetadataForTable(table_name_);
2442 
2443  if (td == nullptr) {
2444  throw std::runtime_error("Table " + table_name_ + " does not exist.");
2445  }
2446  if (td->isView) {
2447  throw std::runtime_error("Insert to views is not supported yet.");
2448  }
2449 
2450  if (!session->checkDBAccessPrivileges(DBObjectType::TableDBObjectType,
2452  table_name_)) {
2453  throw std::runtime_error("User has no insert privileges on " + table_name_ + ".");
2454  }
2455 
2456  // only validate the select query so we get the target types
2457  // correctly, but do not populate the result set
2458  auto result = local_connector.query(query_state_proxy, select_query_, true);
2459  auto source_column_descriptors = local_connector.getColumnDescriptors(result, false);
2460 
2461  std::vector<const ColumnDescriptor*> target_column_descriptors =
2462  get_target_column_descriptors(td);
2463  if (catalog.getAllColumnMetadataForTable(td->tableId, false, false, false).size() !=
2464  target_column_descriptors.size()) {
2465  throw std::runtime_error("Insert into a subset of columns is not supported yet.");
2466  }
2467 
2468  if (source_column_descriptors.size() != target_column_descriptors.size()) {
2469  throw std::runtime_error("The number of source and target columns does not match.");
2470  }
2471 
2472  for (int i = 0; i < source_column_descriptors.size(); i++) {
2473  const ColumnDescriptor* source_cd =
2474  &(*std::next(source_column_descriptors.begin(), i));
2475  const ColumnDescriptor* target_cd = target_column_descriptors.at(i);
2476 
2477  if ((!source_cd->columnType.is_string() && !target_cd->columnType.is_string()) &&
2478  (source_cd->columnType.get_type() != target_cd->columnType.get_type())) {
2479  throw std::runtime_error("Source '" + source_cd->columnName + " " +
2480  source_cd->columnType.get_type_name() +
2481  "' and target '" + target_cd->columnName + " " +
2482  target_cd->columnType.get_type_name() +
2483  "' column types do not match.");
2484  }
2485  if (source_cd->columnType.is_array()) {
2486  if (source_cd->columnType.get_subtype() != target_cd->columnType.get_subtype()) {
2487  throw std::runtime_error("Source '" + source_cd->columnName + " " +
2488  source_cd->columnType.get_type_name() +
2489  "' and target '" + target_cd->columnName + " " +
2490  target_cd->columnType.get_type_name() +
2491  "' array column element types do not match.");
2492  }
2493  }
2494 
2495  if (source_cd->columnType.is_decimal() ||
2496  source_cd->columnType.get_elem_type().is_decimal()) {
2497  SQLTypeInfo sourceType = source_cd->columnType;
2498  SQLTypeInfo targetType = target_cd->columnType;
2499 
2500  if (source_cd->columnType.is_array()) {
2501  sourceType = source_cd->columnType.get_elem_type();
2502  targetType = target_cd->columnType.get_elem_type();
2503  }
2504 
2505  if ((sourceType.get_dimension() != targetType.get_dimension()) ||
2506  (sourceType.get_scale() != targetType.get_scale())) {
2507  throw std::runtime_error(
2508  "Source '" + source_cd->columnName + " " +
2509  source_cd->columnType.get_type_name() + "' and target '" +
2510  target_cd->columnName + " " + target_cd->columnType.get_type_name() +
2511  "' decimal columns dimension and or scale do not match.");
2512  }
2513  }
2514 
2515  if (source_cd->columnType.is_string()) {
2516  if (source_cd->columnType.get_compression() !=
2517  target_cd->columnType.get_compression()) {
2518  throw std::runtime_error("Source '" + source_cd->columnName + " " +
2519  source_cd->columnType.get_type_name() +
2520  "' and target '" + target_cd->columnName + " " +
2521  target_cd->columnType.get_type_name() +
2522  "' columns string encodings do not match.");
2523  }
2524  }
2525 
2526  if (source_cd->columnType.is_timestamp() && target_cd->columnType.is_timestamp()) {
2527  if (source_cd->columnType.get_dimension() !=
2528  target_cd->columnType.get_dimension()) {
2529  throw std::runtime_error("Source '" + source_cd->columnName + " " +
2530  source_cd->columnType.get_type_name() +
2531  "' and target '" + target_cd->columnName + " " +
2532  target_cd->columnType.get_type_name() +
2533  "' timestamp column precisions do not match.");
2534  }
2535  }
2536 
2537  if (!source_cd->columnType.is_string() &&
2538  source_cd->columnType.get_size() > target_cd->columnType.get_size()) {
2539  throw std::runtime_error("Source '" + source_cd->columnName + " " +
2540  source_cd->columnType.get_type_name() +
2541  "' and target '" + target_cd->columnName + " " +
2542  target_cd->columnType.get_type_name() +
2543  "' column encoding sizes do not match.");
2544  }
2545  }
2546  }
2547 
2548  if (!populate_table) {
2549  return;
2550  }
2551 
2552  const TableDescriptor* created_td = catalog.getMetadataForTable(table_name_);
2553  Fragmenter_Namespace::InsertDataLoader insertDataLoader(*leafs_connector_);
2554  AggregatedResult res = leafs_connector_->query(query_state_proxy, select_query_);
2555  auto target_column_descriptors = get_target_column_descriptors(created_td);
2556  auto result_rows = res.rs;
2557  result_rows->setGeoReturnType(ResultSet::GeoReturnType::GeoTargetValue);
2558  const auto num_rows = result_rows->rowCount();
2559 
2560  if (0 == num_rows) {
2561  return;
2562  }
2563 
2564  size_t leaf_count = leafs_connector_->leafCount();
2565 
2566  size_t max_number_of_rows_per_package = std::min(num_rows / leaf_count, 64UL * 1024UL);
2567 
2568  size_t start_row = 0;
2569  size_t num_rows_to_process = std::min(num_rows, max_number_of_rows_per_package);
2570 
2571  // ensure that at least one row is being processed
2572  num_rows_to_process = std::max(num_rows_to_process, 1UL);
2573 
2574  std::vector<std::unique_ptr<TargetValueConverter>> value_converters;
2575 
2577 
2578  const int num_worker_threads = std::thread::hardware_concurrency();
2579 
2580  std::vector<size_t> thread_start_idx(num_worker_threads),
2581  thread_end_idx(num_worker_threads);
2582  bool can_go_parallel = !result_rows->isTruncated() && num_rows_to_process > 20000;
2583 
2584  std::atomic<size_t> row_idx{0};
2585 
2586  auto convert_function = [&result_rows,
2587  &value_converters,
2588  &row_idx,
2589  &num_rows_to_process,
2590  &thread_start_idx,
2591  &thread_end_idx](const int thread_id) {
2592  const int num_cols = value_converters.size();
2593  const size_t start = thread_start_idx[thread_id];
2594  const size_t end = thread_end_idx[thread_id];
2595  size_t idx = 0;
2596  for (idx = start; idx < end; ++idx) {
2597  const auto result_row = result_rows->getRowAtNoTranslations(idx);
2598  if (!result_row.empty()) {
2599  size_t target_row = row_idx.fetch_add(1);
2600 
2601  if (target_row >= num_rows_to_process) {
2602  break;
2603  }
2604 
2605  for (unsigned int col = 0; col < num_cols; col++) {
2606  const auto& mapd_variant = result_row[col];
2607  value_converters[col]->convertToColumnarFormat(target_row, &mapd_variant);
2608  }
2609  }
2610  }
2611 
2612  thread_start_idx[thread_id] = idx;
2613  };
2614 
2615  auto single_threaded_convert_function = [&result_rows,
2616  &value_converters,
2617  &row_idx,
2618  &num_rows_to_process,
2619  &thread_start_idx,
2620  &thread_end_idx](const int thread_id) {
2621  const int num_cols = value_converters.size();
2622  const size_t start = thread_start_idx[thread_id];
2623  const size_t end = thread_end_idx[thread_id];
2624  size_t idx = 0;
2625  for (idx = start; idx < end; ++idx) {
2626  size_t target_row = row_idx.fetch_add(1);
2627 
2628  if (target_row >= num_rows_to_process) {
2629  break;
2630  }
2631  const auto result_row = result_rows->getNextRow(false, false);
2632  CHECK(!result_row.empty());
2633  for (unsigned int col = 0; col < num_cols; col++) {
2634  const auto& mapd_variant = result_row[col];
2635  value_converters[col]->convertToColumnarFormat(target_row, &mapd_variant);
2636  }
2637  }
2638 
2639  thread_start_idx[thread_id] = idx;
2640  };
2641 
2642  if (can_go_parallel) {
2643  const size_t entryCount = result_rows->entryCount();
2644  for (size_t i = 0,
2645  start_entry = 0,
2646  stride = (entryCount + num_worker_threads - 1) / num_worker_threads;
2647  i < num_worker_threads && start_entry < entryCount;
2648  ++i, start_entry += stride) {
2649  const auto end_entry = std::min(start_entry + stride, entryCount);
2650  thread_start_idx[i] = start_entry;
2651  thread_end_idx[i] = end_entry;
2652  }
2653 
2654  } else {
2655  thread_start_idx[0] = 0;
2656  thread_end_idx[0] = result_rows->entryCount();
2657  }
2658 
2659  std::shared_ptr<Executor> executor;
2660 
2662  executor = Executor::getExecutor(catalog.getCurrentDB().dbId);
2663  }
2664 
2665  while (start_row < num_rows) {
2666  try {
2667  value_converters.clear();
2668  row_idx = 0;
2669  int colNum = 0;
2670  for (const auto targetDescriptor : target_column_descriptors) {
2671  auto sourceDataMetaInfo = res.targets_meta[colNum++];
2672 
2674  num_rows_to_process,
2675  catalog,
2676  sourceDataMetaInfo,
2677  targetDescriptor,
2678  targetDescriptor->columnType,
2679  !targetDescriptor->columnType.get_notnull(),
2680  result_rows->getRowSetMemOwner()->getLiteralStringDictProxy(),
2682  ? executor->getStringDictionaryProxy(
2683  sourceDataMetaInfo.get_type_info().get_comp_param(),
2684  result_rows->getRowSetMemOwner(),
2685  true)
2686  : nullptr};
2687  auto converter = factory.create(param);
2688  value_converters.push_back(std::move(converter));
2689  }
2690 
2691  if (can_go_parallel) {
2692  std::vector<std::future<void>> worker_threads;
2693  for (int i = 0; i < num_worker_threads; ++i) {
2694  worker_threads.push_back(std::async(std::launch::async, convert_function, i));
2695  }
2696 
2697  for (auto& child : worker_threads) {
2698  child.wait();
2699  }
2700  for (auto& child : worker_threads) {
2701  child.get();
2702  }
2703 
2704  } else {
2705  single_threaded_convert_function(0);
2706  }
2707 
2708  // finalize the insert data
2709  {
2710  auto finalizer_func =
2711  [](std::unique_ptr<TargetValueConverter>::pointer targetValueConverter) {
2712  targetValueConverter->finalizeDataBlocksForInsertData();
2713  };
2714  std::vector<std::future<void>> worker_threads;
2715  for (auto& converterPtr : value_converters) {
2716  worker_threads.push_back(
2717  std::async(std::launch::async, finalizer_func, converterPtr.get()));
2718  }
2719 
2720  for (auto& child : worker_threads) {
2721  child.wait();
2722  }
2723  for (auto& child : worker_threads) {
2724  child.get();
2725  }
2726  }
2727 
2729  insert_data.databaseId = catalog.getCurrentDB().dbId;
2730  CHECK(created_td);
2731  insert_data.tableId = created_td->tableId;
2732  insert_data.numRows = num_rows_to_process;
2733 
2734  for (int col_idx = 0; col_idx < target_column_descriptors.size(); col_idx++) {
2735  value_converters[col_idx]->addDataBlocksToInsertData(insert_data);
2736  }
2737 
2738  insertDataLoader.insertData(*session, insert_data);
2739  } catch (...) {
2740  try {
2741  if (created_td->nShards) {
2742  const auto shard_tables = catalog.getPhysicalTablesDescriptors(created_td);
2743  for (const auto ptd : shard_tables) {
2744  leafs_connector_->rollback(*session, ptd->tableId);
2745  }
2746  }
2747  leafs_connector_->rollback(*session, created_td->tableId);
2748  } catch (...) {
2749  // eat it
2750  }
2751  throw;
2752  }
2753  start_row += num_rows_to_process;
2754  num_rows_to_process = std::min(num_rows - start_row, max_number_of_rows_per_package);
2755  }
2756 
2757  if (!is_temporary) {
2758  if (created_td->nShards) {
2759  const auto shard_tables = catalog.getPhysicalTablesDescriptors(created_td);
2760  for (const auto ptd : shard_tables) {
2761  leafs_connector_->checkpoint(*session, ptd->tableId);
2762  }
2763  }
2764  leafs_connector_->checkpoint(*session, created_td->tableId);
2765  }
2766 }
2767 
2768 void InsertIntoTableAsSelectStmt::execute(const Catalog_Namespace::SessionInfo& session) {
2769  auto session_copy = session;
2770  auto session_ptr = std::shared_ptr<Catalog_Namespace::SessionInfo>(
2771  &session_copy, boost::null_deleter());
2772  auto query_state = query_state::QueryState::create(session_ptr, select_query_);
2773  auto stdlog = STDLOG(query_state);
2774  populateData(query_state->createQueryStateProxy(), false, true);
2775 }
2776 
2777 void CreateTableAsSelectStmt::execute(const Catalog_Namespace::SessionInfo& session) {
2778  auto session_copy = session;
2779  auto session_ptr = std::shared_ptr<Catalog_Namespace::SessionInfo>(
2780  &session_copy, boost::null_deleter());
2781  auto query_state = query_state::QueryState::create(session_ptr, select_query_);
2782  auto stdlog = STDLOG(query_state);
2783 
2784  LocalConnector local_connector;
2785  auto& catalog = session.getCatalog();
2786  bool create_table = nullptr == leafs_connector_;
2787 
2788  if (create_table) {
2789  // check access privileges
2792  throw std::runtime_error("CTAS failed. Table " + table_name_ +
2793  " will not be created. User has no create privileges.");
2794  }
2795 
2796  if (catalog.getMetadataForTable(table_name_) != nullptr) {
2797  if (if_not_exists_) {
2798  return;
2799  }
2800  throw std::runtime_error("Table " + table_name_ +
2801  " already exists and no data was loaded.");
2802  }
2803 
2804  // only validate the select query so we get the target types
2805  // correctly, but do not populate the result set
2806  auto result =
2807  local_connector.query(query_state->createQueryStateProxy(), select_query_, true);
2808  const auto column_descriptors_for_create =
2809  local_connector.getColumnDescriptors(result, true);
2810 
2811  // some validation as the QE might return some out of range column types
2812  for (auto& cd : column_descriptors_for_create) {
2813  if (cd.columnType.is_decimal() && cd.columnType.get_precision() > 18) {
2814  throw std::runtime_error(cd.columnName + ": Precision too high, max 18.");
2815  }
2816  }
2817 
2818  TableDescriptor td;
2819  td.tableName = table_name_;
2820  td.userId = session.get_currentUser().userId;
2821  td.nColumns = column_descriptors_for_create.size();
2822  td.isView = false;
2823  td.fragmenter = nullptr;
2829  td.keyMetainfo = "[]";
2830  if (is_temporary_) {
2832  } else {
2834  }
2835 
2836  if (!storage_options_.empty()) {
2837  for (auto& p : storage_options_) {
2838  get_table_definitions(td, p, column_descriptors_for_create);
2839  }
2840  }
2841 
2842  catalog.createTable(td, column_descriptors_for_create, {}, true);
2843  // TODO (max): It's transactionally unsafe, should be fixed: we may create object
2844  // w/o privileges
2845  SysCatalog::instance().createDBObject(
2846  session.get_currentUser(), td.tableName, TableDBObjectType, catalog);
2847  }
2848 
2849  try {
2850  populateData(query_state->createQueryStateProxy(), is_temporary_, false);
2851  } catch (...) {
2852  if (!g_cluster) {
2853  const TableDescriptor* created_td = catalog.getMetadataForTable(table_name_);
2854  if (created_td) {
2855  catalog.dropTable(created_td);
2856  }
2857  }
2858  throw;
2859  }
2860 }
2861 
2862 void DropTableStmt::execute(const Catalog_Namespace::SessionInfo& session) {
2863  auto& catalog = session.getCatalog();
2864 
2865  const TableDescriptor* td = catalog.getMetadataForTable(*table, false);
2866  if (td == nullptr) {
2867  if (if_exists) {
2868  return;
2869  }
2870  throw std::runtime_error("Table " + *table + " does not exist.");
2871  }
2872 
2873  // check access privileges
2874  if (!session.checkDBAccessPrivileges(
2876  throw std::runtime_error("Table " + *table +
2877  " will not be dropped. User has no proper privileges.");
2878  }
2879 
2880  if (td->isView) {
2881  throw std::runtime_error(*table + " is a view. Use DROP VIEW.");
2882  }
2883 
2884  auto chkptlLock = getTableLock<mapd_shared_mutex, mapd_unique_lock>(
2885  catalog, *table, LockType::CheckpointLock);
2886  auto table_write_lock = TableLockMgr::getWriteLockForTable(catalog, *table);
2887  catalog.dropTable(td);
2888 }
2889 
2890 void TruncateTableStmt::execute(const Catalog_Namespace::SessionInfo& session) {
2891  auto& catalog = session.getCatalog();
2892  const TableDescriptor* td = catalog.getMetadataForTable(*table);
2893  if (td == nullptr) {
2894  throw std::runtime_error("Table " + *table + " does not exist.");
2895  }
2896 
2897  // check access privileges
2898  std::vector<DBObject> privObjects;
2899  DBObject dbObject(*table, TableDBObjectType);
2900  dbObject.loadKey(catalog);
2902  privObjects.push_back(dbObject);
2903  if (!SysCatalog::instance().checkPrivileges(session.get_currentUser(), privObjects)) {
2904  throw std::runtime_error("Table " + *table + " will not be truncated. User " +
2905  session.get_currentUser().userName +
2906  " has no proper privileges.");
2907  }
2908 
2909  if (td->isView) {
2910  throw std::runtime_error(*table + " is a view. Cannot Truncate.");
2911  }
2912  catalog.truncateTable(td);
2913 }
2914 
2916  const TableDescriptor* td) {
2917  if (session.get_currentUser().isSuper ||
2918  session.get_currentUser().userId == td->userId) {
2919  return;
2920  }
2921  std::vector<DBObject> privObjects;
2922  DBObject dbObject(td->tableName, TableDBObjectType);
2923  dbObject.loadKey(session.getCatalog());
2925  privObjects.push_back(dbObject);
2926  if (!SysCatalog::instance().checkPrivileges(session.get_currentUser(), privObjects)) {
2927  throw std::runtime_error("Current user does not have the privilege to alter table: " +
2928  td->tableName);
2929  }
2930 }
2931 
2932 void RenameUserStmt::execute(const Catalog_Namespace::SessionInfo& session) {
2933  if (!session.get_currentUser().isSuper) {
2934  throw std::runtime_error("Only a super user can rename users.");
2935  }
2936 
2938  if (!SysCatalog::instance().getMetadataForUser(*username_, user)) {
2939  throw std::runtime_error("User " + *username_ + " does not exist.");
2940  }
2941 
2942  SysCatalog::instance().renameUser(*username_, *new_username_);
2943 }
2944 
2945 void RenameDatabaseStmt::execute(const Catalog_Namespace::SessionInfo& session) {
2947  if (!SysCatalog::instance().getMetadataForDB(*database_name_, db)) {
2948  throw std::runtime_error("Database " + *database_name_ + " does not exist.");
2949  }
2950 
2951  if (!session.get_currentUser().isSuper &&
2952  session.get_currentUser().userId != db.dbOwner) {
2953  throw std::runtime_error("Only a super user or the owner can rename the database.");
2954  }
2955 
2956  SysCatalog::instance().renameDatabase(*database_name_, *new_database_name_);
2957 }
2958 
2959 void RenameTableStmt::execute(const Catalog_Namespace::SessionInfo& session) {
2960  auto& catalog = session.getCatalog();
2961  const TableDescriptor* td = catalog.getMetadataForTable(*table);
2962  if (td == nullptr) {
2963  throw std::runtime_error("Table " + *table + " does not exist.");
2964  }
2965  check_alter_table_privilege(session, td);
2966  if (catalog.getMetadataForTable(*new_table_name) != nullptr) {
2967  throw std::runtime_error("Table or View " + *new_table_name + " already exists.");
2968  }
2969  catalog.renameTable(td, *new_table_name);
2970 }
2971 
2972 void DDLStmt::setColumnDescriptor(ColumnDescriptor& cd, const ColumnDef* coldef) {
2973  cd.columnName = *coldef->get_column_name();
2974  SQLType* t = coldef->get_column_type();
2975  t->check_type();
2976  if (t->get_is_array()) {
2978  cd.columnType.set_subtype(t->get_type());
2979  } else {
2980  cd.columnType.set_type(t->get_type());
2981  }
2982  if (IS_GEO(t->get_type())) {
2983  cd.columnType.set_subtype(static_cast<SQLTypes>(t->get_param1()));
2986  } else {
2988  cd.columnType.set_scale(t->get_param2());
2989  }
2991  const ColumnConstraintDef* cc = coldef->get_column_constraint();
2992  if (cc == nullptr) {
2993  cd.columnType.set_notnull(false);
2994  } else {
2996  }
2997  const CompressDef* compression = coldef->get_compression();
2998  if (compression == nullptr) {
2999  // Change default TEXT column behaviour to be DICT encoded
3000  if (cd.columnType.is_string() || cd.columnType.is_string_array()) {
3001  // default to 32-bits
3003  cd.columnType.set_comp_param(32);
3004  } else if (cd.columnType.is_decimal() && cd.columnType.get_precision() <= 4) {
3006  cd.columnType.set_comp_param(16);
3007  } else if (cd.columnType.is_decimal() && cd.columnType.get_precision() <= 9) {
3009  cd.columnType.set_comp_param(32);
3010  } else if (cd.columnType.is_decimal() && cd.columnType.get_precision() > 18) {
3011  throw std::runtime_error(cd.columnName + ": Precision too high, max 18.");
3012  } else if (cd.columnType.is_geometry() && cd.columnType.get_output_srid() == 4326) {
3013  // default to GEOINT 32-bits
3015  cd.columnType.set_comp_param(32);
3016  } else if (type == kDATE && g_use_date_in_days_default_encoding) {
3017  // Days encoding for DATE
3019  cd.columnType.set_comp_param(0);
3020  } else {
3022  cd.columnType.set_comp_param(0);
3023  }
3024  } else {
3025  const std::string& comp = *compression->get_encoding_name();
3026  int comp_param;
3027  if (boost::iequals(comp, "fixed")) {
3028  if (!cd.columnType.is_integer() && !cd.columnType.is_time() &&
3029  !cd.columnType.is_decimal()) {
3030  throw std::runtime_error(
3031  cd.columnName +
3032  ": Fixed encoding is only supported for integer or time columns.");
3033  }
3034  // fixed-bits encoding
3035  if (type == kARRAY) {
3036  type = cd.columnType.get_subtype();
3037  }
3038  switch (type) {
3039  case kSMALLINT:
3040  if (compression->get_encoding_param() != 8) {
3041  throw std::runtime_error(
3042  cd.columnName +
3043  ": Compression parameter for Fixed encoding on SMALLINT must be 8.");
3044  }
3045  break;
3046  case kINT:
3047  if (compression->get_encoding_param() != 8 &&
3048  compression->get_encoding_param() != 16) {
3049  throw std::runtime_error(
3050  cd.columnName +
3051  ": Compression parameter for Fixed encoding on INTEGER must be 8 or 16.");
3052  }
3053  break;
3054  case kBIGINT:
3055  if (compression->get_encoding_param() != 8 &&
3056  compression->get_encoding_param() != 16 &&
3057  compression->get_encoding_param() != 32) {
3058  throw std::runtime_error(cd.columnName +
3059  ": Compression parameter for Fixed encoding on "
3060  "BIGINT must be 8 or 16 or 32.");
3061  }
3062  break;
3063  case kTIMESTAMP:
3064  case kTIME:
3065  if (compression->get_encoding_param() != 32) {
3066  throw std::runtime_error(cd.columnName +
3067  ": Compression parameter for Fixed encoding on "
3068  "TIME or TIMESTAMP must be 32.");
3069  } else if (cd.columnType.is_high_precision_timestamp()) {
3070  throw std::runtime_error(
3071  "Fixed encoding is not supported for TIMESTAMP(3|6|9).");
3072  }
3073  break;
3074  case kDECIMAL:
3075  case kNUMERIC:
3076  if (compression->get_encoding_param() != 32 &&
3077  compression->get_encoding_param() != 16) {
3078  throw std::runtime_error(cd.columnName +
3079  ": Compression parameter for Fixed encoding on "
3080  "DECIMAL must be 16 or 32.");
3081  }
3082 
3083  if (compression->get_encoding_param() == 32 &&
3084  cd.columnType.get_precision() > 9) {
3085  throw std::runtime_error(
3086  cd.columnName + ": Precision too high for Fixed(32) encoding, max 9.");
3087  }
3088 
3089  if (compression->get_encoding_param() == 16 &&
3090  cd.columnType.get_precision() > 4) {
3091  throw std::runtime_error(
3092  cd.columnName + ": Precision too high for Fixed(16) encoding, max 4.");
3093  }
3094  break;
3095  case kDATE:
3096  if (compression->get_encoding_param() != 32 &&
3097  compression->get_encoding_param() != 16) {
3098  throw std::runtime_error(cd.columnName +
3099  ": Compression parameter for Fixed encoding on "
3100  "DATE must be 16 or 32.");
3101  }
3102  break;
3103  default:
3104  throw std::runtime_error(cd.columnName + ": Cannot apply FIXED encoding to " +
3105  t->to_string());
3106  }
3107  if (type == kDATE) {
3109  cd.columnType.set_comp_param(16);
3110  } else {
3112  cd.columnType.set_comp_param(compression->get_encoding_param());
3113  }
3114  } else if (boost::iequals(comp, "rl")) {
3115  // run length encoding
3117  cd.columnType.set_comp_param(0);
3118  // throw std::runtime_error("RL(Run Length) encoding not supported yet.");
3119  } else if (boost::iequals(comp, "diff")) {
3120  // differential encoding
3122  cd.columnType.set_comp_param(0);
3123  // throw std::runtime_error("DIFF(differential) encoding not supported yet.");
3124  } else if (boost::iequals(comp, "dict")) {
3125  if (!cd.columnType.is_string() && !cd.columnType.is_string_array()) {
3126  throw std::runtime_error(
3127  cd.columnName +
3128  ": Dictionary encoding is only supported on string or string array columns.");
3129  }
3130  if (compression->get_encoding_param() == 0) {
3131  comp_param = 32; // default to 32-bits
3132  } else {
3133  comp_param = compression->get_encoding_param();
3134  }
3135  if (cd.columnType.is_string_array() && comp_param != 32) {
3136  throw std::runtime_error(cd.columnName +
3137  ": Compression parameter for string arrays must be 32");
3138  }
3139  if (comp_param != 8 && comp_param != 16 && comp_param != 32) {
3140  throw std::runtime_error(
3141  cd.columnName +
3142  ": Compression parameter for Dictionary encoding must be 8 or 16 or 32.");
3143  }
3144  // diciontary encoding
3146  cd.columnType.set_comp_param(comp_param);
3147  } else if (boost::iequals(comp, "NONE")) {
3148  if (cd.columnType.is_geometry()) {
3150  cd.columnType.set_comp_param(64);
3151  } else {
3152  if (!cd.columnType.is_string() && !cd.columnType.is_string_array()) {
3153  throw std::runtime_error(
3154  cd.columnName +
3155  ": None encoding is only supported on string or string array columns.");
3156  }
3158  cd.columnType.set_comp_param(0);
3159  }
3160  } else if (boost::iequals(comp, "sparse")) {
3161  // sparse column encoding with mostly NULL values
3162  if (cd.columnType.get_notnull()) {
3163  throw std::runtime_error(
3164  cd.columnName + ": Cannot do sparse column encoding on a NOT NULL column.");
3165  }
3166  if (compression->get_encoding_param() == 0 ||
3167  compression->get_encoding_param() % 8 != 0 ||
3168  compression->get_encoding_param() > 48) {
3169  throw std::runtime_error(
3170  cd.columnName +
3171  "Must specify number of bits as 8, 16, 24, 32 or 48 as the parameter to "
3172  "sparse-column encoding.");
3173  }
3175  cd.columnType.set_comp_param(compression->get_encoding_param());
3176  // throw std::runtime_error("SPARSE encoding not supported yet.");
3177  } else if (boost::iequals(comp, "compressed")) {
3178  if (!cd.columnType.is_geometry() || cd.columnType.get_output_srid() != 4326) {
3179  throw std::runtime_error(
3180  cd.columnName +
3181  ": COMPRESSED encoding is only supported on WGS84 geo columns.");
3182  }
3183  if (compression->get_encoding_param() == 0) {
3184  comp_param = 32; // default to 32-bits
3185  } else {
3186  comp_param = compression->get_encoding_param();
3187  }
3188  if (comp_param != 32) {
3189  throw std::runtime_error(cd.columnName +
3190  ": only 32-bit COMPRESSED geo encoding is supported");
3191  }
3192  // encoding longitude/latitude as integers
3194  cd.columnType.set_comp_param(comp_param);
3195  } else if (boost::iequals(comp, "days")) {
3196  // days encoding for dates
3197  if (cd.columnType.get_type() != kDATE) {
3198  throw std::runtime_error(cd.columnName +
3199  ": Days encoding is only supported for DATE columns.");
3200  }
3201  if (compression->get_encoding_param() != 32 &&
3202  compression->get_encoding_param() != 16) {
3203  throw std::runtime_error(cd.columnName +
3204  ": Compression parameter for Days encoding on "
3205  "DATE must be 16 or 32.");
3206  }
3208  cd.columnType.set_comp_param((compression->get_encoding_param() == 16) ? 16 : 0);
3209  } else {
3210  throw std::runtime_error(cd.columnName + ": Invalid column compression scheme " +
3211  comp);
3212  }
3213  }
3214  if (cd.columnType.is_string_array() &&
3216  throw std::runtime_error(
3217  cd.columnName +
3218  ": Array of strings must be dictionary encoded. Specify ENCODING DICT");
3219  }
3220  if (t->get_is_array()) {
3221  int s = -1;
3222  auto array_size = t->get_array_size();
3223  if (array_size > 0) {
3224  auto sti = cd.columnType.get_elem_type();
3225  s = array_size * sti.get_size();
3226  if (s <= 0) {
3227  throw std::runtime_error(cd.columnName + ": Unexpected fixed length array size");
3228  }
3229  }
3230  cd.columnType.set_size(s);
3231 
3232  } else {
3234  }
3235  cd.isSystemCol = false;
3236  cd.isVirtualCol = false;
3237 }
3238 
3239 void AddColumnStmt::check_executable(const Catalog_Namespace::SessionInfo& session) {
3240  auto& catalog = session.getCatalog();
3241  const TableDescriptor* td = catalog.getMetadataForTable(*table);
3242  if (nullptr == td) {
3243  throw std::runtime_error("Table " + *table + " does not exist.");
3244  } else {
3245  if (td->isView) {
3246  throw std::runtime_error("Expecting a table , found view " + *table);
3247  }
3248  };
3249 
3250  check_alter_table_privilege(session, td);
3251 
3252  if (0 == coldefs.size()) {
3253  coldefs.push_back(std::move(coldef));
3254  }
3255 
3256  for (const auto& coldef : coldefs) {
3257  auto& new_column_name = *coldef->get_column_name();
3258  if (catalog.getMetadataForColumn(td->tableId, new_column_name) != nullptr) {
3259  throw std::runtime_error("Column " + new_column_name + " already exists.");
3260  }
3261  if (reserved_keywords.find(boost::to_upper_copy<std::string>(new_column_name)) !=
3262  reserved_keywords.end()) {
3263  throw std::runtime_error("Cannot add column with reserved keyword '" +
3264  new_column_name + "'");
3265  }
3266  }
3267 }
3268 
3269 void AddColumnStmt::execute(const Catalog_Namespace::SessionInfo& session) {
3270  auto& catalog = session.getCatalog();
3271  const TableDescriptor* td = catalog.getMetadataForTable(*table);
3272  check_executable(session);
3273 
3274  catalog.getSqliteConnector().query("BEGIN TRANSACTION");
3275  try {
3276  std::map<const std::string, const ColumnDescriptor> cds;
3277  std::map<const int, const ColumnDef*> cid_coldefs;
3278  for (const auto& coldef : coldefs) {
3279  ColumnDescriptor cd;
3280  setColumnDescriptor(cd, coldef.get());
3281  catalog.addColumn(*td, cd);
3282  cds.emplace(*coldef->get_column_name(), cd);
3283  cid_coldefs.emplace(cd.columnId, coldef.get());
3284 
3285  // expand geo column to phy columns
3286  if (cd.columnType.is_geometry()) {
3287  std::list<ColumnDescriptor> phy_geo_columns;
3288  catalog.expandGeoColumn(cd, phy_geo_columns);
3289  for (auto& cd : phy_geo_columns) {
3290  catalog.addColumn(*td, cd);
3291  cds.emplace(cd.columnName, cd);
3292  cid_coldefs.emplace(cd.columnId, nullptr);
3293  }
3294  }
3295  }
3296 
3297  std::unique_ptr<Importer_NS::Loader> loader(new Importer_NS::Loader(catalog, td));
3298  std::vector<std::unique_ptr<Importer_NS::TypedImportBuffer>> import_buffers;
3299 
3300  // a call of Catalog_Namespace::MapDHandler::prepare_columnar_loader
3302  td->tableName,
3303  td->nColumns - 1,
3304  &loader,
3305  &import_buffers);
3306  loader->setReplicating(true);
3307 
3308  // set_geo_physical_import_buffer below needs a sorted import_buffers
3309  std::sort(import_buffers.begin(),
3310  import_buffers.end(),
3311  [](decltype(import_buffers[0])& a, decltype(import_buffers[0])& b) {
3312  return a->getColumnDesc()->columnId < b->getColumnDesc()->columnId;
3313  });
3314 
3315  size_t nrows = td->fragmenter->getNumRows();
3316  // if sharded, get total nrows from all sharded tables
3317  if (td->nShards > 0) {
3318  const auto physical_tds = catalog.getPhysicalTablesDescriptors(td);
3319  nrows = 0;
3320  std::for_each(physical_tds.begin(), physical_tds.end(), [&nrows](const auto& td) {
3321  nrows += td->fragmenter->getNumRows();
3322  });
3323  }
3324  if (nrows > 0) {
3325  int skip_physical_cols = 0;
3326  for (const auto cit : cid_coldefs) {
3327  const auto cd = catalog.getMetadataForColumn(td->tableId, cit.first);
3328  const auto coldef = cit.second;
3329  const auto column_constraint = coldef ? coldef->get_column_constraint() : nullptr;
3330  std::string defaultval = "";
3331  if (column_constraint) {
3332  auto defaultlp = column_constraint->get_defaultval();
3333  auto defaultsp = dynamic_cast<const StringLiteral*>(defaultlp);
3334  defaultval = defaultsp ? *defaultsp->get_stringval()
3335  : defaultlp ? defaultlp->to_string() : "";
3336  }
3337  bool isnull = column_constraint ? (0 == defaultval.size()) : true;
3338  if (boost::to_upper_copy<std::string>(defaultval) == "NULL") {
3339  isnull = true;
3340  }
3341 
3342  // TODO: remove is_geometry below once if null is allowed for geo
3343  if (isnull) {
3344  if (cd->columnType.is_geometry() ||
3345  (column_constraint && column_constraint->get_notnull())) {
3346  throw std::runtime_error("Default value required for column " +
3347  cd->columnName + " (NULL value not supported)");
3348  }
3349  }
3350 
3351  for (auto it = import_buffers.begin(); it < import_buffers.end(); ++it) {
3352  auto& import_buffer = *it;
3353  if (cd->columnId == import_buffer->getColumnDesc()->columnId) {
3354  if (coldef != nullptr ||
3355  skip_physical_cols-- <= 0) { // skip non-null phy col
3356  import_buffer->add_value(
3357  cd, defaultval, isnull, Importer_NS::CopyParams(), nrows);
3358  // tedious non-null geo default value ...
3359  if (cd->columnType.is_geometry() && !isnull) {
3360  std::vector<double> coords, bounds;
3361  std::vector<int> ring_sizes, poly_rings;
3362  int render_group = 0;
3363  SQLTypeInfo tinfo;
3365  tinfo,
3366  coords,
3367  bounds,
3368  ring_sizes,
3369  poly_rings,
3370  false)) {
3371  throw std::runtime_error("Bad geometry data: '" + defaultval + "'");
3372  }
3373  size_t col_idx = 1 + std::distance(import_buffers.begin(), it);
3375  cd,
3376  import_buffers,
3377  col_idx,
3378  coords,
3379  bounds,
3380  ring_sizes,
3381  poly_rings,
3382  render_group,
3383  nrows);
3384  // skip following phy cols
3385  skip_physical_cols = cd->columnType.get_physical_cols();
3386  }
3387  }
3388  break;
3389  }
3390  }
3391  }
3392  }
3393 
3394  if (!loader->loadNoCheckpoint(import_buffers, nrows)) {
3395  throw std::runtime_error("loadNoCheckpoint failed!");
3396  }
3397  loader->checkpoint();
3398  } catch (...) {
3399  catalog.roll(false);
3400  catalog.getSqliteConnector().query("ROLLBACK TRANSACTION");
3401  throw;
3402  }
3403  catalog.getSqliteConnector().query("END TRANSACTION");
3404  catalog.roll(true);
3405 }
3406 
3407 void RenameColumnStmt::execute(const Catalog_Namespace::SessionInfo& session) {
3408  auto& catalog = session.getCatalog();
3409  const TableDescriptor* td = catalog.getMetadataForTable(*table);
3410  if (td == nullptr) {
3411  throw std::runtime_error("Table " + *table + " does not exist.");
3412  }
3413  check_alter_table_privilege(session, td);
3414  const ColumnDescriptor* cd = catalog.getMetadataForColumn(td->tableId, *column);
3415  if (cd == nullptr) {
3416  throw std::runtime_error("Column " + *column + " does not exist.");
3417  }
3418  if (catalog.getMetadataForColumn(td->tableId, *new_column_name) != nullptr) {
3419  throw std::runtime_error("Column " + *new_column_name + " already exists.");
3420  }
3421  if (reserved_keywords.find(boost::to_upper_copy<std::string>(*new_column_name)) !=
3422  reserved_keywords.end()) {
3423  throw std::runtime_error("Cannot create column with reserved keyword '" +
3424  *new_column_name + "'");
3425  }
3426  catalog.renameColumn(td, cd, *new_column_name);
3427 }
3428 
3429 void CopyTableStmt::execute(const Catalog_Namespace::SessionInfo& session) {
3430  auto importer_factory = [](Catalog_Namespace::Catalog& catalog,
3431  const TableDescriptor* td,
3432  const std::string& file_path,
3433  const Importer_NS::CopyParams& copy_params) {
3434  return boost::make_unique<Importer_NS::Importer>(catalog, td, file_path, copy_params);
3435  };
3436  return execute(session, importer_factory);
3437 }
3438 
3439 void CopyTableStmt::execute(const Catalog_Namespace::SessionInfo& session,
3440  const std::function<std::unique_ptr<Importer_NS::Importer>(
3442  const TableDescriptor*,
3443  const std::string&,
3444  const Importer_NS::CopyParams&)>& importer_factory) {
3445  size_t rows_completed = 0;
3446  size_t rows_rejected = 0;
3447  size_t total_time = 0;
3448  bool load_truncated = false;
3449 
3450  auto& catalog = session.getCatalog();
3451  const TableDescriptor* td = catalog.getMetadataForTable(*table);
3452 
3453  // if the table already exists, it's locked, so check access privileges
3454  if (td) {
3455  std::vector<DBObject> privObjects;
3456  DBObject dbObject(*table, TableDBObjectType);
3457  dbObject.loadKey(catalog);
3459  privObjects.push_back(dbObject);
3460  if (!SysCatalog::instance().checkPrivileges(session.get_currentUser(), privObjects)) {
3461  throw std::runtime_error("Violation of access privileges: user " +
3462  session.get_currentUser().userName +
3463  " has no insert privileges for table " + *table + ".");
3464  }
3465  }
3466 
3467  // since we'll have not only posix file names but also s3/hdfs/... url
3468  // we do not expand wildcard or check file existence here.
3469  // from here on, file_path contains something which may be a url
3470  // or a wildcard of file names;
3471  std::string file_path = *file_pattern;
3472  Importer_NS::CopyParams copy_params;
3473  if (!options.empty()) {
3474  for (auto& p : options) {
3475  if (boost::iequals(*p->get_name(), "max_reject")) {
3476  const IntLiteral* int_literal = dynamic_cast<const IntLiteral*>(p->get_value());
3477  if (int_literal == nullptr) {
3478  throw std::runtime_error("max_reject option must be an integer.");
3479  }
3480  copy_params.max_reject = int_literal->get_intval();
3481  } else if (boost::iequals(*p->get_name(), "buffer_size")) {
3482  const IntLiteral* int_literal = dynamic_cast<const IntLiteral*>(p->get_value());
3483  if (int_literal == nullptr) {
3484  throw std::runtime_error("buffer_size option must be an integer.");
3485  }
3486  copy_params.buffer_size = std::max<size_t>(1 << 20, int_literal->get_intval());
3487  } else if (boost::iequals(*p->get_name(), "threads")) {
3488  const IntLiteral* int_literal = dynamic_cast<const IntLiteral*>(p->get_value());
3489  if (int_literal == nullptr) {
3490  throw std::runtime_error("Threads option must be an integer.");
3491  }
3492  copy_params.threads = int_literal->get_intval();
3493  } else if (boost::iequals(*p->get_name(), "delimiter")) {
3494  const StringLiteral* str_literal =
3495  dynamic_cast<const StringLiteral*>(p->get_value());
3496  if (str_literal == nullptr) {
3497  throw std::runtime_error("Delimiter option must be a string.");
3498  } else if (str_literal->get_stringval()->length() != 1) {
3499  throw std::runtime_error("Delimiter must be a single character string.");
3500  }
3501  copy_params.delimiter = (*str_literal->get_stringval())[0];
3502  } else if (boost::iequals(*p->get_name(), "nulls")) {
3503  const StringLiteral* str_literal =
3504  dynamic_cast<const StringLiteral*>(p->get_value());
3505  if (str_literal == nullptr) {
3506  throw std::runtime_error("Nulls option must be a string.");
3507  }
3508  copy_params.null_str = *str_literal->get_stringval();
3509  } else if (boost::iequals(*p->get_name(), "header")) {
3510  const StringLiteral* str_literal =
3511  dynamic_cast<const StringLiteral*>(p->get_value());
3512  if (str_literal == nullptr) {
3513  throw std::runtime_error("Header option must be a boolean.");
3514  }
3515  copy_params.has_header = bool_from_string_literal(str_literal)
3518 #ifdef ENABLE_IMPORT_PARQUET
3519  } else if (boost::iequals(*p->get_name(), "parquet")) {
3520  const StringLiteral* str_literal =
3521  dynamic_cast<const StringLiteral*>(p->get_value());
3522  if (str_literal == nullptr) {
3523  throw std::runtime_error("Parquet option must be a boolean.");
3524  }
3525  if (bool_from_string_literal(str_literal)) {
3526  // not sure a parquet "table" type is proper, but to make code
3527  // look consistent in some places, let's set "table" type too
3528  copy_params.file_type = Importer_NS::FileType::PARQUET;
3529  }
3530 #endif // ENABLE_IMPORT_PARQUET
3531  } else if (boost::iequals(*p->get_name(), "s3_access_key")) {
3532  const StringLiteral* str_literal =
3533  dynamic_cast<const StringLiteral*>(p->get_value());
3534  if (str_literal == nullptr) {
3535  throw std::runtime_error("Option s3_access_key must be a string.");
3536  }
3537  copy_params.s3_access_key = *str_literal->get_stringval();
3538  } else if (boost::iequals(*p->get_name(), "s3_secret_key")) {
3539  const StringLiteral* str_literal =
3540  dynamic_cast<const StringLiteral*>(p->get_value());
3541  if (str_literal == nullptr) {
3542  throw std::runtime_error("Option s3_secret_key must be a string.");
3543  }
3544  copy_params.s3_secret_key = *str_literal->get_stringval();
3545  } else if (boost::iequals(*p->get_name(), "s3_region")) {
3546  const StringLiteral* str_literal =
3547  dynamic_cast<const StringLiteral*>(p->get_value());
3548  if (str_literal == nullptr) {
3549  throw std::runtime_error("Option s3_region must be a string.");
3550  }
3551  copy_params.s3_region = *str_literal->get_stringval();
3552  } else if (boost::iequals(*p->get_name(), "s3_endpoint")) {
3553  const StringLiteral* str_literal =
3554  dynamic_cast<const StringLiteral*>(p->get_value());
3555  if (str_literal == nullptr) {
3556  throw std::runtime_error("Option s3_endpoint must be a string.");
3557  }
3558  copy_params.s3_endpoint = *str_literal->get_stringval();
3559  } else if (boost::iequals(*p->get_name(), "quote")) {
3560  const StringLiteral* str_literal =
3561  dynamic_cast<const StringLiteral*>(p->get_value());
3562  if (str_literal == nullptr) {
3563  throw std::runtime_error("Quote option must be a string.");
3564  } else if (str_literal->get_stringval()->length() != 1) {
3565  throw std::runtime_error("Quote must be a single character string.");
3566  }
3567  copy_params.quote = (*str_literal->get_stringval())[0];
3568  } else if (boost::iequals(*p->get_name(), "escape")) {
3569  const StringLiteral* str_literal =
3570  dynamic_cast<const StringLiteral*>(p->get_value());
3571  if (str_literal == nullptr) {
3572  throw std::runtime_error("Escape option must be a string.");
3573  } else if (str_literal->get_stringval()->length() != 1) {
3574  throw std::runtime_error("Escape must be a single character string.");
3575  }
3576  copy_params.escape = (*str_literal->get_stringval())[0];
3577  } else if (boost::iequals(*p->get_name(), "line_delimiter")) {
3578  const StringLiteral* str_literal =
3579  dynamic_cast<const StringLiteral*>(p->get_value());
3580  if (str_literal == nullptr) {
3581  throw std::runtime_error("Line_delimiter option must be a string.");
3582  } else if (str_literal->get_stringval()->length() != 1) {
3583  throw std::runtime_error("Line_delimiter must be a single character string.");
3584  }
3585  copy_params.line_delim = (*str_literal->get_stringval())[0];
3586  } else if (boost::iequals(*p->get_name(), "quoted")) {
3587  const StringLiteral* str_literal =
3588  dynamic_cast<const StringLiteral*>(p->get_value());
3589  if (str_literal == nullptr) {
3590  throw std::runtime_error("Quoted option must be a boolean.");
3591  }
3592  copy_params.quoted = bool_from_string_literal(str_literal);
3593  } else if (boost::iequals(*p->get_name(), "plain_text")) {
3594  const StringLiteral* str_literal =
3595  dynamic_cast<const StringLiteral*>(p->get_value());
3596  if (str_literal == nullptr) {
3597  throw std::runtime_error("plain_text option must be a boolean.");
3598  }
3599  copy_params.plain_text = bool_from_string_literal(str_literal);
3600  } else if (boost::iequals(*p->get_name(), "array_marker")) {
3601  const StringLiteral* str_literal =
3602  dynamic_cast<const StringLiteral*>(p->get_value());
3603  if (str_literal == nullptr) {
3604  throw std::runtime_error("Array Marker option must be a string.");
3605  } else if (str_literal->get_stringval()->length() != 2) {
3606  throw std::runtime_error(
3607  "Array Marker option must be exactly two characters. Default is {}.");
3608  }
3609  copy_params.array_begin = (*str_literal->get_stringval())[0];
3610  copy_params.array_end = (*str_literal->get_stringval())[1];
3611  } else if (boost::iequals(*p->get_name(), "array_delimiter")) {
3612  const StringLiteral* str_literal =
3613  dynamic_cast<const StringLiteral*>(p->get_value());
3614  if (str_literal == nullptr) {
3615  throw std::runtime_error("Array Delimiter option must be a string.");
3616  } else if (str_literal->get_stringval()->length() != 1) {
3617  throw std::runtime_error("Array Delimiter must be a single character string.");
3618  }
3619  copy_params.array_delim = (*str_literal->get_stringval())[0];
3620  } else if (boost::iequals(*p->get_name(), "lonlat")) {
3621  const StringLiteral* str_literal =
3622  dynamic_cast<const StringLiteral*>(p->get_value());
3623  if (str_literal == nullptr) {
3624  throw std::runtime_error("Lonlat option must be a boolean.");
3625  }
3626  copy_params.lonlat = bool_from_string_literal(str_literal);
3627  } else if (boost::iequals(*p->get_name(), "geo")) {
3628  const StringLiteral* str_literal =
3629  dynamic_cast<const StringLiteral*>(p->get_value());
3630  if (str_literal == nullptr) {
3631  throw std::runtime_error("Geo option must be a boolean.");
3632  }
3633  copy_params.file_type = bool_from_string_literal(str_literal)
3636  } else if (boost::iequals(*p->get_name(), "geo_coords_type")) {
3637  const StringLiteral* str_literal =
3638  dynamic_cast<const StringLiteral*>(p->get_value());
3639  if (str_literal == nullptr) {
3640  throw std::runtime_error("'geo_coords_type' option must be a string");
3641  }
3642  const std::string* s = str_literal->get_stringval();
3643  if (boost::iequals(*s, "geography")) {
3644  throw std::runtime_error(
3645  "GEOGRAPHY coords type not yet supported. Please use GEOMETRY.");
3646  // copy_params.geo_coords_type = kGEOGRAPHY;
3647  } else if (boost::iequals(*s, "geometry")) {
3648  copy_params.geo_coords_type = kGEOMETRY;
3649  } else {
3650  throw std::runtime_error(
3651  "Invalid string for 'geo_coords_type' option (must be 'GEOGRAPHY' or "
3652  "'GEOMETRY'): " +
3653  *s);
3654  }
3655  } else if (boost::iequals(*p->get_name(), "geo_coords_encoding")) {
3656  const StringLiteral* str_literal =
3657  dynamic_cast<const StringLiteral*>(p->get_value());
3658  if (str_literal == nullptr) {
3659  throw std::runtime_error("'geo_coords_encoding' option must be a string");
3660  }
3661  const std::string* s = str_literal->get_stringval();
3662  if (boost::iequals(*s, "none")) {
3663  copy_params.geo_coords_encoding = kENCODING_NONE;
3664  copy_params.geo_coords_comp_param = 0;
3665  } else if (boost::iequals(*s, "compressed(32)")) {
3666  copy_params.geo_coords_encoding = kENCODING_GEOINT;
3667  copy_params.geo_coords_comp_param = 32;
3668  } else {
3669  throw std::runtime_error(
3670  "Invalid string for 'geo_coords_encoding' option (must be 'NONE' or "
3671  "'COMPRESSED(32)'): " +
3672  *s);
3673  }
3674  } else if (boost::iequals(*p->get_name(), "geo_coords_srid")) {
3675  const IntLiteral* int_literal = dynamic_cast<const IntLiteral*>(p->get_value());
3676  if (int_literal == nullptr) {
3677  throw std::runtime_error("'geo_coords_srid' option must be an integer");
3678  }
3679  const int srid = int_literal->get_intval();
3680  if (srid == 4326 || srid == 3857 || srid == 900913) {
3681  copy_params.geo_coords_srid = srid;
3682  } else {
3683  throw std::runtime_error(
3684  "Invalid value for 'geo_coords_srid' option (must be 4326, 3857, or "
3685  "900913): " +
3686  std::to_string(srid));
3687  }
3688  } else if (boost::iequals(*p->get_name(), "geo_layer_name")) {
3689  const StringLiteral* str_literal =
3690  dynamic_cast<const StringLiteral*>(p->get_value());
3691  if (str_literal == nullptr) {
3692  throw std::runtime_error("'geo_layer_name' option must be a string");
3693  }
3694  const std::string* layer_name = str_literal->get_stringval();
3695  if (layer_name) {
3696  copy_params.geo_layer_name = *layer_name;
3697  } else {
3698  throw std::runtime_error("Invalid value for 'geo_layer_name' option");
3699  }
3700  } else if (boost::iequals(*p->get_name(), "partitions")) {
3701  if (copy_params.file_type == Importer_NS::FileType::POLYGON) {
3702  const auto partitions =
3703  static_cast<const StringLiteral*>(p->get_value())->get_stringval();
3704  CHECK(partitions);
3705  const auto partitions_uc = boost::to_upper_copy<std::string>(*partitions);
3706  if (partitions_uc != "REPLICATED") {
3707  throw std::runtime_error("PARTITIONS must be REPLICATED for geo COPY");
3708  }
3709  _geo_copy_from_partitions = partitions_uc;
3710  } else {
3711  throw std::runtime_error("PARTITIONS option not supported for non-geo COPY: " +
3712  *p->get_name());
3713  }
3714  } else if (boost::iequals(*p->get_name(), "geo_assign_render_groups")) {
3715  const StringLiteral* str_literal =
3716  dynamic_cast<const StringLiteral*>(p->get_value());
3717  if (str_literal == nullptr) {
3718  throw std::runtime_error("geo_assign_render_groups option must be a boolean.");
3719  }
3720  copy_params.geo_assign_render_groups = bool_from_string_literal(str_literal);
3721  } else if (boost::iequals(*p->get_name(), "geo_explode_collections")) {
3722  const StringLiteral* str_literal =
3723  dynamic_cast<const StringLiteral*>(p->get_value());
3724  if (str_literal == nullptr) {
3725  throw std::runtime_error("geo_explode_collections option must be a boolean.");
3726  }
3727  copy_params.geo_explode_collections = bool_from_string_literal(str_literal);
3728  } else {
3729  throw std::runtime_error("Invalid option for COPY: " + *p->get_name());
3730  }
3731  }
3732  }
3733 
3734  std::string tr;
3735  if (copy_params.file_type == Importer_NS::FileType::POLYGON) {
3736  // geo import
3737  // we do nothing here, except stash the parameters so we can
3738  // do the import when we unwind to the top of the handler
3739  _geo_copy_from_file_name = file_path;
3740  _geo_copy_from_copy_params = copy_params;
3741  _was_geo_copy_from = true;
3742 
3743  // the result string
3744  // @TODO simon.eves put something more useful in here
3745  // except we really can't because we haven't done the import yet!
3746  if (td) {
3747  tr = std::string("Appending geo to table '") + *table + std::string("'...");
3748  } else {
3749  tr = std::string("Creating table '") + *table +
3750  std::string("' and importing geo...");
3751  }
3752  } else {
3753  if (td) {
3754  // regular import
3755  auto importer = importer_factory(catalog, td, file_path, copy_params);
3756  auto ms = measure<>::execution([&]() {
3757  auto res = importer->import();
3758  rows_completed += res.rows_completed;
3759  rows_rejected += res.rows_rejected;
3760  load_truncated = res.load_truncated;
3761  });
3762  total_time += ms;
3763 
3764  // results
3765  if (load_truncated || rows_rejected > copy_params.max_reject) {
3766  LOG(ERROR) << "COPY exited early due to reject records count during multi file "
3767  "processing ";
3768  // if we have crossed the truncated load threshold
3769  load_truncated = true;
3770  }
3771  if (!load_truncated) {
3772  tr = std::string("Loaded: " + std::to_string(rows_completed) +
3773  " recs, Rejected: " + std::to_string(rows_rejected) +
3774  " recs in " + std::to_string((double)total_time / 1000.0) +
3775  " secs");
3776  } else {
3777  tr = std::string("Loader truncated due to reject count. Processed : " +
3778  std::to_string(rows_completed) + " recs, Rejected: " +
3779  std::to_string(rows_rejected) + " recs in " +
3780  std::to_string((double)total_time / 1000.0) + " secs");
3781  }
3782  } else {
3783  throw std::runtime_error("Table '" + *table + "' must exist before COPY FROM");
3784  }
3785  }
3786 
3787  return_message.reset(new std::string(tr));
3788  LOG(INFO) << tr;
3789 }
3790 
3791 // CREATE ROLE payroll_dept_role;
3792 void CreateRoleStmt::execute(const Catalog_Namespace::SessionInfo& session) {
3793  const auto& currentUser = session.get_currentUser();
3794  if (!currentUser.isSuper) {
3795  throw std::runtime_error("CREATE ROLE " + get_role() +
3796  " failed. It can only be executed by super user.");
3797  }
3798  SysCatalog::instance().createRole(get_role());
3799 }
3800 
3801 // DROP ROLE payroll_dept_role;
3802 void DropRoleStmt::execute(const Catalog_Namespace::SessionInfo& session) {
3803  const auto& currentUser = session.get_currentUser();
3804  if (!currentUser.isSuper) {
3805  throw std::runtime_error("DROP ROLE " + get_role() +
3806  " failed. It can only be executed by super user.");
3807  }
3808  auto* rl = SysCatalog::instance().getRoleGrantee(get_role());
3809  if (!rl) {
3810  throw std::runtime_error("DROP ROLE " + get_role() +
3811  " failed because role with this name does not exist.");
3812  }
3813  SysCatalog::instance().dropRole(get_role());
3814 }
3815 
3816 std::vector<std::string> splitObjectHierName(const std::string& hierName) {
3817  std::vector<std::string> componentNames;
3818  boost::split(componentNames, hierName, boost::is_any_of("."));
3819  return componentNames;
3820 }
3821 
3822 std::string extractObjectNameFromHierName(const std::string& objectHierName,
3823  const std::string& objectType,
3824  const Catalog_Namespace::Catalog& cat) {
3825  std::string objectName;
3826  std::vector<std::string> componentNames = splitObjectHierName(objectHierName);
3827  if (objectType.compare("DATABASE") == 0) {
3828  if (componentNames.size() == 1) {
3829  objectName = componentNames[0];
3830  } else {
3831  throw std::runtime_error("DB object name is not correct " + objectHierName);
3832  }
3833  } else {
3834  if (objectType.compare("TABLE") == 0 || objectType.compare("DASHBOARD") == 0 ||
3835  objectType.compare("VIEW") == 0) {
3836  switch (componentNames.size()) {
3837  case (1): {
3838  objectName = componentNames[0];
3839  break;
3840  }
3841  case (2): {
3842  objectName = componentNames[1];
3843  break;
3844  }
3845  default: {
3846  throw std::runtime_error("DB object name is not correct " + objectHierName);
3847  }
3848  }
3849  } else {
3850  throw std::runtime_error("DB object type " + objectType + " is not supported.");
3851  }
3852  }
3853  return objectName;
3854 }
3855 
3856 static std::pair<AccessPrivileges, DBObjectType> parseStringPrivs(
3857  const std::string& privs,
3858  const DBObjectType& objectType,
3859  const std::string& object_name) {
3860  static const std::map<std::pair<const std::string, const DBObjectType>,
3861  std::pair<const AccessPrivileges, const DBObjectType>>
3862  privileges_lookup{
3863  {{"ALL"s, DatabaseDBObjectType},
3866  {{"ALL"s, DashboardDBObjectType},
3869 
3870  {{"CREATE TABLE"s, DatabaseDBObjectType},
3872  {{"CREATE"s, DatabaseDBObjectType},
3874  {{"SELECT"s, DatabaseDBObjectType},
3876  {{"INSERT"s, DatabaseDBObjectType},
3878  {{"TRUNCATE"s, DatabaseDBObjectType},
3880  {{"UPDATE"s, DatabaseDBObjectType},
3882  {{"DELETE"s, DatabaseDBObjectType},
3884  {{"DROP"s, DatabaseDBObjectType},
3886  {{"ALTER"s, DatabaseDBObjectType},
3888 
3889  {{"SELECT"s, TableDBObjectType},
3891  {{"INSERT"s, TableDBObjectType},
3893  {{"TRUNCATE"s, TableDBObjectType},
3895  {{"UPDATE"s, TableDBObjectType},
3897  {{"DELETE"s, TableDBObjectType},
3899  {{"DROP"s, TableDBObjectType},
3901  {{"ALTER"s, TableDBObjectType},
3903 
3904  {{"CREATE VIEW"s, DatabaseDBObjectType},
3906  {{"SELECT VIEW"s, DatabaseDBObjectType},
3908  {{"DROP VIEW"s, DatabaseDBObjectType},
3910  {{"SELECT"s, ViewDBObjectType},
3913 
3914  {{"CREATE DASHBOARD"s, DatabaseDBObjectType},
3916  {{"EDIT DASHBOARD"s, DatabaseDBObjectType},
3918  {{"VIEW DASHBOARD"s, DatabaseDBObjectType},
3920  {{"DELETE DASHBOARD"s, DatabaseDBObjectType},
3922  {{"VIEW"s, DashboardDBObjectType},
3924  {{"EDIT"s, DashboardDBObjectType},
3926  {{"DELETE"s, DashboardDBObjectType},
3928 
3929  {{"VIEW SQL EDITOR"s, DatabaseDBObjectType},
3931  {{"ACCESS"s, DatabaseDBObjectType},
3933 
3934  auto result = privileges_lookup.find(std::make_pair(privs, objectType));
3935  if (result == privileges_lookup.end()) {
3936  throw std::runtime_error("Privileges " + privs + " on DB object " + object_name +
3937  " are not correct.");
3938  }
3939  return result->second;
3940 }
3941 
3942 static DBObject createObject(const std::string& objectName, DBObjectType objectType) {
3943  if (objectType == DashboardDBObjectType) {
3944  int32_t dashboard_id = -1;
3945  if (!objectName.empty()) {
3946  try {
3947  dashboard_id = stoi(objectName);
3948  } catch (const std::exception&) {
3949  throw std::runtime_error(
3950  "Privileges on dashboards should be changed via integer dashboard ID");
3951  }
3952  }
3953  return DBObject(dashboard_id, objectType);
3954  } else {
3955  return DBObject(objectName, objectType);
3956  }
3957 }
3958 
3959 // GRANT SELECT/INSERT/CREATE ON TABLE payroll_table TO payroll_dept_role;
3960 void GrantPrivilegesStmt::execute(const Catalog_Namespace::SessionInfo& session) {
3961  auto& catalog = session.getCatalog();
3962  const auto& currentUser = session.get_currentUser();
3963  const auto parserObjectType = boost::to_upper_copy<std::string>(get_object_type());
3964  const auto objectName =
3965  extractObjectNameFromHierName(get_object(), parserObjectType, catalog);
3966  auto objectType = DBObjectTypeFromString(parserObjectType);
3967  DBObject dbObject = createObject(objectName, objectType);
3968  /* verify object ownership if not suser */
3969  if (!currentUser.isSuper) {
3970  if (!SysCatalog::instance().verifyDBObjectOwnership(currentUser, dbObject, catalog)) {
3971  throw std::runtime_error(
3972  "GRANT failed. It can only be executed by super user or owner of the object.");
3973  }
3974  }
3975  /* set proper values of privileges & grant them to the object */
3976  std::vector<DBObject> objects(get_privs().size(), dbObject);
3977  for (size_t i = 0; i < get_privs().size(); ++i) {
3978  std::pair<AccessPrivileges, DBObjectType> priv = parseStringPrivs(
3979  boost::to_upper_copy<std::string>(get_privs()[i]), objectType, get_object());
3980  objects[i].setPrivileges(priv.first);
3981  objects[i].setPermissionType(priv.second);
3982  }
3983  SysCatalog::instance().grantDBObjectPrivilegesBatch(grantees, objects, catalog);
3984 }
3985 
3986 // REVOKE SELECT/INSERT/CREATE ON TABLE payroll_table FROM payroll_dept_role;
3987 void RevokePrivilegesStmt::execute(const Catalog_Namespace::SessionInfo& session) {
3988  auto& catalog = session.getCatalog();
3989  const auto& currentUser = session.get_currentUser();
3990  const auto parserObjectType = boost::to_upper_copy<std::string>(get_object_type());
3991  const auto objectName =
3992  extractObjectNameFromHierName(get_object(), parserObjectType, catalog);
3993  auto objectType = DBObjectTypeFromString(parserObjectType);
3994  DBObject dbObject = createObject(objectName, objectType);
3995  /* verify object ownership if not suser */
3996  if (!currentUser.isSuper) {
3997  if (!SysCatalog::instance().verifyDBObjectOwnership(currentUser, dbObject, catalog)) {
3998  throw std::runtime_error(
3999  "REVOKE failed. It can only be executed by super user or owner of the object.");
4000  }
4001  }
4002  /* set proper values of privileges & grant them to the object */
4003  std::vector<DBObject> objects(get_privs().size(), dbObject);
4004  for (size_t i = 0; i < get_privs().size(); ++i) {
4005  std::pair<AccessPrivileges, DBObjectType> priv = parseStringPrivs(
4006  boost::to_upper_copy<std::string>(get_privs()[i]), objectType, get_object());
4007  objects[i].setPrivileges(priv.first);
4008  objects[i].setPermissionType(priv.second);
4009  }
4010  SysCatalog::instance().revokeDBObjectPrivilegesBatch(grantees, objects, catalog);
4011 }
4012 
4013 // NOTE: not used currently, will we ever use it?
4014 // SHOW ON TABLE payroll_table FOR payroll_dept_role;
4015 void ShowPrivilegesStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4016  auto& catalog = session.getCatalog();
4017  const auto& currentUser = session.get_currentUser();
4018  const auto parserObjectType = boost::to_upper_copy<std::string>(get_object_type());
4019  const auto objectName =
4020  extractObjectNameFromHierName(get_object(), parserObjectType, catalog);
4021  auto objectType = DBObjectTypeFromString(parserObjectType);
4022  DBObject dbObject = createObject(objectName, objectType);
4023  /* verify object ownership if not suser */
4024  if (!currentUser.isSuper) {
4025  if (!SysCatalog::instance().verifyDBObjectOwnership(currentUser, dbObject, catalog)) {
4026  throw std::runtime_error(
4027  "SHOW ON " + get_object() + " FOR " + get_role() +
4028  " failed. It can only be executed by super user or owner of the object.");
4029  }
4030  }
4031  /* get values of privileges for the object and report them */
4032  SysCatalog::instance().getDBObjectPrivileges(get_role(), dbObject, catalog);
4033  AccessPrivileges privs = dbObject.getPrivileges();
4034  printf("\nPRIVILEGES ON %s FOR %s ARE SET AS FOLLOWING: ",
4035  get_object().c_str(),
4036  get_role().c_str());
4037 
4038  if (objectType == DBObjectType::DatabaseDBObjectType) {
4040  printf(" CREATE");
4041  }
4043  printf(" DROP");
4044  }
4045  } else if (objectType == DBObjectType::TableDBObjectType) {
4047  printf(" CREATE");
4048  }
4050  printf(" DROP");
4051  }
4053  printf(" SELECT");
4054  }
4056  printf(" INSERT");
4057  }
4059  printf(" UPDATE");
4060  }
4062  printf(" DELETE");
4063  }
4065  printf(" TRUNCATE");
4066  }
4068  printf(" ALTER");
4069  }
4070  } else if (objectType == DBObjectType::DashboardDBObjectType) {
4072  printf(" CREATE");
4073  }
4075  printf(" DELETE");
4076  }
4078  printf(" VIEW");
4079  }
4081  printf(" EDIT");
4082  }
4083  } else if (objectType == DBObjectType::ViewDBObjectType) {
4085  printf(" CREATE");
4086  }
4088  printf(" DROP");
4089  }
4091  printf(" SELECT");
4092  }
4094  printf(" INSERT");
4095  }
4097  printf(" UPDATE");
4098  }
4100  printf(" DELETE");
4101  }
4102  }
4103  printf(".\n");
4104 }
4105 
4106 // GRANT payroll_dept_role TO joe;
4107 void GrantRoleStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4108  const auto& currentUser = session.get_currentUser();
4109  if (!currentUser.isSuper) {
4110  throw std::runtime_error(
4111  "GRANT failed, because it can only be executed by super user.");
4112  }
4113  if (std::find(get_grantees().begin(), get_grantees().end(), OMNISCI_ROOT_USER) !=
4114  get_grantees().end()) {
4115  throw std::runtime_error(
4116  "Request to grant role failed because mapd root user has all privileges by "
4117  "default.");
4118  }
4119  SysCatalog::instance().grantRoleBatch(get_roles(), get_grantees());
4120 }
4121 
4122 // REVOKE payroll_dept_role FROM joe;get_users
4123 void RevokeRoleStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4124  const auto& currentUser = session.get_currentUser();
4125  if (!currentUser.isSuper) {
4126  throw std::runtime_error(
4127  "REVOKE failed, because it can only be executed by super user.");
4128  }
4129  if (std::find(get_grantees().begin(), get_grantees().end(), OMNISCI_ROOT_USER) !=
4130  get_grantees().end()) {
4131  throw std::runtime_error(
4132  "Request to revoke role failed because privileges can not be revoked from mapd "
4133  "root user.");
4134  }
4135  SysCatalog::instance().revokeRoleBatch(get_roles(), get_grantees());
4136 }
4137 
4138 using dbl = std::numeric_limits<double>;
4139 
4140 void ExportQueryStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4141  auto session_copy = session;
4142  auto session_ptr = std::shared_ptr<Catalog_Namespace::SessionInfo>(
4143  &session_copy, boost::null_deleter());
4144  auto query_state = query_state::QueryState::create(session_ptr, *select_stmt);
4145  auto stdlog = STDLOG(query_state);
4146  if (SysCatalog::instance().isAggregator()) {
4147  // allow copy to statement for stand alone leafs
4148  throw std::runtime_error("Distributed export not supported yet");
4149  }
4150  auto& catalog = session.getCatalog();
4151  Importer_NS::CopyParams copy_params;
4152  if (!options.empty()) {
4153  for (auto& p : options) {
4154  if (boost::iequals(*p->get_name(), "delimiter")) {
4155  const StringLiteral* str_literal =
4156  dynamic_cast<const StringLiteral*>(p->get_value());
4157  if (str_literal == nullptr) {
4158  throw std::runtime_error("Delimiter option must be a string.");
4159  } else if (str_literal->get_stringval()->length() != 1) {
4160  throw std::runtime_error("Delimiter must be a single character string.");
4161  }
4162  copy_params.delimiter = (*str_literal->get_stringval())[0];
4163  } else if (boost::iequals(*p->get_name(), "nulls")) {
4164  const StringLiteral* str_literal =
4165  dynamic_cast<const StringLiteral*>(p->get_value());
4166  if (str_literal == nullptr) {
4167  throw std::runtime_error("Nulls option must be a string.");
4168  }
4169  copy_params.null_str = *str_literal->get_stringval();
4170  } else if (boost::iequals(*p->get_name(), "header")) {
4171  const StringLiteral* str_literal =
4172  dynamic_cast<const StringLiteral*>(p->get_value());
4173  if (str_literal == nullptr) {
4174  throw std::runtime_error("Header option must be a boolean.");
4175  }
4176  copy_params.has_header = bool_from_string_literal(str_literal)
4179  } else if (boost::iequals(*p->get_name(), "quote")) {
4180  const StringLiteral* str_literal =
4181  dynamic_cast<const StringLiteral*>(p->get_value());
4182  if (str_literal == nullptr) {
4183  throw std::runtime_error("Quote option must be a string.");
4184  } else if (str_literal->get_stringval()->length() != 1) {
4185  throw std::runtime_error("Quote must be a single character string.");
4186  }
4187  copy_params.quote = (*str_literal->get_stringval())[0];
4188  } else if (boost::iequals(*p->get_name(), "escape")) {
4189  const StringLiteral* str_literal =
4190  dynamic_cast<const StringLiteral*>(p->get_value());
4191  if (str_literal == nullptr) {
4192  throw std::runtime_error("Escape option must be a string.");
4193  } else if (str_literal->get_stringval()->length() != 1) {
4194  throw std::runtime_error("Escape must be a single character string.");
4195  }
4196  copy_params.escape = (*str_literal->get_stringval())[0];
4197  } else if (boost::iequals(*p->get_name(), "line_delimiter")) {
4198  const StringLiteral* str_literal =
4199  dynamic_cast<const StringLiteral*>(p->get_value());
4200  if (str_literal == nullptr) {
4201  throw std::runtime_error("Line_delimiter option must be a string.");
4202  } else if (str_literal->get_stringval()->length() != 1) {
4203  throw std::runtime_error("Line_delimiter must be a single character string.");
4204  }
4205  copy_params.line_delim = (*str_literal->get_stringval())[0];
4206  } else if (boost::iequals(*p->get_name(), "quoted")) {
4207  const StringLiteral* str_literal =
4208  dynamic_cast<const StringLiteral*>(p->get_value());
4209  if (str_literal == nullptr) {
4210  throw std::runtime_error("Quoted option must be a boolean.");
4211  }
4212  copy_params.quoted = bool_from_string_literal(str_literal);
4213  } else {
4214  throw std::runtime_error("Invalid option for COPY: " + *p->get_name());
4215  }
4216  }
4217  }
4218  std::vector<TargetMetaInfo> targets;
4219  const auto results =
4220  getResultSet(query_state->createQueryStateProxy(), *select_stmt, targets);
4221  TargetMetaInfo* td = targets.data();
4222 
4223  std::ofstream outfile;
4224  if (file_path->empty() || !boost::filesystem::path(*file_path).is_absolute()) {
4225  std::string file_name;
4226  if (file_path->empty()) {
4227  file_name = session.get_session_id() + ".txt";
4228  } else {
4229  file_name = boost::filesystem::path(*file_path).filename().string();
4230  }
4231  *file_path = catalog.getBasePath() + "/mapd_export/" + session.get_session_id() + "/";
4232  if (!boost::filesystem::exists(*file_path)) {
4233  if (!boost::filesystem::create_directories(*file_path)) {
4234  throw std::runtime_error("Directory " + *file_path + " cannot be created.");
4235  }
4236  }
4237  *file_path += file_name;
4238  }
4239  outfile.open(*file_path);
4240  if (!outfile) {
4241  throw std::runtime_error("Cannot open file: " + *file_path);
4242  }
4244  bool not_first = false;
4245  size_t i = 0;
4246  for (const auto& target : targets) {
4247  std::string col_name = target.get_resname();
4248  if (col_name.empty()) {
4249  col_name = "result_" + std::to_string(i + 1);
4250  }
4251  if (not_first) {
4252  outfile << copy_params.delimiter;
4253  } else {
4254  not_first = true;
4255  }
4256  outfile << col_name;
4257  ++i;
4258  }
4259  outfile << copy_params.line_delim;
4260  }
4261  while (true) {
4262  const auto crt_row = results->getNextRow(true, true);
4263  if (crt_row.empty()) {
4264  break;
4265  }
4266  bool not_first = false;
4267  for (size_t i = 0; i < results->colCount(); ++i) {
4268  bool is_null;
4269  const auto tv = crt_row[i];
4270  const auto scalar_tv = boost::get<ScalarTargetValue>(&tv);
4271  if (not_first) {
4272  outfile << copy_params.delimiter;
4273  } else {
4274  not_first = true;
4275  }
4276  if (copy_params.quoted) {
4277  outfile << copy_params.quote;
4278  }
4279  const auto& ti = td[i].get_type_info();
4280  if (!scalar_tv) {
4281  outfile << datum_to_string(crt_row[i], ti, " | ");
4282  if (copy_params.quoted) {
4283  outfile << copy_params.quote;
4284  }
4285  continue;
4286  }
4287  if (boost::get<int64_t>(scalar_tv)) {
4288  auto int_val = *(boost::get<int64_t>(scalar_tv));
4289  switch (ti.get_type()) {
4290  case kBOOLEAN:
4291  is_null = (int_val == NULL_BOOLEAN);
4292  break;
4293  case kTINYINT:
4294  is_null = (int_val == NULL_TINYINT);
4295  break;
4296  case kSMALLINT:
4297  is_null = (int_val == NULL_SMALLINT);
4298  break;
4299  case kINT:
4300  is_null = (int_val == NULL_INT);
4301  break;
4302  case kBIGINT:
4303  is_null = (int_val == NULL_BIGINT);
4304  break;
4305  case kTIME:
4306  case kTIMESTAMP:
4307  case kDATE:
4308  is_null = (int_val == NULL_BIGINT);
4309  break;
4310  default:
4311  is_null = false;
4312  }
4313  if (is_null) {
4314  outfile << copy_params.null_str;
4315  } else if (ti.get_type() == kTIME) {
4316  const auto t = static_cast<time_t>(int_val);
4317  std::tm tm_struct;
4318  gmtime_r(&t, &tm_struct);
4319  char buf[9];
4320  strftime(buf, 9, "%T", &tm_struct);
4321  outfile << buf;
4322  } else {
4323  outfile << int_val;
4324  }
4325  } else if (boost::get<double>(scalar_tv)) {
4326  auto real_val = *(boost::get<double>(scalar_tv));
4327  if (ti.get_type() == kFLOAT) {
4328  is_null = (real_val == NULL_FLOAT);
4329  } else {
4330  is_null = (real_val == NULL_DOUBLE);
4331  }
4332  if (is_null) {
4333  outfile << copy_params.null_str;
4334  } else if (ti.get_type() == kNUMERIC) {
4335  outfile << std::setprecision(ti.get_precision()) << real_val;
4336  } else {
4337  outfile << std::setprecision(std::numeric_limits<double>::digits10 + 1)
4338  << real_val;
4339  }
4340  } else if (boost::get<float>(scalar_tv)) {
4341  CHECK_EQ(kFLOAT, ti.get_type());
4342  auto real_val = *(boost::get<float>(scalar_tv));
4343  if (real_val == NULL_FLOAT) {
4344  outfile << copy_params.null_str;
4345  } else {
4346  outfile << std::setprecision(std::numeric_limits<float>::digits10 + 1)
4347  << real_val;
4348  }
4349  } else {
4350  auto s = boost::get<NullableString>(scalar_tv);
4351  is_null = !s || boost::get<void*>(s);
4352  if (is_null) {
4353  outfile << copy_params.null_str;
4354  } else {
4355  auto s_notnull = boost::get<std::string>(s);
4356  CHECK(s_notnull);
4357  if (!copy_params.quoted) {
4358  outfile << *s_notnull;
4359  } else {
4360  size_t q = s_notnull->find(copy_params.quote);
4361  if (q == std::string::npos) {
4362  outfile << *s_notnull;
4363  } else {
4364  std::string str(*s_notnull);
4365  while (q != std::string::npos) {
4366  str.insert(q, 1, copy_params.escape);
4367  q = str.find(copy_params.quote, q + 2);
4368  }
4369  outfile << str;
4370  }
4371  }
4372  }
4373  }
4374  if (copy_params.quoted) {
4375  outfile << copy_params.quote;
4376  }
4377  }
4378  outfile << copy_params.line_delim;
4379  }
4380  outfile.close();
4381 }
4382 
4383 void CreateViewStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4384  auto session_copy = session;
4385  auto session_ptr = std::shared_ptr<Catalog_Namespace::SessionInfo>(
4386  &session_copy, boost::null_deleter());
4387  auto query_state = query_state::QueryState::create(session_ptr, select_query_);
4388  auto stdlog = STDLOG(query_state);
4389  auto& catalog = session.getCatalog();
4390 
4391  if (catalog.getMetadataForTable(view_name_) != nullptr) {
4392  if (if_not_exists_) {
4393  return;
4394  }
4395  throw std::runtime_error("Table or View " + view_name_ + " already exists.");
4396  }
4397 
4400  throw std::runtime_error("View " + view_name_ +
4401  " will not be created. User has no create view privileges.");
4402  }
4403 
4404  const auto query_after_shim = pg_shim(select_query_);
4405 
4406  // this now also ensures that access permissions are checked
4407  catalog.getCalciteMgr()->process(
4408  query_state->createQueryStateProxy(), query_after_shim, {}, true, false, false);
4409  TableDescriptor td;
4410  td.tableName = view_name_;
4411  td.userId = session.get_currentUser().userId;
4412  td.nColumns = 0;
4413  td.isView = true;
4414  td.viewSQL = query_after_shim;
4415  td.fragmenter = nullptr;
4417  td.maxFragRows =
4418  DEFAULT_FRAGMENT_ROWS; // @todo this stuff should not be InsertOrderFragmenter
4419  td.maxChunkSize =
4420  DEFAULT_MAX_CHUNK_SIZE; // @todo this stuff should not be InsertOrderFragmenter
4423  catalog.createTable(td, {}, {}, true);
4424 
4425  // TODO (max): It's transactionally unsafe, should be fixed: we may create object w/o
4426  // privileges
4427  SysCatalog::instance().createDBObject(
4428  session.get_currentUser(), view_name_, ViewDBObjectType, catalog);
4429 }
4430 
4431 void DropViewStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4432  auto& catalog = session.getCatalog();
4433  const TableDescriptor* td = catalog.getMetadataForTable(*view_name);
4434  if (td == nullptr) {
4435  if (if_exists) {
4436  return;
4437  }
4438  throw std::runtime_error("View " + *view_name + " does not exist.");
4439  }
4440 
4441  if (!session.checkDBAccessPrivileges(
4443  throw std::runtime_error("View " + *view_name +
4444  " will not be dropped. User has no drop view privileges.");
4445  }
4446 
4447  if (!td->isView) {
4448  throw std::runtime_error(*view_name + " is a table. Use DROP TABLE.");
4449  }
4450  catalog.dropTable(td);
4451 }
4452 
4453 void CreateDBStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4454  if (!session.get_currentUser().isSuper) {
4455  throw std::runtime_error(
4456  "CREATE DATABASE command can only be executed by super user.");
4457  }
4459  if (SysCatalog::instance().getMetadataForDB(*db_name, db_meta) && if_not_exists_) {
4460  return;
4461  }
4462  int ownerId = session.get_currentUser().userId;
4463  if (!name_value_list.empty()) {
4464  for (auto& p : name_value_list) {
4465  if (boost::iequals(*p->get_name(), "owner")) {
4466  if (!dynamic_cast<const StringLiteral*>(p->get_value())) {
4467  throw std::runtime_error("Owner name must be a string literal.");
4468  }
4469  const std::string* str =
4470  static_cast<const StringLiteral*>(p->get_value())->get_stringval();
4472  if (!SysCatalog::instance().getMetadataForUser(*str, user)) {
4473  throw std::runtime_error("User " + *str + " does not exist.");
4474  }
4475  ownerId = user.userId;
4476  } else {
4477  throw std::runtime_error("Invalid CREATE DATABASE option " + *p->get_name() +
4478  ". Only OWNER supported.");
4479  }
4480  }
4481  }
4482  SysCatalog::instance().createDatabase(*db_name, ownerId);
4483 }
4484 
4485 void DropDBStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4486  if (!session.get_currentUser().isSuper) {
4487  throw std::runtime_error("DROP DATABASE command can only be executed by super user.");
4488  }
4490  if (!SysCatalog::instance().getMetadataForDB(*db_name, db)) {
4491  if (if_exists_) {
4492  return;
4493  }
4494  throw std::runtime_error("Database " + *db_name + " does not exist.");
4495  }
4496 
4497  if (!session.get_currentUser().isSuper &&
4498  session.get_currentUser().userId != db.dbOwner) {
4499  throw std::runtime_error("Only the super user or the owner can drop database.");
4500  }
4501 
4502  SysCatalog::instance().dropDatabase(db);
4503 }
4504 
4505 void CreateUserStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4506  std::string passwd;
4507  bool is_super = false;
4508  std::string default_db;
4509  for (auto& p : name_value_list) {
4510  if (boost::iequals(*p->get_name(), "password")) {
4511  if (!dynamic_cast<const StringLiteral*>(p->get_value())) {
4512  throw std::runtime_error("Password must be a string literal.");
4513  }
4514  passwd = *static_cast<const StringLiteral*>(p->get_value())->get_stringval();
4515  } else if (boost::iequals(*p->get_name(), "is_super")) {
4516  if (!dynamic_cast<const StringLiteral*>(p->get_value())) {
4517  throw std::runtime_error("IS_SUPER option must be a string literal.");
4518  }
4519  const std::string* str =
4520  static_cast<const StringLiteral*>(p->get_value())->get_stringval();
4521  if (boost::iequals(*str, "true")) {
4522  is_super = true;
4523  } else if (boost::iequals(*str, "false")) {
4524  is_super = false;
4525  } else {
4526  throw std::runtime_error("Value to IS_SUPER must be TRUE or FALSE.");
4527  }
4528  } else if (boost::iequals(*p->get_name(), "default_db")) {
4529  if (!dynamic_cast<const StringLiteral*>(p->get_value())) {
4530  throw std::runtime_error("DEFAULT_DB option must be a string literal.");
4531  }
4532  default_db = *static_cast<const StringLiteral*>(p->get_value())->get_stringval();
4533  } else {
4534  throw std::runtime_error("Invalid CREATE USER option " + *p->get_name() +
4535  ". Should be PASSWORD, IS_SUPER, or DEFAULT_DB.");
4536  }
4537  }
4538  if (!session.get_currentUser().isSuper) {
4539  throw std::runtime_error("Only super user can create new users.");
4540  }
4541  SysCatalog::instance().createUser(*user_name, passwd, is_super, default_db);
4542 }
4543 
4544 void AlterUserStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4545  // Parse the statement
4546  const std::string* passwd = nullptr;
4547  bool is_super = false;
4548  bool* is_superp = nullptr;
4549  const std::string* default_db = nullptr;
4550  for (auto& p : name_value_list) {
4551  if (boost::iequals(*p->get_name(), "password")) {
4552  if (!dynamic_cast<const StringLiteral*>(p->get_value())) {
4553  throw std::runtime_error("Password must be a string literal.");
4554  }
4555  passwd = static_cast<const StringLiteral*>(p->get_value())->get_stringval();
4556  } else if (boost::iequals(*p->get_name(), "is_super")) {
4557  if (!dynamic_cast<const StringLiteral*>(p->get_value())) {
4558  throw std::runtime_error("IS_SUPER option must be a string literal.");
4559  }
4560  const std::string* str =
4561  static_cast<const StringLiteral*>(p->get_value())->get_stringval();
4562  if (boost::iequals(*str, "true")) {
4563  is_super = true;
4564  is_superp = &is_super;
4565  } else if (boost::iequals(*str, "false")) {
4566  is_super = false;
4567  is_superp = &is_super;
4568  } else {
4569  throw std::runtime_error("Value to IS_SUPER must be TRUE or FALSE.");
4570  }
4571  } else if (boost::iequals(*p->get_name(), "default_db")) {
4572  if (dynamic_cast<const StringLiteral*>(p->get_value())) {
4573  default_db = static_cast<const StringLiteral*>(p->get_value())->get_stringval();
4574  } else if (dynamic_cast<const NullLiteral*>(p->get_value())) {
4575  static std::string blank;
4576  default_db = &blank;
4577  } else {
4578  throw std::runtime_error(
4579  "DEFAULT_DB option must be either a string literal or a NULL literal.");
4580  }
4581  } else {
4582  throw std::runtime_error("Invalid ALTER USER option " + *p->get_name() +
4583  ". Should be PASSWORD, DEFAULT_DB or IS_SUPER.");
4584  }
4585  }
4586 
4587  // Check if the user is authorized to execute ALTER USER statement
4589  if (!SysCatalog::instance().getMetadataForUser(*user_name, user)) {
4590  throw std::runtime_error("User " + *user_name + " does not exist.");
4591  }
4592  if (!session.get_currentUser().isSuper) {
4593  if (session.get_currentUser().userId != user.userId) {
4594  throw std::runtime_error("Only super user can change another user's attributes.");
4595  } else if (is_superp) {
4596  throw std::runtime_error(
4597  "A user can only update their own password or default database.");
4598  }
4599  }
4600 
4601  if (passwd || is_superp || default_db) {
4602  SysCatalog::instance().alterUser(user.userId, passwd, is_superp, default_db);
4603  }
4604 }
4605 
4606 void DropUserStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4607  if (!session.get_currentUser().isSuper) {
4608  throw std::runtime_error("Only super user can drop users.");
4609  }
4610  SysCatalog::instance().dropUser(*user_name);
4611 }
4612 
4613 void DumpTableStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4614  // check access privileges
4615  if (!session.checkDBAccessPrivileges(
4617  throw std::runtime_error("Table " + *table +
4618  " will not be dumped. User has no select privileges.");
4619  }
4622  throw std::runtime_error("Table " + *table +
4623  " will not be dumped. User has no create privileges.");
4624  }
4625  auto& catalog = session.getCatalog();
4626  const TableDescriptor* td = catalog.getMetadataForTable(*table);
4627  catalog.dumpTable(td, *path, compression);
4628 }
4629 
4630 void RestoreTableStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4631  auto& catalog = session.getCatalog();
4632  const TableDescriptor* td = catalog.getMetadataForTable(*table, false);
4633  if (td) {
4634  // TODO: v1.0 simply throws to avoid accidentally overwrite target table.
4635  // Will add a REPLACE TABLE to explictly replace target table.
4636  // catalog.restoreTable(session, td, *path, compression);
4637  throw std::runtime_error("Table " + *table + " exists.");
4638  } else {
4639  // check access privileges
4642  throw std::runtime_error("Table " + *table +
4643  " will not be restored. User has no create privileges.");
4644  }
4645  catalog.restoreTable(session, *table, *path, compression);
4646  }
4647 }
4648 
4649 } // namespace Parser
4650 
4651 // this is a non-clustered version of MapDHandler::prepare_columnar_loader,
4652 // exists for non-thrift builds, specifically for test cases and bin/initdb.
4654  const std::string& session,
4655  const std::string& table_name,
4656  size_t num_cols,
4657  std::unique_ptr<Importer_NS::Loader>* loader,
4658  std::vector<std::unique_ptr<Importer_NS::TypedImportBuffer>>* import_buffers) {
4659  auto col_descs = (*loader)->get_column_descs();
4660  for (auto cd : col_descs) {
4661  import_buffers->push_back(std::unique_ptr<Importer_NS::TypedImportBuffer>(
4662  new Importer_NS::TypedImportBuffer(cd, (*loader)->getStringDict(cd))));
4663  }
4664 }
int8_t tinyintval
Definition: sqltypes.h:126
const ColumnConstraintDef * get_column_constraint() const
Definition: ParserNode.h:818
decltype(auto) get_max_rows_def(TableDescriptor &td, const NameValueAssign *p, const std::list< ColumnDescriptor > &columns)
static std::set< std::string > reserved_keywords
static const AccessPrivileges VIEW_SQL_EDITOR
Definition: DBObject.h:145
decltype(auto) get_max_chunk_size_def(TableDescriptor &td, const NameValueAssign *p, const std::list< ColumnDescriptor > &columns)
bool is_boolean() const
Definition: sqltypes.h:484
static const std::map< const std::string, const TableDefFuncPtr > tableDefFuncMap
SQLAgg
Definition: sqldefs.h:71
#define CHECK_EQ(x, y)
Definition: Logger.h:198
size_t shard_column_index(const std::string &name, const std::list< ColumnDescriptor > &columns)
const std::string & get_column() const
Definition: ParserNode.h:939
std::string partitions
std::vector< int > ChunkKey
Definition: types.h:35
HOST DEVICE int get_output_srid() const
Definition: sqltypes.h:332
MapDHandler * get_mapdHandler() const
Definition: SessionInfo.h:89
static const AccessPrivileges VIEW_DASHBOARD
Definition: DBObject.h:164
#define NULL_DOUBLE
Definition: sqltypes.h:179
static const int32_t DROP_VIEW
Definition: DBObject.h:114
SQLType * get_column_type() const
Definition: ParserNode.h:816
static const int32_t SELECT_FROM_VIEW
Definition: DBObject.h:115
const int8_t const int64_t * num_rows
Definition: Analyzer.h:1483
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:81
std::string null_str
Definition: CopyParams.h:47
Definition: sqltypes.h:52
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:334
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:335
SQLTypes
Definition: sqltypes.h:41
static std::shared_ptr< Executor > getExecutor(const int db_id, const std::string &debug_dir="", const std::string &debug_file="", const MapDParameters mapd_parameters=MapDParameters(),::QueryRenderer::QueryRenderManager *render_manager=nullptr)
Definition: Execute.cpp:127
static const int32_t UPDATE_IN_VIEW
Definition: DBObject.h:117
std::string tableName
const std::vector< TargetMetaInfo > targets_meta
bool g_cluster
void add_rte(RangeTableEntry *rte)
Definition: Analyzer.cpp:1274
static const AccessPrivileges ALL_DATABASE
Definition: DBObject.h:144
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
decltype(auto) get_vacuum_def(TableDescriptor &td, const NameValueAssign *p, const std::list< ColumnDescriptor > &columns)
static const AccessPrivileges ALTER_TABLE
Definition: DBObject.h:158
std::string to_string() const
DBObjectType
Definition: DBObject.h:42
static const int32_t CREATE_VIEW
Definition: DBObject.h:113
virtual void prepare_columnar_loader(const std::string &session, const std::string &table_name, size_t num_cols, std::unique_ptr< Importer_NS::Loader > *loader, std::vector< std::unique_ptr< Importer_NS::TypedImportBuffer >> *import_buffers)
#define NULL_BIGINT
Definition: sqltypes.h:177
void expand_star_in_targetlist(const Catalog_Namespace::Catalog &catalog, std::vector< std::shared_ptr< TargetEntry >> &tlist, int rte_idx)
static const AccessPrivileges TRUNCATE_TABLE
Definition: DBObject.h:157
SQLQualifier
Definition: sqldefs.h:69
#define LOG(tag)
Definition: Logger.h:185
std::string datum_to_string(const TargetValue &tv, const SQLTypeInfo &ti, const std::string &delim)
std::string parse_to_ra(query_state::QueryStateProxy query_state_proxy, const std::string &query_str)
Definition: LockMgr.cpp:77
SQLOps
Definition: sqldefs.h:29
Definition: sqldefs.h:35
void set_size(int s)
Definition: sqltypes.h:424
static const AccessPrivileges INSERT_INTO_TABLE
Definition: DBObject.h:154
HOST DEVICE int get_scale() const
Definition: sqltypes.h:331
std::shared_ptr< Analyzer::Expr > deep_copy() const override
Definition: Analyzer.cpp:80
Definition: sqldefs.h:36
static const AccessPrivileges CREATE_DASHBOARD
Definition: DBObject.h:163
static std::shared_ptr< QueryState > create(ARGS &&...args)
Definition: QueryState.h:140
bool get_is_null() const
Definition: Analyzer.h:328
std::vector< uint8_t > compress_coords(std::vector< double > &coords, const SQLTypeInfo &ti)
Definition: Importer.cpp:1422
void set_input_srid(int d)
Definition: sqltypes.h:420
std::shared_ptr< ResultSet > getResultSet(QueryStateProxy query_state_proxy, const std::string select_stmt, std::vector< TargetMetaInfo > &targets, bool validate_only=false)
HOST DEVICE int get_size() const
Definition: sqltypes.h:336
Definition: sqldefs.h:49
#define DEFAULT_MAX_ROWS
HOST DEVICE void set_type(SQLTypes t)
Definition: sqltypes.h:416
void set_order_by(std::list< OrderEntry > *o)
Definition: Analyzer.h:1550
std::string extractObjectNameFromHierName(const std::string &objectHierName, const std::string &objectType, const Catalog_Namespace::Catalog &cat)
void set_fixed_size()
Definition: sqltypes.h:425
static const int32_t ALTER_TABLE
Definition: DBObject.h:94
void set_result_col_list(const std::list< int > &col_list)
Definition: Analyzer.h:1544
int get_num_aggs() const
Definition: Analyzer.h:1525
void set_dimension(int d)
Definition: sqltypes.h:418
void setPrivileges(const AccessPrivileges &privs)
Definition: DBObject.h:203
void set_offset(int64_t o)
Definition: Analyzer.h:1562
std::vector< std::string > splitObjectHierName(const std::string &hierName)
static const AccessPrivileges SELECT_FROM_TABLE
Definition: DBObject.h:153
void set_scale(int s)
Definition: sqltypes.h:421
const ColumnDescriptor * getShardColumnMetadataForTable(const TableDescriptor *td) const
Definition: Catalog.cpp:2881
void set_compression(EncodingType c)
Definition: sqltypes.h:426
bool bool_from_string_literal(const Parser::StringLiteral *str_literal)
Definition: ParserNode.cpp:892
int32_t intval
Definition: sqltypes.h:128
int32_t get_table_id() const
std::string to_string(char const *&&v)
void set_notnull(bool n)
Definition: sqltypes.h:423
void set_output_srid(int s)
Definition: sqltypes.h:422
int get_rte_idx(const std::string &range_var_name) const
Definition: Analyzer.cpp:1263
bool is_number() const
Definition: sqltypes.h:482
static const AccessPrivileges ALL_VIEW
Definition: DBObject.h:170
std::string get_type_name() const
Definition: sqltypes.h:429
EncodingType geo_coords_encoding
Definition: CopyParams.h:73
std::shared_ptr< StringDictionary > stringDict
int tableId
identifies the database into which the data is being inserted
Definition: Fragmenter.h:61
DBObjectType DBObjectTypeFromString(const std::string &type)
Definition: DBObject.cpp:88
Definition: sqldefs.h:71
size_t numRows
a vector of column ids for the row(s) being inserted
Definition: Fragmenter.h:63
float floatval
Definition: sqltypes.h:130
std::numeric_limits< double > dbl
bool get_is_array() const
Definition: ParserNode.h:81
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:417
bool g_use_date_in_days_default_encoding
Definition: ParserNode.cpp:67
static SQLTypeInfo common_string_type(const SQLTypeInfo &type1, const SQLTypeInfo &type2)
Definition: Analyzer.cpp:387
void set_precision(int d)
Definition: sqltypes.h:419
const std::string * get_stringval() const
Definition: ParserNode.h:149
CHECK(cgen_state)
bool is_array() const
Definition: sqltypes.h:485
Classes representing a parse tree.
std::string s3_access_key
Definition: CopyParams.h:62
bool is_time() const
Definition: sqltypes.h:483
size_t g_leaf_count
Definition: ParserNode.cpp:66
int get_logical_size() const
Definition: sqltypes.h:337
RangeTableEntry * get_rte(int rte_idx) const
Definition: Analyzer.h:1556
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:176
QueryState & getQueryState()
Definition: QueryState.h:172
static const int32_t DROP_DATABASE
Definition: DBObject.h:80
#define NULL_TINYINT
Definition: sqltypes.h:174
static const AccessPrivileges DROP_TABLE
Definition: DBObject.h:152
const std::string * get_encoding_name() const
Definition: ParserNode.h:799
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:326
decltype(auto) get_shard_count_def(TableDescriptor &td, const NameValueAssign *p, const std::list< ColumnDescriptor > &columns)
boost::function< void(TableDescriptor &, const NameValueAssign *, const std::list< ColumnDescriptor > &columns)> TableDefFuncPtr
Definition: ParserNode.cpp:76
int64_t bigintval
Definition: sqltypes.h:129
WhichRow get_which_row() const
Definition: Analyzer.h:273
#define NULL_FLOAT
Definition: sqltypes.h:178
std::list< ColumnDescriptor > getColumnDescriptors(AggregatedResult &result, bool for_create)
int64_t get_intval() const
Definition: ParserNode.h:168
static const int32_t DELETE_FROM_TABLE
Definition: DBObject.h:92
bool get_is_distinct() const
Definition: Analyzer.h:1524
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:333
static SQLTypeInfo common_numeric_type(const SQLTypeInfo &type1, const SQLTypeInfo &type2)
Definition: Analyzer.cpp:427
Definition: sqldefs.h:37
Definition: sqldefs.h:71
static const int32_t TRUNCATE_TABLE
Definition: DBObject.h:93
const std::string * get_column_name() const
Definition: ParserNode.h:815
#define DEFAULT_PAGE_SIZE
int16_t smallintval
Definition: sqltypes.h:127
int get_varno() const
Definition: Analyzer.h:275
static const AccessPrivileges DELETE_FROM_TABLE
Definition: DBObject.h:156
ImportHeaderRow has_header
Definition: CopyParams.h:48
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1350
specifies the content in-memory of a row in the column metadata table
#define NULL_INT
Definition: sqltypes.h:176
bool expr_is_null(const Analyzer::Expr *expr)
Definition: ParserNode.cpp:884
static const int32_t EDIT_DASHBOARD
Definition: DBObject.h:105
static const int32_t DELETE_DASHBOARD
Definition: DBObject.h:103
static const AccessPrivileges CREATE_TABLE
Definition: DBObject.h:151
static const int32_t INSERT_INTO_TABLE
Definition: DBObject.h:90
SQLTypeInfoCore< ArrayContextTypeSizer, ExecutorTypePackaging, DateTimeFacilities > SQLTypeInfo
Definition: sqltypes.h:852
std::string keyMetainfo
void set_num_aggs(int a)
Definition: Analyzer.h:1554
void set_group_by(std::list< std::shared_ptr< Analyzer::Expr >> &g)
Definition: Analyzer.h:1548
const SQLTypeInfo & get_type_info() const
Definition: Analyzer.h:78
bool is_geometry() const
Definition: sqltypes.h:489
void set_next_query(Query *q)
Definition: Analyzer.h:1551
std::string * stringval
Definition: sqltypes.h:134
int get_result_table_id() const
Definition: Analyzer.h:1542
static void set_geo_physical_import_buffer(const Catalog_Namespace::Catalog &catalog, const ColumnDescriptor *cd, std::vector< std::unique_ptr< TypedImportBuffer >> &import_buffers, size_t &col_idx, std::vector< double > &coords, std::vector< double > &bounds, std::vector< int > &ring_sizes, std::vector< int > &poly_rings, int render_group, const int64_t replicate_count=0)
Definition: Importer.cpp:1457
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1581
std::string s3_endpoint
Definition: CopyParams.h:65
void set_having_predicate(std::shared_ptr< Analyzer::Expr > p)
Definition: Analyzer.h:1549
std::string get_session_id() const
Definition: SessionInfo.h:98
void set_result_table_id(int id)
Definition: Analyzer.h:1545
void loadKey()
Definition: DBObject.cpp:165
const AccessPrivileges & getPrivileges() const
Definition: DBObject.h:202
void validate_shard_column_type(const ColumnDescriptor &cd)
Definition: sqltypes.h:55
Definition: sqltypes.h:56
void check_alter_table_privilege(const Catalog_Namespace::SessionInfo &session, const TableDescriptor *td)
Definition: sqldefs.h:69
#define TRANSIENT_DICT_ID
Definition: sqltypes.h:189
const std::string OMNISCI_ROOT_USER
Definition: SysCatalog.h:59
int64_t const int32_t sz assert(dest)
void set_is_distinct(bool d)
Definition: Analyzer.h:1546
static const AccessPrivileges SELECT_FROM_VIEW
Definition: DBObject.h:173
bool is_null(const T &v, const SQLTypeInfo &t)
const CompressDef * get_compression() const
Definition: ParserNode.h:817
std::vector< std::shared_ptr< TargetEntry > > & get_targetlist_nonconst()
Definition: Analyzer.h:1529
Catalog & getCatalog() const
Definition: SessionInfo.h:90
bool is_integer() const
Definition: sqltypes.h:479
Datum StringToDatum(const std::string &s, SQLTypeInfo &ti)
Definition: Datum.cpp:90
std::string s3_region
Definition: CopyParams.h:64
Datum get_constval() const
Definition: Analyzer.h:329
bool g_enable_experimental_string_functions
Fragmenter_Namespace::FragmenterType fragType
Definition: sqldefs.h:71
Data_Namespace::MemoryLevel persistenceLevel
SQLTypeInfoCore get_array_type() const
Definition: sqltypes.h:663
void set_is_unionall(bool u)
Definition: Analyzer.h:1552
static const int32_t CREATE_DATABASE
Definition: DBObject.h:79
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:328
std::vector< std::string > split(const std::string &str, const std::string &delim)
split apart a string into a vector of substrings
void insertData(const Catalog_Namespace::SessionInfo &session_info, InsertData &insert_data)
decltype(auto) get_sort_column_def(TableDescriptor &td, const NameValueAssign *p, const std::list< ColumnDescriptor > &columns)
static const AccessPrivileges ALL_DASHBOARD
Definition: DBObject.h:162
static bool getGeoColumns(const std::string &wkt, SQLTypeInfo &ti, std::vector< double > &coords, std::vector< double > &bounds, std::vector< int > &ring_sizes, std::vector< int > &poly_rings, const bool promote_poly_to_mpoly=false)
Definition: geo_types.cpp:459
static const AccessPrivileges ACCESS
Definition: DBObject.h:146
std::string geo_layer_name
Definition: CopyParams.h:78
static const int32_t VIEW_DASHBOARD
Definition: DBObject.h:104
static std::pair< AccessPrivileges, DBObjectType > parseStringPrivs(const std::string &privs, const DBObjectType &objectType, const std::string &object_name)
Definition: sqltypes.h:44
virtual std::shared_ptr< Analyzer::Expr > add_cast(const SQLTypeInfo &new_type_info)
Definition: Analyzer.cpp:671
static const AccessPrivileges ALL_TABLE
Definition: DBObject.h:150
AggregatedResult query(QueryStateProxy, std::string &sql_query_string, bool validate_only)
decltype(auto) get_frag_size_def(TableDescriptor &td, const NameValueAssign *p, const std::list< ColumnDescriptor > &columns)
#define DEFAULT_FRAGMENT_ROWS
void set_comp_param(int p)
Definition: sqltypes.h:427
void set_where_predicate(std::shared_ptr< Analyzer::Expr > p)
Definition: Analyzer.h:1547
int get_param1() const
Definition: ParserNode.h:79
void set_stmt_type(SQLStmtType t)
Definition: Analyzer.h:1553
const ColumnDescriptor * get_column_desc(const Catalog_Namespace::Catalog &catalog, const std::string &name)
int get_array_size() const
Definition: ParserNode.h:83
Definition: sqldefs.h:53
static const int32_t DROP_TABLE
Definition: DBObject.h:88
static const int32_t INSERT_INTO_VIEW
Definition: DBObject.h:116
#define NULL_SMALLINT
Definition: sqltypes.h:175
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:327
virtual std::shared_ptr< Analyzer::Expr > analyze(const Catalog_Namespace::Catalog &catalog, Analyzer::Query &query, TlistRefType allow_tlist_ref=TLIST_NONE) const =0
static const AccessPrivileges DROP_VIEW
Definition: DBObject.h:172
bool is_string() const
Definition: sqltypes.h:477
std::shared_ptr< ResultSet > rs
static const int32_t DELETE_FROM_VIEW
Definition: DBObject.h:118
static const AccessPrivileges CREATE_VIEW
Definition: DBObject.h:171
static const int32_t CREATE_TABLE
Definition: DBObject.h:87
void get_table_definitions(TableDescriptor &td, const std::unique_ptr< NameValueAssign > &p, const std::list< ColumnDescriptor > &columns)
decltype(auto) get_partions_def(TableDescriptor &td, const NameValueAssign *p, const std::list< ColumnDescriptor > &columns)
SQLTypes get_type() const
Definition: ParserNode.h:78
static const int32_t CREATE_DASHBOARD
Definition: DBObject.h:102
decltype(auto) get_page_size_def(TableDescriptor &td, const NameValueAssign *p, const std::list< ColumnDescriptor > &columns)
int64_t inline_fixed_encoding_null_val(const SQL_TYPE_INFO &ti)
decltype(auto) get_property_value(const NameValueAssign *p, ASSIGNMENT op, VALIDATE validate=VALIDATE())
const std::list< std::shared_ptr< Analyzer::Expr > > & get_group_by() const
Definition: Analyzer.h:1534
The data to be inserted using the fragment manager.
Definition: Fragmenter.h:59
static DBObject createObject(const std::string &objectName, DBObjectType objectType)
#define DEFAULT_MAX_CHUNK_SIZE
std::string viewSQL
static const AccessPrivileges DELETE_DASHBOARD
Definition: DBObject.h:166
int get_precision() const
Definition: sqltypes.h:329
Definition: sqltypes.h:48
static const int32_t SELECT_FROM_TABLE
Definition: DBObject.h:89
SQLTypeInfoCore get_elem_type() const
Definition: sqltypes.h:659
SQLTypeInfo columnType
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
specifies the content in-memory of a row in the table metadata table
bool checkDBAccessPrivileges(const DBObjectType &permissionType, const AccessPrivileges &privs, const std::string &objectName="") const
Definition: SessionInfo.cpp:24
Definition: Analyzer.h:1362
void set_limit(int64_t l)
Definition: Analyzer.h:1560
std::string s3_secret_key
Definition: CopyParams.h:63
bool is_decimal() const
Definition: sqltypes.h:480