OmniSciDB  addbbd5075
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
ParserNode.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
25 #include "ParserNode.h"
26 
27 #include "../Analyzer/RangeTableEntry.h"
28 #include "../Catalog/Catalog.h"
29 #include "../Catalog/SharedDictionaryValidator.h"
30 #include "../Fragmenter/InsertOrderFragmenter.h"
31 #include "../Fragmenter/TargetValueConvertersFactories.h"
32 #include "../Import/Importer.h"
33 #include "../Planner/Planner.h"
34 #include "../QueryEngine/CalciteAdapter.h"
35 #include "../QueryEngine/Execute.h"
36 #include "../QueryEngine/ExtensionFunctionsWhitelist.h"
37 #include "../QueryEngine/RelAlgExecutor.h"
38 #include "../Shared/TimeGM.h"
39 #include "../Shared/geo_types.h"
40 #include "../Shared/mapd_glob.h"
41 #include "../Shared/measure.h"
42 #include "../Shared/shard_key.h"
43 #include "LockMgr/LockMgr.h"
44 #include "LockMgr/TableLockMgr.h"
45 #include "ReservedKeywords.h"
46 #include "gen-cpp/CalciteServer.h"
47 #include "parser.h"
48 
49 #include <boost/algorithm/string.hpp>
50 #include <boost/core/null_deleter.hpp>
51 #include <boost/filesystem.hpp>
52 #include <boost/function.hpp>
53 
54 #include <rapidjson/document.h>
55 #include <rapidjson/stringbuffer.h>
56 #include <rapidjson/writer.h>
57 
58 #include <cassert>
59 #include <cmath>
60 #include <limits>
61 #include <random>
62 #include <stdexcept>
63 #include <type_traits>
64 #include <typeinfo>
65 
66 size_t g_leaf_count{0};
69 
70 using namespace Lock_Namespace;
72 using namespace std::string_literals;
73 
74 using TableDefFuncPtr = boost::function<void(TableDescriptor&,
75  const NameValueAssign*,
76  const std::list<ColumnDescriptor>& columns)>;
77 
78 namespace Importer_NS {
79 std::vector<uint8_t> compress_coords(std::vector<double>& coords, const SQLTypeInfo& ti);
80 } // namespace Importer_NS
81 
82 namespace Parser {
83 std::shared_ptr<Analyzer::Expr> NullLiteral::analyze(
84  const Catalog_Namespace::Catalog& catalog,
85  Analyzer::Query& query,
86  TlistRefType allow_tlist_ref) const {
87  return makeExpr<Analyzer::Constant>(kNULLT, true);
88 }
89 
90 std::shared_ptr<Analyzer::Expr> StringLiteral::analyze(
91  const Catalog_Namespace::Catalog& catalog,
92  Analyzer::Query& query,
93  TlistRefType allow_tlist_ref) const {
94  return analyzeValue(*stringval);
95 }
96 
97 std::shared_ptr<Analyzer::Expr> StringLiteral::analyzeValue(
98  const std::string& stringval) {
99  SQLTypeInfo ti(kVARCHAR, stringval.length(), 0, true);
100  Datum d;
101  d.stringval = new std::string(stringval);
102  return makeExpr<Analyzer::Constant>(ti, false, d);
103 }
104 
105 std::shared_ptr<Analyzer::Expr> IntLiteral::analyze(
106  const Catalog_Namespace::Catalog& catalog,
107  Analyzer::Query& query,
108  TlistRefType allow_tlist_ref) const {
109  return analyzeValue(intval);
110 }
111 
112 std::shared_ptr<Analyzer::Expr> IntLiteral::analyzeValue(const int64_t intval) {
113  SQLTypes t;
114  Datum d;
115  if (intval >= INT16_MIN && intval <= INT16_MAX) {
116  t = kSMALLINT;
117  d.smallintval = (int16_t)intval;
118  } else if (intval >= INT32_MIN && intval <= INT32_MAX) {
119  t = kINT;
120  d.intval = (int32_t)intval;
121  } else {
122  t = kBIGINT;
123  d.bigintval = intval;
124  }
125  return makeExpr<Analyzer::Constant>(t, false, d);
126 }
127 
128 std::shared_ptr<Analyzer::Expr> FixedPtLiteral::analyze(
129  const Catalog_Namespace::Catalog& catalog,
130  Analyzer::Query& query,
131  TlistRefType allow_tlist_ref) const {
132  SQLTypeInfo ti(kNUMERIC, 0, 0, false);
133  Datum d = StringToDatum(*fixedptval, ti);
134  return makeExpr<Analyzer::Constant>(ti, false, d);
135 }
136 
137 std::shared_ptr<Analyzer::Expr> FixedPtLiteral::analyzeValue(const int64_t numericval,
138  const int scale,
139  const int precision) {
140  SQLTypeInfo ti(kNUMERIC, 0, 0, false);
141  ti.set_scale(scale);
142  ti.set_precision(precision);
143  Datum d;
144  d.bigintval = numericval;
145  return makeExpr<Analyzer::Constant>(ti, false, d);
146 }
147 
148 std::shared_ptr<Analyzer::Expr> FloatLiteral::analyze(
149  const Catalog_Namespace::Catalog& catalog,
150  Analyzer::Query& query,
151  TlistRefType allow_tlist_ref) const {
152  Datum d;
153  d.floatval = floatval;
154  return makeExpr<Analyzer::Constant>(kFLOAT, false, d);
155 }
156 
157 std::shared_ptr<Analyzer::Expr> DoubleLiteral::analyze(
158  const Catalog_Namespace::Catalog& catalog,
159  Analyzer::Query& query,
160  TlistRefType allow_tlist_ref) const {
161  Datum d;
162  d.doubleval = doubleval;
163  return makeExpr<Analyzer::Constant>(kDOUBLE, false, d);
164 }
165 
166 std::shared_ptr<Analyzer::Expr> TimestampLiteral::analyze(
167  const Catalog_Namespace::Catalog& catalog,
168  Analyzer::Query& query,
169  TlistRefType allow_tlist_ref) const {
170  return get(timestampval_);
171 }
172 
173 std::shared_ptr<Analyzer::Expr> TimestampLiteral::get(const int64_t timestampval) {
174  Datum d;
175  d.bigintval = timestampval;
176  return makeExpr<Analyzer::Constant>(kTIMESTAMP, false, d);
177 }
178 
179 std::shared_ptr<Analyzer::Expr> UserLiteral::analyze(
180  const Catalog_Namespace::Catalog& catalog,
181  Analyzer::Query& query,
182  TlistRefType allow_tlist_ref) const {
183  throw std::runtime_error("USER literal not supported yet.");
184  return nullptr;
185 }
186 
187 std::shared_ptr<Analyzer::Expr> ArrayLiteral::analyze(
188  const Catalog_Namespace::Catalog& catalog,
189  Analyzer::Query& query,
190  TlistRefType allow_tlist_ref) const {
191  SQLTypeInfo ti = SQLTypeInfo(kARRAY, true);
192  bool set_subtype = true;
193  std::list<std::shared_ptr<Analyzer::Expr>> value_exprs;
194  for (auto& p : value_list) {
195  auto e = p->analyze(catalog, query, allow_tlist_ref);
196  CHECK(e);
197  auto c = std::dynamic_pointer_cast<Analyzer::Constant>(e);
198  if (c != nullptr && c->get_is_null()) {
199  value_exprs.push_back(c);
200  continue;
201  }
202  auto subtype = e->get_type_info().get_type();
203  if (subtype == kNULLT) {
204  // NULL element
205  } else if (set_subtype) {
206  ti.set_subtype(subtype);
207  set_subtype = false;
208  } else {
209  if (ti.get_subtype() != subtype) {
210  throw std::runtime_error("ARRAY element literals should be of the same type.");
211  }
212  }
213  value_exprs.push_back(e);
214  }
215  std::shared_ptr<Analyzer::Expr> result =
216  makeExpr<Analyzer::Constant>(ti, false, value_exprs);
217  return result;
218 }
219 
220 std::string ArrayLiteral::to_string() const {
221  std::string str = "{";
222  bool notfirst = false;
223  for (auto& p : value_list) {
224  if (notfirst) {
225  str += ", ";
226  } else {
227  notfirst = true;
228  }
229  str += p->to_string();
230  }
231  str += "}";
232  return str;
233 }
234 
235 std::shared_ptr<Analyzer::Expr> OperExpr::analyze(
236  const Catalog_Namespace::Catalog& catalog,
237  Analyzer::Query& query,
238  TlistRefType allow_tlist_ref) const {
239  auto left_expr = left->analyze(catalog, query, allow_tlist_ref);
240  const auto& left_type = left_expr->get_type_info();
241  if (right == nullptr) {
242  return makeExpr<Analyzer::UOper>(
243  left_type, left_expr->get_contains_agg(), optype, left_expr->decompress());
244  }
245  if (optype == kARRAY_AT) {
246  if (left_type.get_type() != kARRAY) {
247  throw std::runtime_error(left->to_string() + " is not of array type.");
248  }
249  auto right_expr = right->analyze(catalog, query, allow_tlist_ref);
250  const auto& right_type = right_expr->get_type_info();
251  if (!right_type.is_integer()) {
252  throw std::runtime_error(right->to_string() + " is not of integer type.");
253  }
254  return makeExpr<Analyzer::BinOper>(
255  left_type.get_elem_type(), false, kARRAY_AT, kONE, left_expr, right_expr);
256  }
257  auto right_expr = right->analyze(catalog, query, allow_tlist_ref);
258  return normalize(optype, opqualifier, left_expr, right_expr);
259 }
260 
261 std::shared_ptr<Analyzer::Expr> OperExpr::normalize(
262  const SQLOps optype,
263  const SQLQualifier qual,
264  std::shared_ptr<Analyzer::Expr> left_expr,
265  std::shared_ptr<Analyzer::Expr> right_expr) {
266  if (left_expr->get_type_info().is_date_in_days() ||
267  right_expr->get_type_info().is_date_in_days()) {
268  // Do not propogate encoding
269  left_expr = left_expr->decompress();
270  right_expr = right_expr->decompress();
271  }
272  const auto& left_type = left_expr->get_type_info();
273  auto right_type = right_expr->get_type_info();
274  if (qual != kONE) {
275  // subquery not supported yet.
276  CHECK(!std::dynamic_pointer_cast<Analyzer::Subquery>(right_expr));
277  if (right_type.get_type() != kARRAY) {
278  throw std::runtime_error(
279  "Existential or universal qualifiers can only be used in front of a subquery "
280  "or an "
281  "expression of array type.");
282  }
283  right_type = right_type.get_elem_type();
284  }
285  SQLTypeInfo new_left_type;
286  SQLTypeInfo new_right_type;
287  auto result_type = Analyzer::BinOper::analyze_type_info(
288  optype, left_type, right_type, &new_left_type, &new_right_type);
289  if (result_type.is_timeinterval()) {
290  return makeExpr<Analyzer::BinOper>(
291  result_type, false, optype, qual, left_expr, right_expr);
292  }
293  if (left_type != new_left_type) {
294  left_expr = left_expr->add_cast(new_left_type);
295  }
296  if (right_type != new_right_type) {
297  if (qual == kONE) {
298  right_expr = right_expr->add_cast(new_right_type);
299  } else {
300  right_expr = right_expr->add_cast(new_right_type.get_array_type());
301  }
302  }
303  auto check_compression = IS_COMPARISON(optype);
304  if (check_compression) {
305  if (new_left_type.get_compression() == kENCODING_DICT &&
306  new_right_type.get_compression() == kENCODING_DICT &&
307  new_left_type.get_comp_param() == new_right_type.get_comp_param()) {
308  // do nothing
309  } else if (new_left_type.get_compression() == kENCODING_DICT &&
310  new_right_type.get_compression() == kENCODING_NONE) {
311  SQLTypeInfo ti(new_right_type);
312  ti.set_compression(new_left_type.get_compression());
313  ti.set_comp_param(new_left_type.get_comp_param());
314  ti.set_fixed_size();
315  right_expr = right_expr->add_cast(ti);
316  } else if (new_right_type.get_compression() == kENCODING_DICT &&
317  new_left_type.get_compression() == kENCODING_NONE) {
318  SQLTypeInfo ti(new_left_type);
319  ti.set_compression(new_right_type.get_compression());
320  ti.set_comp_param(new_right_type.get_comp_param());
321  ti.set_fixed_size();
322  left_expr = left_expr->add_cast(ti);
323  } else {
324  left_expr = left_expr->decompress();
325  right_expr = right_expr->decompress();
326  }
327  } else {
328  left_expr = left_expr->decompress();
329  right_expr = right_expr->decompress();
330  }
331  bool has_agg = (left_expr->get_contains_agg() || right_expr->get_contains_agg());
332  return makeExpr<Analyzer::BinOper>(
333  result_type, has_agg, optype, qual, left_expr, right_expr);
334 }
335 
336 std::shared_ptr<Analyzer::Expr> SubqueryExpr::analyze(
337  const Catalog_Namespace::Catalog& catalog,
338  Analyzer::Query& query,
339  TlistRefType allow_tlist_ref) const {
340  throw std::runtime_error("Subqueries are not supported yet.");
341  return nullptr;
342 }
343 
344 std::shared_ptr<Analyzer::Expr> IsNullExpr::analyze(
345  const Catalog_Namespace::Catalog& catalog,
346  Analyzer::Query& query,
347  TlistRefType allow_tlist_ref) const {
348  auto arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
349  auto result = makeExpr<Analyzer::UOper>(kBOOLEAN, kISNULL, arg_expr);
350  if (is_not) {
351  result = makeExpr<Analyzer::UOper>(kBOOLEAN, kNOT, result);
352  }
353  return result;
354 }
355 
356 std::shared_ptr<Analyzer::Expr> InSubquery::analyze(
357  const Catalog_Namespace::Catalog& catalog,
358  Analyzer::Query& query,
359  TlistRefType allow_tlist_ref) const {
360  throw std::runtime_error("Subqueries are not supported yet.");
361  return nullptr;
362 }
363 
364 std::shared_ptr<Analyzer::Expr> InValues::analyze(
365  const Catalog_Namespace::Catalog& catalog,
366  Analyzer::Query& query,
367  TlistRefType allow_tlist_ref) const {
368  auto arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
369  SQLTypeInfo ti = arg_expr->get_type_info();
370  bool dict_comp = ti.get_compression() == kENCODING_DICT;
371  std::list<std::shared_ptr<Analyzer::Expr>> value_exprs;
372  for (auto& p : value_list) {
373  auto e = p->analyze(catalog, query, allow_tlist_ref);
374  if (ti != e->get_type_info()) {
375  if (ti.is_string() && e->get_type_info().is_string()) {
376  ti = Analyzer::BinOper::common_string_type(ti, e->get_type_info());
377  } else if (ti.is_number() && e->get_type_info().is_number()) {
378  ti = Analyzer::BinOper::common_numeric_type(ti, e->get_type_info());
379  } else {
380  throw std::runtime_error("IN expressions must contain compatible types.");
381  }
382  }
383  if (dict_comp) {
384  value_exprs.push_back(e->add_cast(arg_expr->get_type_info()));
385  } else {
386  value_exprs.push_back(e);
387  }
388  }
389  if (!dict_comp) {
390  arg_expr = arg_expr->decompress();
391  arg_expr = arg_expr->add_cast(ti);
392  std::list<std::shared_ptr<Analyzer::Expr>> cast_vals;
393  for (auto p : value_exprs) {
394  cast_vals.push_back(p->add_cast(ti));
395  }
396  value_exprs.swap(cast_vals);
397  }
398  std::shared_ptr<Analyzer::Expr> result =
399  makeExpr<Analyzer::InValues>(arg_expr, value_exprs);
400  if (is_not) {
401  result = makeExpr<Analyzer::UOper>(kBOOLEAN, kNOT, result);
402  }
403  return result;
404 }
405 
406 std::shared_ptr<Analyzer::Expr> BetweenExpr::analyze(
407  const Catalog_Namespace::Catalog& catalog,
408  Analyzer::Query& query,
409  TlistRefType allow_tlist_ref) const {
410  auto arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
411  auto lower_expr = lower->analyze(catalog, query, allow_tlist_ref);
412  auto upper_expr = upper->analyze(catalog, query, allow_tlist_ref);
413  SQLTypeInfo new_left_type, new_right_type;
415  arg_expr->get_type_info(),
416  lower_expr->get_type_info(),
417  &new_left_type,
418  &new_right_type);
419  auto lower_pred =
420  makeExpr<Analyzer::BinOper>(kBOOLEAN,
421  kGE,
422  kONE,
423  arg_expr->add_cast(new_left_type)->decompress(),
424  lower_expr->add_cast(new_right_type)->decompress());
426  arg_expr->get_type_info(),
427  lower_expr->get_type_info(),
428  &new_left_type,
429  &new_right_type);
430  auto upper_pred = makeExpr<Analyzer::BinOper>(
431  kBOOLEAN,
432  kLE,
433  kONE,
434  arg_expr->deep_copy()->add_cast(new_left_type)->decompress(),
435  upper_expr->add_cast(new_right_type)->decompress());
436  std::shared_ptr<Analyzer::Expr> result =
437  makeExpr<Analyzer::BinOper>(kBOOLEAN, kAND, kONE, lower_pred, upper_pred);
438  if (is_not) {
439  result = makeExpr<Analyzer::UOper>(kBOOLEAN, kNOT, result);
440  }
441  return result;
442 }
443 
444 std::shared_ptr<Analyzer::Expr> CharLengthExpr::analyze(
445  const Catalog_Namespace::Catalog& catalog,
446  Analyzer::Query& query,
447  TlistRefType allow_tlist_ref) const {
448  auto arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
449  if (!arg_expr->get_type_info().is_string()) {
450  throw std::runtime_error(
451  "expression in char_length clause must be of a string type.");
452  }
453  std::shared_ptr<Analyzer::Expr> result =
454  makeExpr<Analyzer::CharLengthExpr>(arg_expr->decompress(), calc_encoded_length);
455  return result;
456 }
457 
458 std::shared_ptr<Analyzer::Expr> CardinalityExpr::analyze(
459  const Catalog_Namespace::Catalog& catalog,
460  Analyzer::Query& query,
461  TlistRefType allow_tlist_ref) const {
462  auto arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
463  if (!arg_expr->get_type_info().is_array()) {
464  throw std::runtime_error(
465  "expression in cardinality clause must be of an array type.");
466  }
467  std::shared_ptr<Analyzer::Expr> result =
468  makeExpr<Analyzer::CardinalityExpr>(arg_expr->decompress());
469  return result;
470 }
471 
472 void LikeExpr::check_like_expr(const std::string& like_str, char escape_char) {
473  if (like_str.back() == escape_char) {
474  throw std::runtime_error("LIKE pattern must not end with escape character.");
475  }
476 }
477 
478 bool LikeExpr::test_is_simple_expr(const std::string& like_str, char escape_char) {
479  // if not bounded by '%' then not a simple string
480  if (like_str.size() < 2 || like_str[0] != '%' || like_str[like_str.size() - 1] != '%') {
481  return false;
482  }
483  // if the last '%' is escaped then not a simple string
484  if (like_str[like_str.size() - 2] == escape_char &&
485  like_str[like_str.size() - 3] != escape_char) {
486  return false;
487  }
488  for (size_t i = 1; i < like_str.size() - 1; i++) {
489  if (like_str[i] == '%' || like_str[i] == '_' || like_str[i] == '[' ||
490  like_str[i] == ']') {
491  if (like_str[i - 1] != escape_char) {
492  return false;
493  }
494  }
495  }
496  return true;
497 }
498 
499 void LikeExpr::erase_cntl_chars(std::string& like_str, char escape_char) {
500  char prev_char = '\0';
501  // easier to create new string of allowable chars
502  // rather than erase chars from
503  // existing string
504  std::string new_str;
505  for (char& cur_char : like_str) {
506  if (cur_char == '%' || cur_char == escape_char) {
507  if (prev_char != escape_char) {
508  prev_char = cur_char;
509  continue;
510  }
511  }
512  new_str.push_back(cur_char);
513  prev_char = cur_char;
514  }
515  like_str = new_str;
516 }
517 
518 std::shared_ptr<Analyzer::Expr> LikeExpr::analyze(
519  const Catalog_Namespace::Catalog& catalog,
520  Analyzer::Query& query,
521  TlistRefType allow_tlist_ref) const {
522  auto arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
523  auto like_expr = like_string->analyze(catalog, query, allow_tlist_ref);
524  auto escape_expr = escape_string == nullptr
525  ? nullptr
526  : escape_string->analyze(catalog, query, allow_tlist_ref);
527  return LikeExpr::get(arg_expr, like_expr, escape_expr, is_ilike, is_not);
528 }
529 
530 std::shared_ptr<Analyzer::Expr> LikeExpr::get(std::shared_ptr<Analyzer::Expr> arg_expr,
531  std::shared_ptr<Analyzer::Expr> like_expr,
532  std::shared_ptr<Analyzer::Expr> escape_expr,
533  const bool is_ilike,
534  const bool is_not) {
535  if (!arg_expr->get_type_info().is_string()) {
536  throw std::runtime_error("expression before LIKE must be of a string type.");
537  }
538  if (!like_expr->get_type_info().is_string()) {
539  throw std::runtime_error("expression after LIKE must be of a string type.");
540  }
541  char escape_char = '\\';
542  if (escape_expr != nullptr) {
543  if (!escape_expr->get_type_info().is_string()) {
544  throw std::runtime_error("expression after ESCAPE must be of a string type.");
545  }
546  if (!escape_expr->get_type_info().is_string()) {
547  throw std::runtime_error("expression after ESCAPE must be of a string type.");
548  }
549  auto c = std::dynamic_pointer_cast<Analyzer::Constant>(escape_expr);
550  if (c != nullptr && c->get_constval().stringval->length() > 1) {
551  throw std::runtime_error("String after ESCAPE must have a single character.");
552  }
553  escape_char = (*c->get_constval().stringval)[0];
554  }
555  auto c = std::dynamic_pointer_cast<Analyzer::Constant>(like_expr);
556  bool is_simple = false;
557  if (c != nullptr) {
558  std::string& pattern = *c->get_constval().stringval;
559  if (is_ilike) {
560  std::transform(pattern.begin(), pattern.end(), pattern.begin(), ::tolower);
561  }
562  check_like_expr(pattern, escape_char);
563  is_simple = test_is_simple_expr(pattern, escape_char);
564  if (is_simple) {
565  erase_cntl_chars(pattern, escape_char);
566  }
567  }
568  std::shared_ptr<Analyzer::Expr> result = makeExpr<Analyzer::LikeExpr>(
569  arg_expr->decompress(), like_expr, escape_expr, is_ilike, is_simple);
570  if (is_not) {
571  result = makeExpr<Analyzer::UOper>(kBOOLEAN, kNOT, result);
572  }
573  return result;
574 }
575 
576 void RegexpExpr::check_pattern_expr(const std::string& pattern_str, char escape_char) {
577  if (pattern_str.back() == escape_char) {
578  throw std::runtime_error("REGEXP pattern must not end with escape character.");
579  }
580 }
581 
582 bool RegexpExpr::translate_to_like_pattern(std::string& pattern_str, char escape_char) {
583  char prev_char = '\0';
584  char prev_prev_char = '\0';
585  std::string like_str;
586  for (char& cur_char : pattern_str) {
587  if (prev_char == escape_char || isalnum(cur_char) || cur_char == ' ' ||
588  cur_char == '.') {
589  like_str.push_back((cur_char == '.') ? '_' : cur_char);
590  prev_prev_char = prev_char;
591  prev_char = cur_char;
592  continue;
593  }
594  if (prev_char == '.' && prev_prev_char != escape_char) {
595  if (cur_char == '*' || cur_char == '+') {
596  if (cur_char == '*') {
597  like_str.pop_back();
598  }
599  // .* --> %
600  // .+ --> _%
601  like_str.push_back('%');
602  prev_prev_char = prev_char;
603  prev_char = cur_char;
604  continue;
605  }
606  }
607  return false;
608  }
609  pattern_str = like_str;
610  return true;
611 }
612 
613 std::shared_ptr<Analyzer::Expr> RegexpExpr::analyze(
614  const Catalog_Namespace::Catalog& catalog,
615  Analyzer::Query& query,
616  TlistRefType allow_tlist_ref) const {
617  auto arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
618  auto pattern_expr = pattern_string->analyze(catalog, query, allow_tlist_ref);
619  auto escape_expr = escape_string == nullptr
620  ? nullptr
621  : escape_string->analyze(catalog, query, allow_tlist_ref);
622  return RegexpExpr::get(arg_expr, pattern_expr, escape_expr, is_not);
623 }
624 
625 std::shared_ptr<Analyzer::Expr> RegexpExpr::get(
626  std::shared_ptr<Analyzer::Expr> arg_expr,
627  std::shared_ptr<Analyzer::Expr> pattern_expr,
628  std::shared_ptr<Analyzer::Expr> escape_expr,
629  const bool is_not) {
630  if (!arg_expr->get_type_info().is_string()) {
631  throw std::runtime_error("expression before REGEXP must be of a string type.");
632  }
633  if (!pattern_expr->get_type_info().is_string()) {
634  throw std::runtime_error("expression after REGEXP must be of a string type.");
635  }
636  char escape_char = '\\';
637  if (escape_expr != nullptr) {
638  if (!escape_expr->get_type_info().is_string()) {
639  throw std::runtime_error("expression after ESCAPE must be of a string type.");
640  }
641  if (!escape_expr->get_type_info().is_string()) {
642  throw std::runtime_error("expression after ESCAPE must be of a string type.");
643  }
644  auto c = std::dynamic_pointer_cast<Analyzer::Constant>(escape_expr);
645  if (c != nullptr && c->get_constval().stringval->length() > 1) {
646  throw std::runtime_error("String after ESCAPE must have a single character.");
647  }
648  escape_char = (*c->get_constval().stringval)[0];
649  if (escape_char != '\\') {
650  throw std::runtime_error("Only supporting '\\' escape character.");
651  }
652  }
653  auto c = std::dynamic_pointer_cast<Analyzer::Constant>(pattern_expr);
654  if (c != nullptr) {
655  std::string& pattern = *c->get_constval().stringval;
656  if (translate_to_like_pattern(pattern, escape_char)) {
657  return LikeExpr::get(arg_expr, pattern_expr, escape_expr, false, is_not);
658  }
659  }
660  std::shared_ptr<Analyzer::Expr> result =
661  makeExpr<Analyzer::RegexpExpr>(arg_expr->decompress(), pattern_expr, escape_expr);
662  if (is_not) {
663  result = makeExpr<Analyzer::UOper>(kBOOLEAN, kNOT, result);
664  }
665  return result;
666 }
667 
668 std::shared_ptr<Analyzer::Expr> LikelihoodExpr::analyze(
669  const Catalog_Namespace::Catalog& catalog,
670  Analyzer::Query& query,
671  TlistRefType allow_tlist_ref) const {
672  auto arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
673  return LikelihoodExpr::get(arg_expr, likelihood, is_not);
674 }
675 
676 std::shared_ptr<Analyzer::Expr> LikelihoodExpr::get(
677  std::shared_ptr<Analyzer::Expr> arg_expr,
678  float likelihood,
679  const bool is_not) {
680  if (!arg_expr->get_type_info().is_boolean()) {
681  throw std::runtime_error("likelihood expression expects boolean type.");
682  }
683  std::shared_ptr<Analyzer::Expr> result = makeExpr<Analyzer::LikelihoodExpr>(
684  arg_expr->decompress(), is_not ? 1 - likelihood : likelihood);
685  return result;
686 }
687 
688 std::shared_ptr<Analyzer::Expr> ExistsExpr::analyze(
689  const Catalog_Namespace::Catalog& catalog,
690  Analyzer::Query& query,
691  TlistRefType allow_tlist_ref) const {
692  throw std::runtime_error("Subqueries are not supported yet.");
693  return nullptr;
694 }
695 
696 std::shared_ptr<Analyzer::Expr> ColumnRef::analyze(
697  const Catalog_Namespace::Catalog& catalog,
698  Analyzer::Query& query,
699  TlistRefType allow_tlist_ref) const {
700  int table_id{0};
701  int rte_idx{0};
702  const ColumnDescriptor* cd{nullptr};
703  if (column == nullptr) {
704  throw std::runtime_error("invalid column name *.");
705  }
706  if (table != nullptr) {
707  rte_idx = query.get_rte_idx(*table);
708  if (rte_idx < 0) {
709  throw std::runtime_error("range variable or table name " + *table +
710  " does not exist.");
711  }
712  Analyzer::RangeTableEntry* rte = query.get_rte(rte_idx);
713  cd = rte->get_column_desc(catalog, *column);
714  if (cd == nullptr) {
715  throw std::runtime_error("Column name " + *column + " does not exist.");
716  }
717  table_id = rte->get_table_id();
718  } else {
719  bool found = false;
720  int i = 0;
721  for (auto rte : query.get_rangetable()) {
722  cd = rte->get_column_desc(catalog, *column);
723  if (cd != nullptr && !found) {
724  found = true;
725  rte_idx = i;
726  table_id = rte->get_table_id();
727  } else if (cd != nullptr && found) {
728  throw std::runtime_error("Column name " + *column + " is ambiguous.");
729  }
730  i++;
731  }
732  if (cd == nullptr && allow_tlist_ref != TlistRefType::TLIST_NONE) {
733  // check if this is a reference to a targetlist entry
734  bool found = false;
735  int varno = -1;
736  int i = 1;
737  std::shared_ptr<Analyzer::TargetEntry> tle;
738  for (auto p : query.get_targetlist()) {
739  if (*column == p->get_resname() && !found) {
740  found = true;
741  varno = i;
742  tle = p;
743  } else if (*column == p->get_resname() && found) {
744  throw std::runtime_error("Output alias " + *column + " is ambiguous.");
745  }
746  i++;
747  }
748  if (found) {
749  if (dynamic_cast<Analyzer::Var*>(tle->get_expr())) {
750  Analyzer::Var* v = static_cast<Analyzer::Var*>(tle->get_expr());
752  return v->deep_copy();
753  }
754  }
755  if (allow_tlist_ref == TlistRefType::TLIST_COPY) {
756  return tle->get_expr()->deep_copy();
757  } else {
758  return makeExpr<Analyzer::Var>(
759  tle->get_expr()->get_type_info(), Analyzer::Var::kOUTPUT, varno);
760  }
761  }
762  }
763  if (cd == nullptr) {
764  throw std::runtime_error("Column name " + *column + " does not exist.");
765  }
766  }
767  return makeExpr<Analyzer::ColumnVar>(cd->columnType, table_id, cd->columnId, rte_idx);
768 }
769 
770 std::shared_ptr<Analyzer::Expr> FunctionRef::analyze(
771  const Catalog_Namespace::Catalog& catalog,
772  Analyzer::Query& query,
773  TlistRefType allow_tlist_ref) const {
774  SQLTypeInfo result_type;
775  SQLAgg agg_type;
776  std::shared_ptr<Analyzer::Expr> arg_expr;
777  bool is_distinct = false;
778  if (boost::iequals(*name, "count")) {
779  result_type = SQLTypeInfo(kBIGINT, false);
780  agg_type = kCOUNT;
781  if (arg) {
782  arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
783  const SQLTypeInfo& ti = arg_expr->get_type_info();
784  if (ti.is_string() && (ti.get_compression() != kENCODING_DICT || !distinct)) {
785  throw std::runtime_error(
786  "Strings must be dictionary-encoded in COUNT(DISTINCT).");
787  }
788  if (ti.get_type() == kARRAY && !distinct) {
789  throw std::runtime_error("Only COUNT(DISTINCT) is supported on arrays.");
790  }
791  }
792  is_distinct = distinct;
793  } else {
794  if (!arg) {
795  throw std::runtime_error("Cannot compute " + *name + " with argument '*'.");
796  }
797  if (boost::iequals(*name, "min")) {
798  agg_type = kMIN;
799  arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
800  arg_expr = arg_expr->decompress();
801  result_type = arg_expr->get_type_info();
802  } else if (boost::iequals(*name, "max")) {
803  agg_type = kMAX;
804  arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
805  arg_expr = arg_expr->decompress();
806  result_type = arg_expr->get_type_info();
807  } else if (boost::iequals(*name, "avg")) {
808  agg_type = kAVG;
809  arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
810  if (!arg_expr->get_type_info().is_number()) {
811  throw std::runtime_error("Cannot compute AVG on non-number-type arguments.");
812  }
813  arg_expr = arg_expr->decompress();
814  result_type = SQLTypeInfo(kDOUBLE, false);
815  } else if (boost::iequals(*name, "sum")) {
816  agg_type = kSUM;
817  arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
818  if (!arg_expr->get_type_info().is_number()) {
819  throw std::runtime_error("Cannot compute SUM on non-number-type arguments.");
820  }
821  arg_expr = arg_expr->decompress();
822  result_type = arg_expr->get_type_info().is_integer() ? SQLTypeInfo(kBIGINT, false)
823  : arg_expr->get_type_info();
824  } else if (boost::iequals(*name, "unnest")) {
825  arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
826  const SQLTypeInfo& arg_ti = arg_expr->get_type_info();
827  if (arg_ti.get_type() != kARRAY) {
828  throw std::runtime_error(arg->to_string() + " is not of array type.");
829  }
830  return makeExpr<Analyzer::UOper>(arg_ti.get_elem_type(), false, kUNNEST, arg_expr);
831  } else {
832  throw std::runtime_error("invalid function name: " + *name);
833  }
834  if (arg_expr->get_type_info().is_string() ||
835  arg_expr->get_type_info().get_type() == kARRAY) {
836  throw std::runtime_error(
837  "Only COUNT(DISTINCT ) aggregate is supported on strings and arrays.");
838  }
839  }
840  int naggs = query.get_num_aggs();
841  query.set_num_aggs(naggs + 1);
842  return makeExpr<Analyzer::AggExpr>(
843  result_type, agg_type, arg_expr, is_distinct, nullptr);
844 }
845 
846 std::shared_ptr<Analyzer::Expr> CastExpr::analyze(
847  const Catalog_Namespace::Catalog& catalog,
848  Analyzer::Query& query,
849  TlistRefType allow_tlist_ref) const {
850  target_type->check_type();
851  auto arg_expr = arg->analyze(catalog, query, allow_tlist_ref);
852  SQLTypeInfo ti(target_type->get_type(),
853  target_type->get_param1(),
854  target_type->get_param2(),
855  arg_expr->get_type_info().get_notnull());
856  if (arg_expr->get_type_info().get_type() != target_type->get_type() &&
857  arg_expr->get_type_info().get_compression() != kENCODING_NONE) {
858  arg_expr->decompress();
859  }
860  return arg_expr->add_cast(ti);
861 }
862 
863 std::shared_ptr<Analyzer::Expr> CaseExpr::analyze(
864  const Catalog_Namespace::Catalog& catalog,
865  Analyzer::Query& query,
866  TlistRefType allow_tlist_ref) const {
867  SQLTypeInfo ti;
868  std::list<std::pair<std::shared_ptr<Analyzer::Expr>, std::shared_ptr<Analyzer::Expr>>>
869  expr_pair_list;
870  for (auto& p : when_then_list) {
871  auto e1 = p->get_expr1()->analyze(catalog, query, allow_tlist_ref);
872  if (e1->get_type_info().get_type() != kBOOLEAN) {
873  throw std::runtime_error("Only boolean expressions can be used after WHEN.");
874  }
875  auto e2 = p->get_expr2()->analyze(catalog, query, allow_tlist_ref);
876  expr_pair_list.emplace_back(e1, e2);
877  }
878  auto else_e = else_expr ? else_expr->analyze(catalog, query, allow_tlist_ref) : nullptr;
879  return normalize(expr_pair_list, else_e);
880 }
881 
882 namespace {
883 
884 bool expr_is_null(const Analyzer::Expr* expr) {
885  if (expr->get_type_info().get_type() == kNULLT) {
886  return true;
887  }
888  const auto const_expr = dynamic_cast<const Analyzer::Constant*>(expr);
889  return const_expr && const_expr->get_is_null();
890 }
891 
893  const std::string* s = str_literal->get_stringval();
894  if (*s == "t" || *s == "true" || *s == "T" || *s == "True") {
895  return true;
896  } else if (*s == "f" || *s == "false" || *s == "F" || *s == "False") {
897  return false;
898  } else {
899  throw std::runtime_error("Invalid string for boolean " + *s);
900  }
901 }
902 
903 } // namespace
904 
905 std::shared_ptr<Analyzer::Expr> CaseExpr::normalize(
906  const std::list<std::pair<std::shared_ptr<Analyzer::Expr>,
907  std::shared_ptr<Analyzer::Expr>>>& expr_pair_list,
908  const std::shared_ptr<Analyzer::Expr> else_e_in) {
909  SQLTypeInfo ti;
910  bool has_agg = false;
911  for (auto& p : expr_pair_list) {
912  auto e1 = p.first;
913  CHECK(e1->get_type_info().is_boolean());
914  auto e2 = p.second;
915  if (ti.get_type() == kNULLT) {
916  ti = e2->get_type_info();
917  } else if (e2->get_type_info().get_type() == kNULLT) {
918  ti.set_notnull(false);
919  e2->set_type_info(ti);
920  } else if (ti != e2->get_type_info()) {
921  if (ti.is_string() && e2->get_type_info().is_string()) {
922  ti = Analyzer::BinOper::common_string_type(ti, e2->get_type_info());
923  } else if (ti.is_number() && e2->get_type_info().is_number()) {
924  ti = Analyzer::BinOper::common_numeric_type(ti, e2->get_type_info());
925  } else if (ti.is_boolean() && e2->get_type_info().is_boolean()) {
926  ti = Analyzer::BinOper::common_numeric_type(ti, e2->get_type_info());
927  } else {
928  throw std::runtime_error(
929  "expressions in THEN clause must be of the same or compatible types.");
930  }
931  }
932  if (e2->get_contains_agg()) {
933  has_agg = true;
934  }
935  }
936  auto else_e = else_e_in;
937  if (else_e) {
938  if (else_e->get_contains_agg()) {
939  has_agg = true;
940  }
941  if (expr_is_null(else_e.get())) {
942  ti.set_notnull(false);
943  else_e->set_type_info(ti);
944  } else if (ti != else_e->get_type_info()) {
945  ti.set_notnull(false);
946  if (ti.is_string() && else_e->get_type_info().is_string()) {
947  ti = Analyzer::BinOper::common_string_type(ti, else_e->get_type_info());
948  } else if (ti.is_number() && else_e->get_type_info().is_number()) {
949  ti = Analyzer::BinOper::common_numeric_type(ti, else_e->get_type_info());
950  } else if (ti.is_boolean() && else_e->get_type_info().is_boolean()) {
951  ti = Analyzer::BinOper::common_numeric_type(ti, else_e->get_type_info());
952  } else {
953  throw std::runtime_error(
954  "expressions in ELSE clause must be of the same or compatible types as those "
955  "in the THEN clauses.");
956  }
957  }
958  }
959  std::list<std::pair<std::shared_ptr<Analyzer::Expr>, std::shared_ptr<Analyzer::Expr>>>
960  cast_expr_pair_list;
961  for (auto p : expr_pair_list) {
962  ti.set_notnull(false);
963  cast_expr_pair_list.emplace_back(p.first, p.second->add_cast(ti));
964  }
965  if (else_e != nullptr) {
966  else_e = else_e->add_cast(ti);
967  } else {
968  Datum d;
969  // always create an else expr so that executor doesn't need to worry about it
970  ti.set_notnull(false);
971  else_e = makeExpr<Analyzer::Constant>(ti, true, d);
972  }
973  if (ti.get_type() == kNULLT) {
974  throw std::runtime_error(
975  "Can't deduce the type for case expressions, all branches null");
976  }
977  return makeExpr<Analyzer::CaseExpr>(ti, has_agg, cast_expr_pair_list, else_e);
978 }
979 
980 std::string CaseExpr::to_string() const {
981  std::string str("CASE ");
982  for (auto& p : when_then_list) {
983  str += "WHEN " + p->get_expr1()->to_string() + " THEN " +
984  p->get_expr2()->to_string() + " ";
985  }
986  if (else_expr != nullptr) {
987  str += "ELSE " + else_expr->to_string();
988  }
989  str += " END";
990  return str;
991 }
992 
993 void UnionQuery::analyze(const Catalog_Namespace::Catalog& catalog,
994  Analyzer::Query& query) const {
995  left->analyze(catalog, query);
996  Analyzer::Query* right_query = new Analyzer::Query();
997  right->analyze(catalog, *right_query);
998  query.set_next_query(right_query);
999  query.set_is_unionall(is_unionall);
1000 }
1001 
1002 void QuerySpec::analyze_having_clause(const Catalog_Namespace::Catalog& catalog,
1003  Analyzer::Query& query) const {
1004  std::shared_ptr<Analyzer::Expr> p;
1005  if (having_clause != nullptr) {
1006  p = having_clause->analyze(catalog, query, Expr::TlistRefType::TLIST_COPY);
1007  if (p->get_type_info().get_type() != kBOOLEAN) {
1008  throw std::runtime_error("Only boolean expressions can be in HAVING clause.");
1009  }
1010  p->check_group_by(query.get_group_by());
1011  }
1012  query.set_having_predicate(p);
1013 }
1014 
1015 void QuerySpec::analyze_group_by(const Catalog_Namespace::Catalog& catalog,
1016  Analyzer::Query& query) const {
1017  std::list<std::shared_ptr<Analyzer::Expr>> groupby;
1018  if (!groupby_clause.empty()) {
1019  int gexpr_no = 1;
1020  std::shared_ptr<Analyzer::Expr> gexpr;
1021  const std::vector<std::shared_ptr<Analyzer::TargetEntry>>& tlist =
1022  query.get_targetlist();
1023  for (auto& c : groupby_clause) {
1024  // special-case ordinal numbers in GROUP BY
1025  if (dynamic_cast<Literal*>(c.get())) {
1026  IntLiteral* i = dynamic_cast<IntLiteral*>(c.get());
1027  if (!i) {
1028  throw std::runtime_error("Invalid literal in GROUP BY clause.");
1029  }
1030  int varno = (int)i->get_intval();
1031  if (varno <= 0 || varno > static_cast<int>(tlist.size())) {
1032  throw std::runtime_error("Invalid ordinal number in GROUP BY clause.");
1033  }
1034  if (tlist[varno - 1]->get_expr()->get_contains_agg()) {
1035  throw std::runtime_error(
1036  "Ordinal number in GROUP BY cannot reference an expression containing "
1037  "aggregate "
1038  "functions.");
1039  }
1040  gexpr = makeExpr<Analyzer::Var>(
1041  tlist[varno - 1]->get_expr()->get_type_info(), Analyzer::Var::kOUTPUT, varno);
1042  } else {
1043  gexpr = c->analyze(catalog, query, Expr::TlistRefType::TLIST_REF);
1044  }
1045  const SQLTypeInfo gti = gexpr->get_type_info();
1046  bool set_new_type = false;
1047  SQLTypeInfo ti(gti);
1048  if (gti.is_string() && gti.get_compression() == kENCODING_NONE) {
1049  set_new_type = true;
1052  ti.set_fixed_size();
1053  }
1054  std::shared_ptr<Analyzer::Var> v;
1055  if (std::dynamic_pointer_cast<Analyzer::Var>(gexpr)) {
1056  v = std::static_pointer_cast<Analyzer::Var>(gexpr);
1057  int n = v->get_varno();
1058  gexpr = tlist[n - 1]->get_own_expr();
1059  auto cv = std::dynamic_pointer_cast<Analyzer::ColumnVar>(gexpr);
1060  if (cv != nullptr) {
1061  // inherit all ColumnVar info for lineage.
1062  *std::static_pointer_cast<Analyzer::ColumnVar>(v) = *cv;
1063  }
1064  v->set_which_row(Analyzer::Var::kGROUPBY);
1065  v->set_varno(gexpr_no);
1066  tlist[n - 1]->set_expr(v);
1067  }
1068  if (set_new_type) {
1069  auto new_e = gexpr->add_cast(ti);
1070  groupby.push_back(new_e);
1071  if (v != nullptr) {
1072  v->set_type_info(new_e->get_type_info());
1073  }
1074  } else {
1075  groupby.push_back(gexpr);
1076  }
1077  gexpr_no++;
1078  }
1079  }
1080  if (query.get_num_aggs() > 0 || !groupby.empty()) {
1081  for (auto t : query.get_targetlist()) {
1082  auto e = t->get_expr();
1083  e->check_group_by(groupby);
1084  }
1085  }
1086  query.set_group_by(groupby);
1087 }
1088 
1089 void QuerySpec::analyze_where_clause(const Catalog_Namespace::Catalog& catalog,
1090  Analyzer::Query& query) const {
1091  if (where_clause == nullptr) {
1092  query.set_where_predicate(nullptr);
1093  return;
1094  }
1095  auto p = where_clause->analyze(catalog, query, Expr::TlistRefType::TLIST_COPY);
1096  if (p->get_type_info().get_type() != kBOOLEAN) {
1097  throw std::runtime_error("Only boolean expressions can be in WHERE clause.");
1098  }
1099  query.set_where_predicate(p);
1100 }
1101 
1102 void QuerySpec::analyze_select_clause(const Catalog_Namespace::Catalog& catalog,
1103  Analyzer::Query& query) const {
1104  std::vector<std::shared_ptr<Analyzer::TargetEntry>>& tlist =
1105  query.get_targetlist_nonconst();
1106  if (select_clause.empty()) {
1107  // this means SELECT *
1108  int rte_idx = 0;
1109  for (auto rte : query.get_rangetable()) {
1110  rte->expand_star_in_targetlist(catalog, tlist, rte_idx++);
1111  }
1112  } else {
1113  for (auto& p : select_clause) {
1114  const Parser::Expr* select_expr = p->get_select_expr();
1115  // look for the case of range_var.*
1116  if (typeid(*select_expr) == typeid(ColumnRef) &&
1117  dynamic_cast<const ColumnRef*>(select_expr)->get_column() == nullptr) {
1118  const std::string* range_var_name =
1119  dynamic_cast<const ColumnRef*>(select_expr)->get_table();
1120  int rte_idx = query.get_rte_idx(*range_var_name);
1121  if (rte_idx < 0) {
1122  throw std::runtime_error("invalid range variable name: " + *range_var_name);
1123  }
1124  Analyzer::RangeTableEntry* rte = query.get_rte(rte_idx);
1125  rte->expand_star_in_targetlist(catalog, tlist, rte_idx);
1126  } else {
1127  auto e = select_expr->analyze(catalog, query);
1128  std::string resname;
1129 
1130  if (p->get_alias() != nullptr) {
1131  resname = *p->get_alias();
1132  } else if (std::dynamic_pointer_cast<Analyzer::ColumnVar>(e) &&
1133  !std::dynamic_pointer_cast<Analyzer::Var>(e)) {
1134  auto colvar = std::static_pointer_cast<Analyzer::ColumnVar>(e);
1135  const ColumnDescriptor* col_desc = catalog.getMetadataForColumn(
1136  colvar->get_table_id(), colvar->get_column_id());
1137  resname = col_desc->columnName;
1138  }
1139  if (e->get_type_info().get_type() == kNULLT) {
1140  throw std::runtime_error(
1141  "Untyped NULL in SELECT clause. Use CAST to specify a type.");
1142  }
1143  auto o = std::static_pointer_cast<Analyzer::UOper>(e);
1144  bool unnest = (o != nullptr && o->get_optype() == kUNNEST);
1145  auto tle = std::make_shared<Analyzer::TargetEntry>(resname, e, unnest);
1146  tlist.push_back(tle);
1147  }
1148  }
1149  }
1150 }
1151 
1152 void QuerySpec::analyze_from_clause(const Catalog_Namespace::Catalog& catalog,
1153  Analyzer::Query& query) const {
1155  for (auto& p : from_clause) {
1156  const TableDescriptor* table_desc;
1157  table_desc = catalog.getMetadataForTable(*p->get_table_name());
1158  if (table_desc == nullptr) {
1159  throw std::runtime_error("Table " + *p->get_table_name() + " does not exist.");
1160  }
1161  std::string range_var;
1162  if (p->get_range_var() == nullptr) {
1163  range_var = *p->get_table_name();
1164  } else {
1165  range_var = *p->get_range_var();
1166  }
1167  rte = new Analyzer::RangeTableEntry(range_var, table_desc, nullptr);
1168  query.add_rte(rte);
1169  }
1170 }
1171 
1172 void QuerySpec::analyze(const Catalog_Namespace::Catalog& catalog,
1173  Analyzer::Query& query) const {
1175  analyze_from_clause(catalog, query);
1176  analyze_select_clause(catalog, query);
1177  analyze_where_clause(catalog, query);
1178  analyze_group_by(catalog, query);
1179  analyze_having_clause(catalog, query);
1180 }
1181 
1182 void SelectStmt::analyze(const Catalog_Namespace::Catalog& catalog,
1183  Analyzer::Query& query) const {
1184  query.set_stmt_type(kSELECT);
1185  query.set_limit(limit);
1186  if (offset < 0) {
1187  throw std::runtime_error("OFFSET cannot be negative.");
1188  }
1189  query.set_offset(offset);
1190  query_expr->analyze(catalog, query);
1191  if (orderby_clause.empty() && !query.get_is_distinct()) {
1192  query.set_order_by(nullptr);
1193  return;
1194  }
1195  const std::vector<std::shared_ptr<Analyzer::TargetEntry>>& tlist =
1196  query.get_targetlist();
1197  std::list<Analyzer::OrderEntry>* order_by = new std::list<Analyzer::OrderEntry>();
1198  if (!orderby_clause.empty()) {
1199  for (auto& p : orderby_clause) {
1200  int tle_no = p->get_colno();
1201  if (tle_no == 0) {
1202  // use column name
1203  // search through targetlist for matching name
1204  const std::string* name = p->get_column()->get_column();
1205  tle_no = 1;
1206  bool found = false;
1207  for (auto tle : tlist) {
1208  if (tle->get_resname() == *name) {
1209  found = true;
1210  break;
1211  }
1212  tle_no++;
1213  }
1214  if (!found) {
1215  throw std::runtime_error("invalid name in order by: " + *name);
1216  }
1217  }
1218  order_by->push_back(
1219  Analyzer::OrderEntry(tle_no, p->get_is_desc(), p->get_nulls_first()));
1220  }
1221  }
1222  if (query.get_is_distinct()) {
1223  // extend order_by to include all targetlist entries.
1224  for (int i = 1; i <= static_cast<int>(tlist.size()); i++) {
1225  bool in_orderby = false;
1226  std::for_each(order_by->begin(),
1227  order_by->end(),
1228  [&in_orderby, i](const Analyzer::OrderEntry& oe) {
1229  in_orderby = in_orderby || (i == oe.tle_no);
1230  });
1231  if (!in_orderby) {
1232  order_by->push_back(Analyzer::OrderEntry(i, false, false));
1233  }
1234  }
1235  }
1236  query.set_order_by(order_by);
1237 }
1238 
1239 std::string SQLType::to_string() const {
1240  std::string str;
1241  switch (type) {
1242  case kBOOLEAN:
1243  str = "BOOLEAN";
1244  break;
1245  case kCHAR:
1246  str = "CHAR(" + boost::lexical_cast<std::string>(param1) + ")";
1247  break;
1248  case kVARCHAR:
1249  str = "VARCHAR(" + boost::lexical_cast<std::string>(param1) + ")";
1250  break;
1251  case kTEXT:
1252  str = "TEXT";
1253  break;
1254  case kNUMERIC:
1255  str = "NUMERIC(" + boost::lexical_cast<std::string>(param1);
1256  if (param2 > 0) {
1257  str += ", " + boost::lexical_cast<std::string>(param2);
1258  }
1259  str += ")";
1260  break;
1261  case kDECIMAL:
1262  str = "DECIMAL(" + boost::lexical_cast<std::string>(param1);
1263  if (param2 > 0) {
1264  str += ", " + boost::lexical_cast<std::string>(param2);
1265  }
1266  str += ")";
1267  break;
1268  case kBIGINT:
1269  str = "BIGINT";
1270  break;
1271  case kINT:
1272  str = "INT";
1273  break;
1274  case kTINYINT:
1275  str = "TINYINT";
1276  break;
1277  case kSMALLINT:
1278  str = "SMALLINT";
1279  break;
1280  case kFLOAT:
1281  str = "FLOAT";
1282  break;
1283  case kDOUBLE:
1284  str = "DOUBLE";
1285  break;
1286  case kTIME:
1287  str = "TIME";
1288  if (param1 < 6) {
1289  str += "(" + boost::lexical_cast<std::string>(param1) + ")";
1290  }
1291  break;
1292  case kTIMESTAMP:
1293  str = "TIMESTAMP";
1294  if (param1 <= 9) {
1295  str += "(" + boost::lexical_cast<std::string>(param1) + ")";
1296  }
1297  break;
1298  case kDATE:
1299  str = "DATE";
1300  break;
1301  default:
1302  assert(false);
1303  break;
1304  }
1305  if (is_array) {
1306  str += "[";
1307  if (array_size > 0) {
1308  str += boost::lexical_cast<std::string>(array_size);
1309  }
1310  str += "]";
1311  }
1312  return str;
1313 }
1314 
1315 std::string SelectEntry::to_string() const {
1316  std::string str = select_expr->to_string();
1317  if (alias != nullptr) {
1318  str += " AS " + *alias;
1319  }
1320  return str;
1321 }
1322 
1323 std::string TableRef::to_string() const {
1324  std::string str = *table_name;
1325  if (range_var != nullptr) {
1326  str += " " + *range_var;
1327  }
1328  return str;
1329 }
1330 
1331 std::string ColumnRef::to_string() const {
1332  std::string str;
1333  if (table == nullptr) {
1334  str = *column;
1335  } else if (column == nullptr) {
1336  str = *table + ".*";
1337  } else {
1338  str = *table + "." + *column;
1339  }
1340  return str;
1341 }
1342 
1343 std::string OperExpr::to_string() const {
1344  std::string op_str[] = {
1345  "=", "===", "<>", "<", ">", "<=", ">=", " AND ", " OR ", "NOT", "-", "+", "*", "/"};
1346  std::string str;
1347  if (optype == kUMINUS) {
1348  str = "-(" + left->to_string() + ")";
1349  } else if (optype == kNOT) {
1350  str = "NOT (" + left->to_string() + ")";
1351  } else if (optype == kARRAY_AT) {
1352  str = left->to_string() + "[" + right->to_string() + "]";
1353  } else if (optype == kUNNEST) {
1354  str = "UNNEST(" + left->to_string() + ")";
1355  } else if (optype == kIN) {
1356  str = "(" + left->to_string() + " IN " + right->to_string() + ")";
1357  } else {
1358  str = "(" + left->to_string() + op_str[optype] + right->to_string() + ")";
1359  }
1360  return str;
1361 }
1362 
1363 std::string InExpr::to_string() const {
1364  std::string str = arg->to_string();
1365  if (is_not) {
1366  str += " NOT IN ";
1367  } else {
1368  str += " IN ";
1369  }
1370  return str;
1371 }
1372 
1373 std::string ExistsExpr::to_string() const {
1374  return "EXISTS (" + query->to_string() + ")";
1375 }
1376 
1377 std::string SubqueryExpr::to_string() const {
1378  std::string str;
1379  str = "(";
1380  str += query->to_string();
1381  str += ")";
1382  return str;
1383 }
1384 
1385 std::string IsNullExpr::to_string() const {
1386  std::string str = arg->to_string();
1387  if (is_not) {
1388  str += " IS NOT NULL";
1389  } else {
1390  str += " IS NULL";
1391  }
1392  return str;
1393 }
1394 
1395 std::string InSubquery::to_string() const {
1396  std::string str = InExpr::to_string();
1397  str += subquery->to_string();
1398  return str;
1399 }
1400 
1401 std::string InValues::to_string() const {
1402  std::string str = InExpr::to_string() + "(";
1403  bool notfirst = false;
1404  for (auto& p : value_list) {
1405  if (notfirst) {
1406  str += ", ";
1407  } else {
1408  notfirst = true;
1409  }
1410  str += p->to_string();
1411  }
1412  str += ")";
1413  return str;
1414 }
1415 
1416 std::string BetweenExpr::to_string() const {
1417  std::string str = arg->to_string();
1418  if (is_not) {
1419  str += " NOT BETWEEN ";
1420  } else {
1421  str += " BETWEEN ";
1422  }
1423  str += lower->to_string() + " AND " + upper->to_string();
1424  return str;
1425 }
1426 
1427 std::string CharLengthExpr::to_string() const {
1428  std::string str;
1429  if (calc_encoded_length) {
1430  str = "CHAR_LENGTH (" + arg->to_string() + ")";
1431  } else {
1432  str = "LENGTH (" + arg->to_string() + ")";
1433  }
1434  return str;
1435 }
1436 
1437 std::string CardinalityExpr::to_string() const {
1438  std::string str = "CARDINALITY(" + arg->to_string() + ")";
1439  return str;
1440 }
1441 
1442 std::string LikeExpr::to_string() const {
1443  std::string str = arg->to_string();
1444  if (is_not) {
1445  str += " NOT LIKE ";
1446  } else {
1447  str += " LIKE ";
1448  }
1449  str += like_string->to_string();
1450  if (escape_string != nullptr) {
1451  str += " ESCAPE " + escape_string->to_string();
1452  }
1453  return str;
1454 }
1455 
1456 std::string RegexpExpr::to_string() const {
1457  std::string str = arg->to_string();
1458  if (is_not) {
1459  str += " NOT REGEXP ";
1460  } else {
1461  str += " REGEXP ";
1462  }
1463  str += pattern_string->to_string();
1464  if (escape_string != nullptr) {
1465  str += " ESCAPE " + escape_string->to_string();
1466  }
1467  return str;
1468 }
1469 
1470 std::string LikelihoodExpr::to_string() const {
1471  std::string str = " LIKELIHOOD ";
1472  str += arg->to_string();
1473  str += " ";
1474  str += boost::lexical_cast<std::string>(is_not ? 1.0 - likelihood : likelihood);
1475  return str;
1476 }
1477 
1478 std::string FunctionRef::to_string() const {
1479  std::string str = *name + "(";
1480  if (distinct) {
1481  str += "DISTINCT ";
1482  }
1483  if (arg == nullptr) {
1484  str += "*)";
1485  } else {
1486  str += arg->to_string() + ")";
1487  }
1488  return str;
1489 }
1490 
1491 std::string QuerySpec::to_string() const {
1492  std::string query_str = "SELECT ";
1493  if (is_distinct) {
1494  query_str += "DISTINCT ";
1495  }
1496  if (select_clause.empty()) {
1497  query_str += "* ";
1498  } else {
1499  bool notfirst = false;
1500  for (auto& p : select_clause) {
1501  if (notfirst) {
1502  query_str += ", ";
1503  } else {
1504  notfirst = true;
1505  }
1506  query_str += p->to_string();
1507  }
1508  }
1509  query_str += " FROM ";
1510  bool notfirst = false;
1511  for (auto& p : from_clause) {
1512  if (notfirst) {
1513  query_str += ", ";
1514  } else {
1515  notfirst = true;
1516  }
1517  query_str += p->to_string();
1518  }
1519  if (where_clause) {
1520  query_str += " WHERE " + where_clause->to_string();
1521  }
1522  if (!groupby_clause.empty()) {
1523  query_str += " GROUP BY ";
1524  bool notfirst = false;
1525  for (auto& p : groupby_clause) {
1526  if (notfirst) {
1527  query_str += ", ";
1528  } else {
1529  notfirst = true;
1530  }
1531  query_str += p->to_string();
1532  }
1533  }
1534  if (having_clause) {
1535  query_str += " HAVING " + having_clause->to_string();
1536  }
1537  query_str += ";";
1538  return query_str;
1539 }
1540 
1541 void InsertStmt::analyze(const Catalog_Namespace::Catalog& catalog,
1542  Analyzer::Query& query) const {
1543  query.set_stmt_type(kINSERT);
1544  const TableDescriptor* td = catalog.getMetadataForTable(*table);
1545  if (td == nullptr) {
1546  throw std::runtime_error("Table " + *table + " does not exist.");
1547  }
1548  if (td->isView) {
1549  throw std::runtime_error("Insert to views is not supported yet.");
1550  }
1551  query.set_result_table_id(td->tableId);
1552  std::list<int> result_col_list;
1553  if (column_list.empty()) {
1554  const std::list<const ColumnDescriptor*> all_cols =
1555  catalog.getAllColumnMetadataForTable(td->tableId, false, false, true);
1556  for (auto cd : all_cols) {
1557  result_col_list.push_back(cd->columnId);
1558  }
1559  } else {
1560  for (auto& c : column_list) {
1561  const ColumnDescriptor* cd = catalog.getMetadataForColumn(td->tableId, *c);
1562  if (cd == nullptr) {
1563  throw std::runtime_error("Column " + *c + " does not exist.");
1564  }
1565  result_col_list.push_back(cd->columnId);
1566  const auto& col_ti = cd->columnType;
1567  if (col_ti.get_physical_cols() > 0) {
1568  CHECK(cd->columnType.is_geometry());
1569  for (auto i = 1; i <= col_ti.get_physical_cols(); i++) {
1570  const ColumnDescriptor* pcd =
1571  catalog.getMetadataForColumn(td->tableId, cd->columnId + i);
1572  if (pcd == nullptr) {
1573  throw std::runtime_error("Column " + *c + "'s metadata is incomplete.");
1574  }
1575  result_col_list.push_back(pcd->columnId);
1576  }
1577  }
1578  }
1579  if (catalog.getAllColumnMetadataForTable(td->tableId, false, false, true).size() !=
1580  result_col_list.size()) {
1581  throw std::runtime_error("Insert into a subset of columns is not supported yet.");
1582  }
1583  }
1584  query.set_result_col_list(result_col_list);
1585 }
1586 
1587 size_t InsertValuesStmt::determineLeafIndex(const Catalog_Namespace::Catalog& catalog,
1588  size_t num_leafs) {
1589  const TableDescriptor* td = catalog.getMetadataForTable(*table);
1590  if (td == nullptr) {
1591  throw std::runtime_error("Table " + *table + " does not exist.");
1592  }
1593  if (td->isView) {
1594  throw std::runtime_error("Insert to views is not supported yet.");
1595  }
1596 
1597  if (td->partitions == "REPLICATED") {
1598  throw std::runtime_error("Cannot determine leaf on replicated table.");
1599  }
1600 
1601  if (0 == td->nShards) {
1602  std::random_device rd;
1603  std::mt19937_64 gen(rd());
1604  std::uniform_int_distribution<size_t> dis;
1605  const auto leaf_idx = dis(gen) % num_leafs;
1606  return leaf_idx;
1607  }
1608 
1609  size_t indexOfShardColumn = 0;
1610  const ColumnDescriptor* shardColumn = catalog.getShardColumnMetadataForTable(td);
1611  CHECK(shardColumn);
1612 
1613  if (column_list.empty()) {
1614  auto all_cols =
1615  catalog.getAllColumnMetadataForTable(td->tableId, false, false, false);
1616  auto iter = std::find(all_cols.begin(), all_cols.end(), shardColumn);
1617  CHECK(iter != all_cols.end());
1618  indexOfShardColumn = std::distance(all_cols.begin(), iter);
1619  } else {
1620  for (auto& c : column_list) {
1621  if (*c == shardColumn->columnName) {
1622  break;
1623  }
1624  indexOfShardColumn++;
1625  }
1626 
1627  if (indexOfShardColumn == column_list.size()) {
1628  throw std::runtime_error("No value defined for shard column.");
1629  }
1630  }
1631 
1632  if (indexOfShardColumn >= value_list.size()) {
1633  throw std::runtime_error("No value defined for shard column.");
1634  }
1635 
1636  auto& shardColumnValueExpr = *(std::next(value_list.begin(), indexOfShardColumn));
1637 
1638  Analyzer::Query query;
1639  auto e = shardColumnValueExpr->analyze(catalog, query);
1640  e = e->add_cast(shardColumn->columnType);
1641  const Analyzer::Constant* con = dynamic_cast<Analyzer::Constant*>(e.get());
1642  if (!con) {
1643  auto col_cast = dynamic_cast<const Analyzer::UOper*>(e.get());
1644  CHECK(col_cast);
1645  CHECK_EQ(kCAST, col_cast->get_optype());
1646  con = dynamic_cast<const Analyzer::Constant*>(col_cast->get_operand());
1647  }
1648  CHECK(con);
1649 
1650  Datum d = con->get_constval();
1651 
1652  auto shard_count = td->nShards * num_leafs;
1653  int64_t shardId = 0;
1654 
1655  if (con->get_is_null()) {
1657  shard_count);
1658  } else if (shardColumn->columnType.is_string()) {
1659  auto dictDesc =
1660  catalog.getMetadataForDict(shardColumn->columnType.get_comp_param(), true);
1661  auto str_id = dictDesc->stringDict->getOrAdd(*d.stringval);
1662  bool invalid = false;
1663 
1664  if (4 == shardColumn->columnType.get_size()) {
1665  invalid = str_id > max_valid_int_value<int32_t>();
1666  } else if (2 == shardColumn->columnType.get_size()) {
1667  invalid = str_id > max_valid_int_value<uint16_t>();
1668  } else if (1 == shardColumn->columnType.get_size()) {
1669  invalid = str_id > max_valid_int_value<uint8_t>();
1670  }
1671 
1672  if (invalid || str_id == inline_int_null_value<int32_t>()) {
1673  str_id = inline_fixed_encoding_null_val(shardColumn->columnType);
1674  }
1675  shardId = SHARD_FOR_KEY(str_id, shard_count);
1676  } else {
1677  switch (shardColumn->columnType.get_logical_size()) {
1678  case 8:
1679  shardId = SHARD_FOR_KEY(d.bigintval, shard_count);
1680  break;
1681  case 4:
1682  shardId = SHARD_FOR_KEY(d.intval, shard_count);
1683  break;
1684  case 2:
1685  shardId = SHARD_FOR_KEY(d.smallintval, shard_count);
1686  break;
1687  case 1:
1688  shardId = SHARD_FOR_KEY(d.tinyintval, shard_count);
1689  break;
1690  default:
1691  CHECK(false);
1692  }
1693  }
1694 
1695  return shardId / td->nShards;
1696 }
1697 
1698 void InsertValuesStmt::analyze(const Catalog_Namespace::Catalog& catalog,
1699  Analyzer::Query& query) const {
1700  InsertStmt::analyze(catalog, query);
1701  std::vector<std::shared_ptr<Analyzer::TargetEntry>>& tlist =
1702  query.get_targetlist_nonconst();
1703  const auto tableId = query.get_result_table_id();
1704  const std::list<const ColumnDescriptor*> non_phys_cols =
1705  catalog.getAllColumnMetadataForTable(tableId, false, false, false);
1706  if (non_phys_cols.size() != value_list.size()) {
1707  throw std::runtime_error("Insert has more target columns than expressions.");
1708  }
1709  std::list<int>::const_iterator it = query.get_result_col_list().begin();
1710  for (auto& v : value_list) {
1711  auto e = v->analyze(catalog, query);
1712  const ColumnDescriptor* cd =
1713  catalog.getMetadataForColumn(query.get_result_table_id(), *it);
1714  CHECK(cd);
1715  if (cd->columnType.get_notnull()) {
1716  auto c = std::dynamic_pointer_cast<Analyzer::Constant>(e);
1717  if (c != nullptr && c->get_is_null()) {
1718  throw std::runtime_error("Cannot insert NULL into column " + cd->columnName);
1719  }
1720  }
1721  e = e->add_cast(cd->columnType);
1722  tlist.emplace_back(new Analyzer::TargetEntry("", e, false));
1723  ++it;
1724 
1725  const auto& col_ti = cd->columnType;
1726  if (col_ti.get_physical_cols() > 0) {
1727  CHECK(cd->columnType.is_geometry());
1728  std::string* wkt{nullptr};
1729  auto c = std::dynamic_pointer_cast<Analyzer::Constant>(e);
1730  if (c) {
1731  wkt = c->get_constval().stringval;
1732  } else {
1733  auto uoper = std::dynamic_pointer_cast<Analyzer::UOper>(e);
1734  if (uoper && uoper->get_optype() == kCAST) {
1735  auto c = dynamic_cast<const Analyzer::Constant*>(uoper->get_operand());
1736  if (c) {
1737  wkt = c->get_constval().stringval;
1738  }
1739  }
1740  }
1741  if (!wkt) {
1742  throw std::runtime_error("Expecting a WKT string for column " + cd->columnName);
1743  }
1744  std::vector<double> coords;
1745  std::vector<double> bounds;
1746  std::vector<int> ring_sizes;
1747  std::vector<int> poly_rings;
1748  int render_group =
1749  0; // @TODO simon.eves where to get render_group from in this context?!
1750  SQLTypeInfo import_ti;
1752  *wkt, import_ti, coords, bounds, ring_sizes, poly_rings)) {
1753  throw std::runtime_error("Cannot read geometry to insert into column " +
1754  cd->columnName);
1755  }
1756  if (cd->columnType.get_type() != import_ti.get_type()) {
1757  // allow POLYGON to be inserted into MULTIPOLYGON column
1758  if (!(import_ti.get_type() == SQLTypes::kPOLYGON &&
1760  throw std::runtime_error("Imported geometry doesn't match the type of column " +
1761  cd->columnName);
1762  }
1763  }
1764  // TODO: check if import SRID matches columns SRID, may need to transform before
1765  // inserting
1766 
1767  int nextColumnOffset = 1;
1768 
1769  const ColumnDescriptor* cd_coords = catalog.getMetadataForColumn(
1770  query.get_result_table_id(), cd->columnId + nextColumnOffset);
1771  CHECK(cd_coords);
1772  CHECK_EQ(cd_coords->columnType.get_type(), kARRAY);
1773  CHECK_EQ(cd_coords->columnType.get_subtype(), kTINYINT);
1774  auto compressed_coords = Importer_NS::compress_coords(coords, col_ti);
1775  std::list<std::shared_ptr<Analyzer::Expr>> value_exprs;
1776  for (auto cc : compressed_coords) {
1777  Datum d;
1778  d.tinyintval = cc;
1779  auto e = makeExpr<Analyzer::Constant>(kTINYINT, false, d);
1780  value_exprs.push_back(e);
1781  }
1782  tlist.emplace_back(new Analyzer::TargetEntry(
1783  "",
1784  makeExpr<Analyzer::Constant>(cd_coords->columnType, false, value_exprs),
1785  false));
1786  ++it;
1787  nextColumnOffset++;
1788 
1789  if (cd->columnType.get_type() == kPOLYGON ||
1790  cd->columnType.get_type() == kMULTIPOLYGON) {
1791  // Put ring sizes array into separate physical column
1792  const ColumnDescriptor* cd_ring_sizes = catalog.getMetadataForColumn(
1793  query.get_result_table_id(), cd->columnId + nextColumnOffset);
1794  CHECK(cd_ring_sizes);
1795  CHECK_EQ(cd_ring_sizes->columnType.get_type(), kARRAY);
1796  CHECK_EQ(cd_ring_sizes->columnType.get_subtype(), kINT);
1797  std::list<std::shared_ptr<Analyzer::Expr>> value_exprs;
1798  for (auto c : ring_sizes) {
1799  Datum d;
1800  d.intval = c;
1801  auto e = makeExpr<Analyzer::Constant>(kINT, false, d);
1802  value_exprs.push_back(e);
1803  }
1804  tlist.emplace_back(new Analyzer::TargetEntry(
1805  "",
1806  makeExpr<Analyzer::Constant>(cd_ring_sizes->columnType, false, value_exprs),
1807  false));
1808  ++it;
1809  nextColumnOffset++;
1810 
1811  if (cd->columnType.get_type() == kMULTIPOLYGON) {
1812  // Put poly_rings array into separate physical column
1813  const ColumnDescriptor* cd_poly_rings = catalog.getMetadataForColumn(
1814  query.get_result_table_id(), cd->columnId + nextColumnOffset);
1815  CHECK(cd_poly_rings);
1816  CHECK_EQ(cd_poly_rings->columnType.get_type(), kARRAY);
1817  CHECK_EQ(cd_poly_rings->columnType.get_subtype(), kINT);
1818  std::list<std::shared_ptr<Analyzer::Expr>> value_exprs;
1819  for (auto c : poly_rings) {
1820  Datum d;
1821  d.intval = c;
1822  auto e = makeExpr<Analyzer::Constant>(kINT, false, d);
1823  value_exprs.push_back(e);
1824  }
1825  tlist.emplace_back(new Analyzer::TargetEntry(
1826  "",
1827  makeExpr<Analyzer::Constant>(cd_poly_rings->columnType, false, value_exprs),
1828  false));
1829  ++it;
1830  nextColumnOffset++;
1831  }
1832  }
1833 
1834  if (cd->columnType.get_type() == kLINESTRING ||
1835  cd->columnType.get_type() == kPOLYGON ||
1836  cd->columnType.get_type() == kMULTIPOLYGON) {
1837  const ColumnDescriptor* cd_bounds = catalog.getMetadataForColumn(
1838  query.get_result_table_id(), cd->columnId + nextColumnOffset);
1839  CHECK(cd_bounds);
1840  CHECK_EQ(cd_bounds->columnType.get_type(), kARRAY);
1841  CHECK_EQ(cd_bounds->columnType.get_subtype(), kDOUBLE);
1842  std::list<std::shared_ptr<Analyzer::Expr>> value_exprs;
1843  for (auto b : bounds) {
1844  Datum d;
1845  d.doubleval = b;
1846  auto e = makeExpr<Analyzer::Constant>(kDOUBLE, false, d);
1847  value_exprs.push_back(e);
1848  }
1849  tlist.emplace_back(new Analyzer::TargetEntry(
1850  "",
1851  makeExpr<Analyzer::Constant>(cd_bounds->columnType, false, value_exprs),
1852  false));
1853  ++it;
1854  nextColumnOffset++;
1855  }
1856 
1857  if (cd->columnType.get_type() == kPOLYGON ||
1858  cd->columnType.get_type() == kMULTIPOLYGON) {
1859  // Put render group into separate physical column
1860  const ColumnDescriptor* cd_render_group = catalog.getMetadataForColumn(
1861  query.get_result_table_id(), cd->columnId + nextColumnOffset);
1862  CHECK(cd_render_group);
1863  CHECK_EQ(cd_render_group->columnType.get_type(), kINT);
1864  Datum d;
1865  d.intval = render_group;
1866  tlist.emplace_back(new Analyzer::TargetEntry(
1867  "",
1868  makeExpr<Analyzer::Constant>(cd_render_group->columnType, false, d),
1869  false));
1870  ++it;
1871  nextColumnOffset++;
1872  }
1873  }
1874  }
1875 }
1876 
1877 void InsertQueryStmt::analyze(const Catalog_Namespace::Catalog& catalog,
1878  Analyzer::Query& insert_query) const {
1879  InsertStmt::analyze(catalog, insert_query);
1880  query->analyze(catalog, insert_query);
1881 }
1882 
1883 void UpdateStmt::analyze(const Catalog_Namespace::Catalog& catalog,
1884  Analyzer::Query& query) const {
1885  throw std::runtime_error("UPDATE statement not supported yet.");
1886 }
1887 
1888 void DeleteStmt::analyze(const Catalog_Namespace::Catalog& catalog,
1889  Analyzer::Query& query) const {
1890  throw std::runtime_error("DELETE statement not supported yet.");
1891 }
1892 
1893 void SQLType::check_type() {
1894  switch (type) {
1895  case kCHAR:
1896  case kVARCHAR:
1897  if (param1 <= 0) {
1898  throw std::runtime_error("CHAR and VARCHAR must have a positive dimension.");
1899  }
1900  break;
1901  case kDECIMAL:
1902  case kNUMERIC:
1903  if (param1 <= 0) {
1904  throw std::runtime_error("DECIMAL and NUMERIC must have a positive precision.");
1905  } else if (param1 > 19) {
1906  throw std::runtime_error(
1907  "DECIMAL and NUMERIC precision cannot be larger than 19.");
1908  } else if (param1 <= param2) {
1909  throw std::runtime_error(
1910  "DECIMAL and NUMERIC must have precision larger than scale.");
1911  }
1912  break;
1913  case kTIMESTAMP:
1914  if (param1 == -1) {
1915  param1 = 0; // set default to 0
1916  } else if (param1 != 0 && param1 != 3 && param1 != 6 &&
1917  param1 != 9) { // support ms, us, ns
1918  throw std::runtime_error(
1919  "Only TIMESTAMP(n) where n = (0,3,6,9) are supported now.");
1920  }
1921  break;
1922  case kTIME:
1923  if (param1 == -1) {
1924  param1 = 0; // default precision is 0
1925  }
1926  if (param1 > 0) { // @TODO(wei) support sub-second precision later.
1927  throw std::runtime_error("Only TIME(0) is supported now.");
1928  }
1929  break;
1930  case kPOINT:
1931  case kLINESTRING:
1932  case kPOLYGON:
1933  case kMULTIPOLYGON:
1934  // Storing SRID in param1
1935  break;
1936  default:
1937  param1 = 0;
1938  break;
1939  }
1940 }
1941 
1942 namespace {
1943 
1945  const auto& col_ti = cd.columnType;
1946  if (col_ti.is_integer() ||
1947  (col_ti.is_string() && col_ti.get_compression() == kENCODING_DICT) ||
1948  col_ti.is_time()) {
1949  return;
1950  }
1951  throw std::runtime_error("Cannot shard on type " + col_ti.get_type_name() +
1952  ", encoding " + col_ti.get_compression_name());
1953 }
1954 
1955 size_t shard_column_index(const std::string& name,
1956  const std::list<ColumnDescriptor>& columns) {
1957  size_t index = 1;
1958  for (const auto& cd : columns) {
1959  if (cd.columnName == name) {
1961  return index;
1962  }
1963  ++index;
1964  if (cd.columnType.is_geometry()) {
1965  index += cd.columnType.get_physical_cols();
1966  }
1967  }
1968  // Not found, return 0
1969  return 0;
1970 }
1971 
1972 size_t sort_column_index(const std::string& name,
1973  const std::list<ColumnDescriptor>& columns) {
1974  size_t index = 1;
1975  for (const auto& cd : columns) {
1976  if (boost::to_upper_copy<std::string>(cd.columnName) == name) {
1977  return index;
1978  }
1979  ++index;
1980  if (cd.columnType.is_geometry()) {
1981  index += cd.columnType.get_physical_cols();
1982  }
1983  }
1984  // Not found, return 0
1985  return 0;
1986 }
1987 
1988 void set_string_field(rapidjson::Value& obj,
1989  const std::string& field_name,
1990  const std::string& field_value,
1991  rapidjson::Document& document) {
1992  rapidjson::Value field_name_json_str;
1993  field_name_json_str.SetString(
1994  field_name.c_str(), field_name.size(), document.GetAllocator());
1995  rapidjson::Value field_value_json_str;
1996  field_value_json_str.SetString(
1997  field_value.c_str(), field_value.size(), document.GetAllocator());
1998  obj.AddMember(field_name_json_str, field_value_json_str, document.GetAllocator());
1999 }
2000 
2002  const ShardKeyDef* shard_key_def,
2003  const std::vector<SharedDictionaryDef>& shared_dict_defs) {
2004  rapidjson::Document document;
2005  auto& allocator = document.GetAllocator();
2006  rapidjson::Value arr(rapidjson::kArrayType);
2007  if (shard_key_def) {
2008  rapidjson::Value shard_key_obj(rapidjson::kObjectType);
2009  set_string_field(shard_key_obj, "type", "SHARD KEY", document);
2010  set_string_field(shard_key_obj, "name", shard_key_def->get_column(), document);
2011  arr.PushBack(shard_key_obj, allocator);
2012  }
2013  for (const auto& shared_dict_def : shared_dict_defs) {
2014  rapidjson::Value shared_dict_obj(rapidjson::kObjectType);
2015  set_string_field(shared_dict_obj, "type", "SHARED DICTIONARY", document);
2016  set_string_field(shared_dict_obj, "name", shared_dict_def.get_column(), document);
2018  shared_dict_obj, "foreign_table", shared_dict_def.get_foreign_table(), document);
2019  set_string_field(shared_dict_obj,
2020  "foreign_column",
2021  shared_dict_def.get_foreign_column(),
2022  document);
2023  arr.PushBack(shared_dict_obj, allocator);
2024  }
2025  rapidjson::StringBuffer buffer;
2026  rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
2027  arr.Accept(writer);
2028  return buffer.GetString();
2029 }
2030 
2031 template <typename LITERAL_TYPE,
2032  typename ASSIGNMENT,
2033  typename VALIDATE = DefaultValidate<LITERAL_TYPE>>
2034 decltype(auto) get_property_value(const NameValueAssign* p,
2035  ASSIGNMENT op,
2036  VALIDATE validate = VALIDATE()) {
2037  const auto val = validate(p);
2038  return op(val);
2039 }
2040 
2042  const NameValueAssign* p,
2043  const std::list<ColumnDescriptor>& columns) {
2044  return get_property_value<IntLiteral>(p,
2045  [&td](const auto val) { td.maxFragRows = val; });
2046 }
2047 
2049  const NameValueAssign* p,
2050  const std::list<ColumnDescriptor>& columns) {
2051  return get_property_value<IntLiteral>(p,
2052  [&td](const auto val) { td.maxChunkSize = val; });
2053 }
2054 
2056  const NameValueAssign* p,
2057  const std::list<ColumnDescriptor>& columns) {
2058  return get_property_value<IntLiteral>(p,
2059  [&td](const auto val) { td.fragPageSize = val; });
2060 }
2062  const NameValueAssign* p,
2063  const std::list<ColumnDescriptor>& columns) {
2064  return get_property_value<IntLiteral>(p, [&td](const auto val) { td.maxRows = val; });
2065 }
2067  const NameValueAssign* p,
2068  const std::list<ColumnDescriptor>& columns) {
2069  return get_property_value<StringLiteral>(p, [&td](const auto partitions_uc) {
2070  if (partitions_uc != "SHARDED" && partitions_uc != "REPLICATED") {
2071  throw std::runtime_error("PARTITIONS must be SHARDED or REPLICATED");
2072  }
2073  if (td.shardedColumnId != 0 && partitions_uc == "REPLICATED") {
2074  throw std::runtime_error(
2075  "A table cannot be sharded and replicated at the same time");
2076  };
2077  td.partitions = partitions_uc;
2078  });
2079 }
2081  const NameValueAssign* p,
2082  const std::list<ColumnDescriptor>& columns) {
2083  if (!td.shardedColumnId) {
2084  throw std::runtime_error("SHARD KEY must be defined.");
2085  }
2086  return get_property_value<IntLiteral>(p, [&td](const auto shard_count) {
2087  if (g_leaf_count && shard_count % g_leaf_count) {
2088  throw std::runtime_error(
2089  "SHARD_COUNT must be a multiple of the number of leaves in the cluster.");
2090  }
2091  td.nShards = g_leaf_count ? shard_count / g_leaf_count : shard_count;
2092  if (!td.shardedColumnId && !td.nShards) {
2093  throw std::runtime_error(
2094  "Must specify the number of shards through the SHARD_COUNT option");
2095  };
2096  });
2097 }
2098 
2099 decltype(auto) get_vacuum_def(TableDescriptor& td,
2100  const NameValueAssign* p,
2101  const std::list<ColumnDescriptor>& columns) {
2102  return get_property_value<StringLiteral>(p, [&td](const auto vacuum_uc) {
2103  if (vacuum_uc != "IMMEDIATE" && vacuum_uc != "DELAYED") {
2104  throw std::runtime_error("VACUUM must be IMMEDIATE or DELAYED");
2105  }
2106  td.hasDeletedCol = boost::iequals(vacuum_uc, "IMMEDIATE") ? false : true;
2107  });
2108 }
2109 
2111  const NameValueAssign* p,
2112  const std::list<ColumnDescriptor>& columns) {
2113  return get_property_value<StringLiteral>(p, [&td, &columns](const auto sort_upper) {
2114  td.sortedColumnId = sort_column_index(sort_upper, columns);
2115  if (!td.sortedColumnId) {
2116  throw std::runtime_error("Specified sort column " + sort_upper + " doesn't exist");
2117  }
2118  });
2119 }
2120 
2121 static const std::map<const std::string, const TableDefFuncPtr> tableDefFuncMap = {
2122  {"fragment_size"s, get_frag_size_def},
2123  {"max_chunk_size"s, get_max_chunk_size_def},
2124  {"page_size"s, get_page_size_def},
2125  {"max_rows"s, get_max_rows_def},
2126  {"partitions"s, get_partions_def},
2127  {"shard_count"s, get_shard_count_def},
2128  {"vacuum"s, get_vacuum_def},
2129  {"sort_column"s, get_sort_column_def}};
2130 
2132  const std::unique_ptr<NameValueAssign>& p,
2133  const std::list<ColumnDescriptor>& columns) {
2134  const auto it = tableDefFuncMap.find(boost::to_lower_copy<std::string>(*p->get_name()));
2135  if (it == tableDefFuncMap.end()) {
2136  throw std::runtime_error("Invalid CREATE TABLE option " + *p->get_name() +
2137  ". Should be FRAGMENT_SIZE, PAGE_SIZE, MAX_ROWS, "
2138  "PARTITIONS, VACUUM, SORT_COLUMN or SHARD_COUNT.");
2139  }
2140  return it->second(td, p.get(), columns);
2141 }
2142 
2143 } // namespace
2144 
2145 void CreateTableStmt::executeDryRun(const Catalog_Namespace::SessionInfo& session,
2146  TableDescriptor& td,
2147  std::list<ColumnDescriptor>& columns,
2148  std::vector<SharedDictionaryDef>& shared_dict_defs) {
2149  std::unordered_set<std::string> uc_col_names;
2150  const auto& catalog = session.getCatalog();
2151  const ShardKeyDef* shard_key_def{nullptr};
2152  for (auto& e : table_element_list_) {
2153  if (dynamic_cast<SharedDictionaryDef*>(e.get())) {
2154  auto shared_dict_def = static_cast<SharedDictionaryDef*>(e.get());
2156  this, shared_dict_def, columns, shared_dict_defs, catalog);
2157  shared_dict_defs.push_back(*shared_dict_def);
2158  continue;
2159  }
2160  if (dynamic_cast<ShardKeyDef*>(e.get())) {
2161  if (shard_key_def) {
2162  throw std::runtime_error("Specified more than one shard key");
2163  }
2164  shard_key_def = static_cast<const ShardKeyDef*>(e.get());
2165  continue;
2166  }
2167  if (!dynamic_cast<ColumnDef*>(e.get())) {
2168  throw std::runtime_error("Table constraints are not supported yet.");
2169  }
2170  ColumnDef* coldef = static_cast<ColumnDef*>(e.get());
2171  ColumnDescriptor cd;
2172  cd.columnName = *coldef->get_column_name();
2173  const auto uc_col_name = boost::to_upper_copy<std::string>(cd.columnName);
2174  if (reserved_keywords.find(uc_col_name) != reserved_keywords.end()) {
2175  throw std::runtime_error("Cannot create column with reserved keyword '" +
2176  cd.columnName + "'");
2177  }
2178  const auto it_ok = uc_col_names.insert(uc_col_name);
2179  if (!it_ok.second) {
2180  throw std::runtime_error("Column '" + cd.columnName + "' defined more than once");
2181  }
2182 
2183  setColumnDescriptor(cd, coldef);
2184  columns.push_back(cd);
2185  }
2186 
2187  td.tableName = *table_;
2188  td.nColumns = columns.size();
2189  td.isView = false;
2190  td.fragmenter = nullptr;
2196  if (shard_key_def) {
2197  td.shardedColumnId = shard_column_index(shard_key_def->get_column(), columns);
2198  if (!td.shardedColumnId) {
2199  throw std::runtime_error("Specified shard column " + shard_key_def->get_column() +
2200  " doesn't exist");
2201  }
2202  }
2203  if (is_temporary_) {
2205  } else {
2207  }
2208  if (!storage_options_.empty()) {
2209  for (auto& p : storage_options_) {
2210  get_table_definitions(td, p, columns);
2211  }
2212  }
2213  if (td.shardedColumnId && !td.nShards) {
2214  throw std::runtime_error("SHARD_COUNT needs to be specified with SHARD_KEY.");
2215  }
2216  td.keyMetainfo = serialize_key_metainfo(shard_key_def, shared_dict_defs);
2217 }
2218 
2219 void CreateTableStmt::execute(const Catalog_Namespace::SessionInfo& session) {
2220  auto& catalog = session.getCatalog();
2221  // check access privileges
2224  throw std::runtime_error("Table " + *table_ +
2225  " will not be created. User has no create privileges.");
2226  }
2227 
2228  if (catalog.getMetadataForTable(*table_) != nullptr) {
2229  if (if_not_exists_) {
2230  return;
2231  }
2232  throw std::runtime_error("Table " + *table_ + " already exists.");
2233  }
2234  TableDescriptor td;
2235  std::list<ColumnDescriptor> columns;
2236  std::vector<SharedDictionaryDef> shared_dict_defs;
2237 
2238  executeDryRun(session, td, columns, shared_dict_defs);
2239  td.userId = session.get_currentUser().userId;
2240 
2241  catalog.createShardedTable(td, columns, shared_dict_defs);
2242  // TODO (max): It's transactionally unsafe, should be fixed: we may create object w/o
2243  // privileges
2244  SysCatalog::instance().createDBObject(
2245  session.get_currentUser(), td.tableName, TableDBObjectType, catalog);
2246 }
2247 
2248 std::shared_ptr<ResultSet> getResultSet(QueryStateProxy query_state_proxy,
2249  const std::string select_stmt,
2250  std::vector<TargetMetaInfo>& targets,
2251  bool validate_only = false) {
2252  auto const session = query_state_proxy.getQueryState().getConstSessionInfo();
2253  auto& catalog = session->getCatalog();
2254 
2255  auto executor = Executor::getExecutor(catalog.getCurrentDB().dbId);
2256 
2257 #ifdef HAVE_CUDA
2258  const auto device_type = session->get_executor_device_type();
2259 #else
2260  const auto device_type = ExecutorDeviceType::CPU;
2261 #endif // HAVE_CUDA
2262  auto calcite_mgr = catalog.getCalciteMgr();
2263 
2264  // TODO MAT this should actually get the global or the session parameter for
2265  // view optimization
2266  const auto query_ra =
2267  calcite_mgr
2268  ->process(query_state_proxy, pg_shim(select_stmt), {}, true, false, false)
2269  .plan_result;
2270  CompilationOptions co = {
2271  device_type, true, ExecutorOptLevel::LoopStrengthReduction, false};
2272  // TODO(adb): Need a better method of dropping constants into this ExecutionOptions
2273  // struct
2274  ExecutionOptions eo = {false,
2275  true,
2276  false,
2277  true,
2278  false,
2279  false,
2280  validate_only,
2281  false,
2282  10000,
2283  false,
2284  false,
2285  0.9};
2286  RelAlgExecutor ra_executor(executor.get(), catalog, query_ra);
2287  ExecutionResult result{std::make_shared<ResultSet>(std::vector<TargetInfo>{},
2290  nullptr,
2291  nullptr),
2292  {}};
2293  result = ra_executor.executeRelAlgQuery(co, eo, nullptr);
2294  targets = result.getTargetsMeta();
2295 
2296  return result.getRows();
2297 }
2298 
2299 AggregatedResult InsertIntoTableAsSelectStmt::LocalConnector::query(
2300  QueryStateProxy query_state_proxy,
2301  std::string& sql_query_string,
2302  bool validate_only) {
2303  auto const session = query_state_proxy.getQueryState().getConstSessionInfo();
2304  std::string pg_shimmed_select_query = pg_shim(sql_query_string);
2305  const auto query_ra = parse_to_ra(query_state_proxy, pg_shimmed_select_query);
2306 
2307  // Note that the below command should *only* get read locks on the source table for
2308  // InsertIntoAsSelect. Write locks will be taken on the target table in
2309  // InsertOrderFragmenter::deleteFragments. This potentially leaves a small window where
2310  // the target table is not locked, and should be investigated.
2311  std::vector<TableLock> table_locks;
2312  TableLockMgr::getTableLocks(session->getCatalog(), query_ra, table_locks);
2313 
2314  std::vector<TargetMetaInfo> target_metainfos;
2315 
2316  auto result_rows =
2317  getResultSet(query_state_proxy, sql_query_string, target_metainfos, validate_only);
2318  AggregatedResult res = {result_rows, target_metainfos};
2319  return res;
2320 }
2321 
2322 AggregatedResult InsertIntoTableAsSelectStmt::LocalConnector::query(
2323  QueryStateProxy query_state_proxy,
2324  std::string& sql_query_string) {
2325  return query(query_state_proxy, sql_query_string, false);
2326 }
2327 
2328 void InsertIntoTableAsSelectStmt::LocalConnector::insertDataToLeaf(
2329  const Catalog_Namespace::SessionInfo& session,
2330  const size_t leaf_idx,
2331  Fragmenter_Namespace::InsertData& insert_data) {
2332  CHECK(leaf_idx == 0);
2333  auto& catalog = session.getCatalog();
2334  auto created_td = catalog.getMetadataForTable(insert_data.tableId);
2335  // get CheckpointLock on the table before trying to create
2336  // its 1st fragment
2337  ChunkKey chunkKey = {catalog.getCurrentDB().dbId, created_td->tableId};
2338  mapd_unique_lock<mapd_shared_mutex> chkptlLock(
2341  // [ TableWriteLock ] lock is deferred in
2342  // InsertOrderFragmenter::deleteFragments
2343  created_td->fragmenter->insertDataNoCheckpoint(insert_data);
2344 }
2345 
2346 void InsertIntoTableAsSelectStmt::LocalConnector::checkpoint(
2347  const Catalog_Namespace::SessionInfo& session,
2348  int tableId) {
2349  auto& catalog = session.getCatalog();
2350  auto dbId = catalog.getCurrentDB().dbId;
2351  catalog.getDataMgr().checkpoint(dbId, tableId);
2352 }
2353 
2354 void InsertIntoTableAsSelectStmt::LocalConnector::rollback(
2355  const Catalog_Namespace::SessionInfo& session,
2356  int tableId) {
2357  auto& catalog = session.getCatalog();
2358  auto dbId = catalog.getCurrentDB().dbId;
2359  catalog.getDataMgr().checkpoint(dbId, tableId);
2360 }
2361 
2362 std::list<ColumnDescriptor>
2363 InsertIntoTableAsSelectStmt::LocalConnector::getColumnDescriptors(
2365  bool for_create) {
2366  std::list<ColumnDescriptor> column_descriptors;
2367  std::list<ColumnDescriptor> column_descriptors_for_create;
2368 
2369  int rowid_suffix = 0;
2370  for (const auto& target_metainfo : result.targets_meta) {
2371  ColumnDescriptor cd;
2372  cd.columnName = target_metainfo.get_resname();
2373  if (cd.columnName == "rowid") {
2374  cd.columnName += std::to_string(rowid_suffix++);
2375  }
2376  cd.columnType = target_metainfo.get_physical_type_info();
2377 
2378  ColumnDescriptor cd_for_create = cd;
2379 
2381  // we need to reset the comp param (as this points to the actual dictionary)
2382  if (cd.columnType.is_array()) {
2383  // for dict encoded arrays, it is always 4 bytes
2384  cd_for_create.columnType.set_comp_param(32);
2385  } else {
2386  cd_for_create.columnType.set_comp_param(cd.columnType.get_size() * 8);
2387  }
2388  }
2389 
2390  column_descriptors_for_create.push_back(cd_for_create);
2391  column_descriptors.push_back(cd);
2392  }
2393 
2394  if (for_create) {
2395  return column_descriptors_for_create;
2396  }
2397 
2398  return column_descriptors;
2399 }
2400 
2401 void InsertIntoTableAsSelectStmt::populateData(QueryStateProxy query_state_proxy,
2402  bool is_temporary,
2403  bool validate_table) {
2404  auto const session = query_state_proxy.getQueryState().getConstSessionInfo();
2405  LocalConnector local_connector;
2406 
2407  bool populate_table = false;
2408 
2409  if (leafs_connector_) {
2410  populate_table = true;
2411  } else {
2412  leafs_connector_ = &local_connector;
2413  if (!g_cluster) {
2414  populate_table = true;
2415  }
2416  }
2417 
2418  auto& catalog = session->getCatalog();
2419  auto get_target_column_descriptors = [this, &catalog](const TableDescriptor* td) {
2420  std::vector<const ColumnDescriptor*> target_column_descriptors;
2421  if (column_list_.empty()) {
2422  auto list = catalog.getAllColumnMetadataForTable(td->tableId, false, false, false);
2423  target_column_descriptors = {std::begin(list), std::end(list)};
2424 
2425  } else {
2426  for (auto& c : column_list_) {
2427  const ColumnDescriptor* cd = catalog.getMetadataForColumn(td->tableId, *c);
2428  if (cd == nullptr) {
2429  throw std::runtime_error("Column " + *c + " does not exist.");
2430  }
2431  target_column_descriptors.push_back(cd);
2432  }
2433  }
2434 
2435  return target_column_descriptors;
2436  };
2437 
2438  if (validate_table) {
2439  // check access privileges
2440  const TableDescriptor* td = catalog.getMetadataForTable(table_name_);
2441 
2442  if (td == nullptr) {
2443  throw std::runtime_error("Table " + table_name_ + " does not exist.");
2444  }
2445  if (td->isView) {
2446  throw std::runtime_error("Insert to views is not supported yet.");
2447  }
2448 
2449  if (!session->checkDBAccessPrivileges(DBObjectType::TableDBObjectType,
2451  table_name_)) {
2452  throw std::runtime_error("User has no insert privileges on " + table_name_ + ".");
2453  }
2454 
2455  // only validate the select query so we get the target types
2456  // correctly, but do not populate the result set
2457  auto result = local_connector.query(query_state_proxy, select_query_, true);
2458  auto source_column_descriptors = local_connector.getColumnDescriptors(result, false);
2459 
2460  std::vector<const ColumnDescriptor*> target_column_descriptors =
2461  get_target_column_descriptors(td);
2462  if (catalog.getAllColumnMetadataForTable(td->tableId, false, false, false).size() !=
2463  target_column_descriptors.size()) {
2464  throw std::runtime_error("Insert into a subset of columns is not supported yet.");
2465  }
2466 
2467  if (source_column_descriptors.size() != target_column_descriptors.size()) {
2468  throw std::runtime_error("The number of source and target columns does not match.");
2469  }
2470 
2471  for (int i = 0; i < source_column_descriptors.size(); i++) {
2472  const ColumnDescriptor* source_cd =
2473  &(*std::next(source_column_descriptors.begin(), i));
2474  const ColumnDescriptor* target_cd = target_column_descriptors.at(i);
2475 
2476  if ((!source_cd->columnType.is_string() && !target_cd->columnType.is_string()) &&
2477  (source_cd->columnType.get_type() != target_cd->columnType.get_type())) {
2478  throw std::runtime_error("Source '" + source_cd->columnName + " " +
2479  source_cd->columnType.get_type_name() +
2480  "' and target '" + target_cd->columnName + " " +
2481  target_cd->columnType.get_type_name() +
2482  "' column types do not match.");
2483  }
2484  if (source_cd->columnType.is_array()) {
2485  if (source_cd->columnType.get_subtype() != target_cd->columnType.get_subtype()) {
2486  throw std::runtime_error("Source '" + source_cd->columnName + " " +
2487  source_cd->columnType.get_type_name() +
2488  "' and target '" + target_cd->columnName + " " +
2489  target_cd->columnType.get_type_name() +
2490  "' array column element types do not match.");
2491  }
2492  }
2493 
2494  if (source_cd->columnType.is_decimal() ||
2495  source_cd->columnType.get_elem_type().is_decimal()) {
2496  SQLTypeInfo sourceType = source_cd->columnType;
2497  SQLTypeInfo targetType = target_cd->columnType;
2498 
2499  if (source_cd->columnType.is_array()) {
2500  sourceType = source_cd->columnType.get_elem_type();
2501  targetType = target_cd->columnType.get_elem_type();
2502  }
2503 
2504  if ((sourceType.get_dimension() != targetType.get_dimension()) ||
2505  (sourceType.get_scale() != targetType.get_scale())) {
2506  throw std::runtime_error(
2507  "Source '" + source_cd->columnName + " " +
2508  source_cd->columnType.get_type_name() + "' and target '" +
2509  target_cd->columnName + " " + target_cd->columnType.get_type_name() +
2510  "' decimal columns dimension and or scale do not match.");
2511  }
2512  }
2513 
2514  if (source_cd->columnType.is_string()) {
2515  if (source_cd->columnType.get_compression() !=
2516  target_cd->columnType.get_compression()) {
2517  throw std::runtime_error("Source '" + source_cd->columnName + " " +
2518  source_cd->columnType.get_type_name() +
2519  "' and target '" + target_cd->columnName + " " +
2520  target_cd->columnType.get_type_name() +
2521  "' columns string encodings do not match.");
2522  }
2523  }
2524 
2525  if (source_cd->columnType.is_timestamp() && target_cd->columnType.is_timestamp()) {
2526  if (source_cd->columnType.get_dimension() !=
2527  target_cd->columnType.get_dimension()) {
2528  throw std::runtime_error("Source '" + source_cd->columnName + " " +
2529  source_cd->columnType.get_type_name() +
2530  "' and target '" + target_cd->columnName + " " +
2531  target_cd->columnType.get_type_name() +
2532  "' timestamp column precisions do not match.");
2533  }
2534  }
2535 
2536  if (!source_cd->columnType.is_string() && !source_cd->columnType.is_geometry() &&
2537  source_cd->columnType.get_size() > target_cd->columnType.get_size()) {
2538  throw std::runtime_error("Source '" + source_cd->columnName + " " +
2539  source_cd->columnType.get_type_name() +
2540  "' and target '" + target_cd->columnName + " " +
2541  target_cd->columnType.get_type_name() +
2542  "' column encoding sizes do not match.");
2543  }
2544  }
2545  }
2546 
2547  if (!populate_table) {
2548  return;
2549  }
2550 
2551  const TableDescriptor* created_td = catalog.getMetadataForTable(table_name_);
2552  Fragmenter_Namespace::InsertDataLoader insertDataLoader(*leafs_connector_);
2553  AggregatedResult res = leafs_connector_->query(query_state_proxy, select_query_);
2554  auto target_column_descriptors = get_target_column_descriptors(created_td);
2555  auto result_rows = res.rs;
2556  result_rows->setGeoReturnType(ResultSet::GeoReturnType::GeoTargetValue);
2557  const auto num_rows = result_rows->rowCount();
2558 
2559  if (0 == num_rows) {
2560  return;
2561  }
2562 
2563  size_t leaf_count = leafs_connector_->leafCount();
2564 
2565  size_t max_number_of_rows_per_package = std::min(num_rows / leaf_count, 64UL * 1024UL);
2566 
2567  size_t start_row = 0;
2568  size_t num_rows_to_process = std::min(num_rows, max_number_of_rows_per_package);
2569 
2570  // ensure that at least one row is being processed
2571  num_rows_to_process = std::max(num_rows_to_process, 1UL);
2572 
2573  std::vector<std::unique_ptr<TargetValueConverter>> value_converters;
2574 
2576 
2577  const int num_worker_threads = std::thread::hardware_concurrency();
2578 
2579  std::vector<size_t> thread_start_idx(num_worker_threads),
2580  thread_end_idx(num_worker_threads);
2581  bool can_go_parallel = !result_rows->isTruncated() && num_rows_to_process > 20000;
2582 
2583  std::atomic<size_t> row_idx{0};
2584 
2585  auto convert_function = [&result_rows,
2586  &value_converters,
2587  &row_idx,
2588  &num_rows_to_process,
2589  &thread_start_idx,
2590  &thread_end_idx](const int thread_id) {
2591  const int num_cols = value_converters.size();
2592  const size_t start = thread_start_idx[thread_id];
2593  const size_t end = thread_end_idx[thread_id];
2594  size_t idx = 0;
2595  for (idx = start; idx < end; ++idx) {
2596  const auto result_row = result_rows->getRowAtNoTranslations(idx);
2597  if (!result_row.empty()) {
2598  size_t target_row = row_idx.fetch_add(1);
2599 
2600  if (target_row >= num_rows_to_process) {
2601  break;
2602  }
2603 
2604  for (unsigned int col = 0; col < num_cols; col++) {
2605  const auto& mapd_variant = result_row[col];
2606  value_converters[col]->convertToColumnarFormat(target_row, &mapd_variant);
2607  }
2608  }
2609  }
2610 
2611  thread_start_idx[thread_id] = idx;
2612  };
2613 
2614  auto single_threaded_convert_function = [&result_rows,
2615  &value_converters,
2616  &row_idx,
2617  &num_rows_to_process,
2618  &thread_start_idx,
2619  &thread_end_idx](const int thread_id) {
2620  const int num_cols = value_converters.size();
2621  const size_t start = thread_start_idx[thread_id];
2622  const size_t end = thread_end_idx[thread_id];
2623  size_t idx = 0;
2624  for (idx = start; idx < end; ++idx) {
2625  size_t target_row = row_idx.fetch_add(1);
2626 
2627  if (target_row >= num_rows_to_process) {
2628  break;
2629  }
2630  const auto result_row = result_rows->getNextRow(false, false);
2631  CHECK(!result_row.empty());
2632  for (unsigned int col = 0; col < num_cols; col++) {
2633  const auto& mapd_variant = result_row[col];
2634  value_converters[col]->convertToColumnarFormat(target_row, &mapd_variant);
2635  }
2636  }
2637 
2638  thread_start_idx[thread_id] = idx;
2639  };
2640 
2641  if (can_go_parallel) {
2642  const size_t entryCount = result_rows->entryCount();
2643  for (size_t i = 0,
2644  start_entry = 0,
2645  stride = (entryCount + num_worker_threads - 1) / num_worker_threads;
2646  i < num_worker_threads && start_entry < entryCount;
2647  ++i, start_entry += stride) {
2648  const auto end_entry = std::min(start_entry + stride, entryCount);
2649  thread_start_idx[i] = start_entry;
2650  thread_end_idx[i] = end_entry;
2651  }
2652 
2653  } else {
2654  thread_start_idx[0] = 0;
2655  thread_end_idx[0] = result_rows->entryCount();
2656  }
2657 
2658  std::shared_ptr<Executor> executor;
2659 
2661  executor = Executor::getExecutor(catalog.getCurrentDB().dbId);
2662  }
2663 
2664  while (start_row < num_rows) {
2665  try {
2666  value_converters.clear();
2667  row_idx = 0;
2668  int colNum = 0;
2669  for (const auto targetDescriptor : target_column_descriptors) {
2670  auto sourceDataMetaInfo = res.targets_meta[colNum++];
2671 
2673  num_rows_to_process,
2674  catalog,
2675  sourceDataMetaInfo,
2676  targetDescriptor,
2677  targetDescriptor->columnType,
2678  !targetDescriptor->columnType.get_notnull(),
2679  result_rows->getRowSetMemOwner()->getLiteralStringDictProxy(),
2681  ? executor->getStringDictionaryProxy(
2682  sourceDataMetaInfo.get_type_info().get_comp_param(),
2683  result_rows->getRowSetMemOwner(),
2684  true)
2685  : nullptr};
2686  auto converter = factory.create(param);
2687  value_converters.push_back(std::move(converter));
2688  }
2689 
2690  if (can_go_parallel) {
2691  std::vector<std::future<void>> worker_threads;
2692  for (int i = 0; i < num_worker_threads; ++i) {
2693  worker_threads.push_back(std::async(std::launch::async, convert_function, i));
2694  }
2695 
2696  for (auto& child : worker_threads) {
2697  child.wait();
2698  }
2699  for (auto& child : worker_threads) {
2700  child.get();
2701  }
2702 
2703  } else {
2704  single_threaded_convert_function(0);
2705  }
2706 
2707  // finalize the insert data
2708  {
2709  auto finalizer_func =
2710  [](std::unique_ptr<TargetValueConverter>::pointer targetValueConverter) {
2711  targetValueConverter->finalizeDataBlocksForInsertData();
2712  };
2713  std::vector<std::future<void>> worker_threads;
2714  for (auto& converterPtr : value_converters) {
2715  worker_threads.push_back(
2716  std::async(std::launch::async, finalizer_func, converterPtr.get()));
2717  }
2718 
2719  for (auto& child : worker_threads) {
2720  child.wait();
2721  }
2722  for (auto& child : worker_threads) {
2723  child.get();
2724  }
2725  }
2726 
2728  insert_data.databaseId = catalog.getCurrentDB().dbId;
2729  CHECK(created_td);
2730  insert_data.tableId = created_td->tableId;
2731  insert_data.numRows = num_rows_to_process;
2732 
2733  for (int col_idx = 0; col_idx < target_column_descriptors.size(); col_idx++) {
2734  value_converters[col_idx]->addDataBlocksToInsertData(insert_data);
2735  }
2736 
2737  insertDataLoader.insertData(*session, insert_data);
2738  } catch (...) {
2739  try {
2740  if (created_td->nShards) {
2741  const auto shard_tables = catalog.getPhysicalTablesDescriptors(created_td);
2742  for (const auto ptd : shard_tables) {
2743  leafs_connector_->rollback(*session, ptd->tableId);
2744  }
2745  }
2746  leafs_connector_->rollback(*session, created_td->tableId);
2747  } catch (...) {
2748  // eat it
2749  }
2750  throw;
2751  }
2752  start_row += num_rows_to_process;
2753  num_rows_to_process = std::min(num_rows - start_row, max_number_of_rows_per_package);
2754  }
2755 
2756  if (!is_temporary) {
2757  if (created_td->nShards) {
2758  const auto shard_tables = catalog.getPhysicalTablesDescriptors(created_td);
2759  for (const auto ptd : shard_tables) {
2760  leafs_connector_->checkpoint(*session, ptd->tableId);
2761  }
2762  }
2763  leafs_connector_->checkpoint(*session, created_td->tableId);
2764  }
2765 }
2766 
2767 void InsertIntoTableAsSelectStmt::execute(const Catalog_Namespace::SessionInfo& session) {
2768  auto session_copy = session;
2769  auto session_ptr = std::shared_ptr<Catalog_Namespace::SessionInfo>(
2770  &session_copy, boost::null_deleter());
2771  auto query_state = query_state::QueryState::create(session_ptr, select_query_);
2772  auto stdlog = STDLOG(query_state);
2773  populateData(query_state->createQueryStateProxy(), false, true);
2774 }
2775 
2776 void CreateTableAsSelectStmt::execute(const Catalog_Namespace::SessionInfo& session) {
2777  auto session_copy = session;
2778  auto session_ptr = std::shared_ptr<Catalog_Namespace::SessionInfo>(
2779  &session_copy, boost::null_deleter());
2780  auto query_state = query_state::QueryState::create(session_ptr, select_query_);
2781  auto stdlog = STDLOG(query_state);
2782 
2783  LocalConnector local_connector;
2784  auto& catalog = session.getCatalog();
2785  bool create_table = nullptr == leafs_connector_;
2786 
2787  if (create_table) {
2788  // check access privileges
2791  throw std::runtime_error("CTAS failed. Table " + table_name_ +
2792  " will not be created. User has no create privileges.");
2793  }
2794 
2795  if (catalog.getMetadataForTable(table_name_) != nullptr) {
2796  if (if_not_exists_) {
2797  return;
2798  }
2799  throw std::runtime_error("Table " + table_name_ +
2800  " already exists and no data was loaded.");
2801  }
2802 
2803  // only validate the select query so we get the target types
2804  // correctly, but do not populate the result set
2805  auto result =
2806  local_connector.query(query_state->createQueryStateProxy(), select_query_, true);
2807  const auto column_descriptors_for_create =
2808  local_connector.getColumnDescriptors(result, true);
2809 
2810  // some validation as the QE might return some out of range column types
2811  for (auto& cd : column_descriptors_for_create) {
2812  if (cd.columnType.is_decimal() && cd.columnType.get_precision() > 18) {
2813  throw std::runtime_error(cd.columnName + ": Precision too high, max 18.");
2814  }
2815  }
2816 
2817  TableDescriptor td;
2818  td.tableName = table_name_;
2819  td.userId = session.get_currentUser().userId;
2820  td.nColumns = column_descriptors_for_create.size();
2821  td.isView = false;
2822  td.fragmenter = nullptr;
2828  td.keyMetainfo = "[]";
2829  if (is_temporary_) {
2831  } else {
2833  }
2834 
2835  if (!storage_options_.empty()) {
2836  for (auto& p : storage_options_) {
2837  get_table_definitions(td, p, column_descriptors_for_create);
2838  }
2839  }
2840 
2841  catalog.createTable(td, column_descriptors_for_create, {}, true);
2842  // TODO (max): It's transactionally unsafe, should be fixed: we may create object
2843  // w/o privileges
2844  SysCatalog::instance().createDBObject(
2845  session.get_currentUser(), td.tableName, TableDBObjectType, catalog);
2846  }
2847 
2848  try {
2849  populateData(query_state->createQueryStateProxy(), is_temporary_, false);
2850  } catch (...) {
2851  if (!g_cluster) {
2852  const TableDescriptor* created_td = catalog.getMetadataForTable(table_name_);
2853  if (created_td) {
2854  catalog.dropTable(created_td);
2855  }
2856  }
2857  throw;
2858  }
2859 }
2860 
2861 void DropTableStmt::execute(const Catalog_Namespace::SessionInfo& session) {
2862  auto& catalog = session.getCatalog();
2863 
2864  const TableDescriptor* td = catalog.getMetadataForTable(*table, false);
2865  if (td == nullptr) {
2866  if (if_exists) {
2867  return;
2868  }
2869  throw std::runtime_error("Table " + *table + " does not exist.");
2870  }
2871 
2872  // check access privileges
2873  if (!session.checkDBAccessPrivileges(
2875  throw std::runtime_error("Table " + *table +
2876  " will not be dropped. User has no proper privileges.");
2877  }
2878 
2879  if (td->isView) {
2880  throw std::runtime_error(*table + " is a view. Use DROP VIEW.");
2881  }
2882 
2883  auto chkptlLock = getTableLock<mapd_shared_mutex, mapd_unique_lock>(
2884  catalog, *table, LockType::CheckpointLock);
2885  auto table_write_lock = TableLockMgr::getWriteLockForTable(catalog, *table);
2886  catalog.dropTable(td);
2887 }
2888 
2889 void TruncateTableStmt::execute(const Catalog_Namespace::SessionInfo& session) {
2890  auto& catalog = session.getCatalog();
2891  const TableDescriptor* td = catalog.getMetadataForTable(*table);
2892  if (td == nullptr) {
2893  throw std::runtime_error("Table " + *table + " does not exist.");
2894  }
2895 
2896  // check access privileges
2897  std::vector<DBObject> privObjects;
2898  DBObject dbObject(*table, TableDBObjectType);
2899  dbObject.loadKey(catalog);
2901  privObjects.push_back(dbObject);
2902  if (!SysCatalog::instance().checkPrivileges(session.get_currentUser(), privObjects)) {
2903  throw std::runtime_error("Table " + *table + " will not be truncated. User " +
2904  session.get_currentUser().userName +
2905  " has no proper privileges.");
2906  }
2907 
2908  if (td->isView) {
2909  throw std::runtime_error(*table + " is a view. Cannot Truncate.");
2910  }
2911  catalog.truncateTable(td);
2912 }
2913 
2915  const TableDescriptor* td) {
2916  if (session.get_currentUser().isSuper ||
2917  session.get_currentUser().userId == td->userId) {
2918  return;
2919  }
2920  std::vector<DBObject> privObjects;
2921  DBObject dbObject(td->tableName, TableDBObjectType);
2922  dbObject.loadKey(session.getCatalog());
2924  privObjects.push_back(dbObject);
2925  if (!SysCatalog::instance().checkPrivileges(session.get_currentUser(), privObjects)) {
2926  throw std::runtime_error("Current user does not have the privilege to alter table: " +
2927  td->tableName);
2928  }
2929 }
2930 
2931 void RenameUserStmt::execute(const Catalog_Namespace::SessionInfo& session) {
2932  if (!session.get_currentUser().isSuper) {
2933  throw std::runtime_error("Only a super user can rename users.");
2934  }
2935 
2937  if (!SysCatalog::instance().getMetadataForUser(*username_, user)) {
2938  throw std::runtime_error("User " + *username_ + " does not exist.");
2939  }
2940 
2941  SysCatalog::instance().renameUser(*username_, *new_username_);
2942 }
2943 
2944 void RenameDatabaseStmt::execute(const Catalog_Namespace::SessionInfo& session) {
2946  if (!SysCatalog::instance().getMetadataForDB(*database_name_, db)) {
2947  throw std::runtime_error("Database " + *database_name_ + " does not exist.");
2948  }
2949 
2950  if (!session.get_currentUser().isSuper &&
2951  session.get_currentUser().userId != db.dbOwner) {
2952  throw std::runtime_error("Only a super user or the owner can rename the database.");
2953  }
2954 
2955  SysCatalog::instance().renameDatabase(*database_name_, *new_database_name_);
2956 }
2957 
2958 void RenameTableStmt::execute(const Catalog_Namespace::SessionInfo& session) {
2959  auto& catalog = session.getCatalog();
2960  const TableDescriptor* td = catalog.getMetadataForTable(*table);
2961  if (td == nullptr) {
2962  throw std::runtime_error("Table " + *table + " does not exist.");
2963  }
2964  check_alter_table_privilege(session, td);
2965  if (catalog.getMetadataForTable(*new_table_name) != nullptr) {
2966  throw std::runtime_error("Table or View " + *new_table_name + " already exists.");
2967  }
2968  catalog.renameTable(td, *new_table_name);
2969 }
2970 
2971 void DDLStmt::setColumnDescriptor(ColumnDescriptor& cd, const ColumnDef* coldef) {
2972  cd.columnName = *coldef->get_column_name();
2973  SQLType* t = coldef->get_column_type();
2974  t->check_type();
2975  if (t->get_is_array()) {
2977  cd.columnType.set_subtype(t->get_type());
2978  } else {
2979  cd.columnType.set_type(t->get_type());
2980  }
2981  if (IS_GEO(t->get_type())) {
2982  cd.columnType.set_subtype(static_cast<SQLTypes>(t->get_param1()));
2985  } else {
2987  cd.columnType.set_scale(t->get_param2());
2988  }
2990  const ColumnConstraintDef* cc = coldef->get_column_constraint();
2991  if (cc == nullptr) {
2992  cd.columnType.set_notnull(false);
2993  } else {
2995  }
2996  const CompressDef* compression = coldef->get_compression();
2997  if (compression == nullptr) {
2998  // Change default TEXT column behaviour to be DICT encoded
2999  if (cd.columnType.is_string() || cd.columnType.is_string_array()) {
3000  // default to 32-bits
3002  cd.columnType.set_comp_param(32);
3003  } else if (cd.columnType.is_decimal() && cd.columnType.get_precision() <= 4) {
3005  cd.columnType.set_comp_param(16);
3006  } else if (cd.columnType.is_decimal() && cd.columnType.get_precision() <= 9) {
3008  cd.columnType.set_comp_param(32);
3009  } else if (cd.columnType.is_decimal() && cd.columnType.get_precision() > 18) {
3010  throw std::runtime_error(cd.columnName + ": Precision too high, max 18.");
3011  } else if (cd.columnType.is_geometry() && cd.columnType.get_output_srid() == 4326) {
3012  // default to GEOINT 32-bits
3014  cd.columnType.set_comp_param(32);
3015  } else if (type == kDATE && g_use_date_in_days_default_encoding) {
3016  // Days encoding for DATE
3018  cd.columnType.set_comp_param(0);
3019  } else {
3021  cd.columnType.set_comp_param(0);
3022  }
3023  } else {
3024  const std::string& comp = *compression->get_encoding_name();
3025  int comp_param;
3026  if (boost::iequals(comp, "fixed")) {
3027  if (!cd.columnType.is_integer() && !cd.columnType.is_time() &&
3028  !cd.columnType.is_decimal()) {
3029  throw std::runtime_error(
3030  cd.columnName +
3031  ": Fixed encoding is only supported for integer or time columns.");
3032  }
3033  // fixed-bits encoding
3034  if (type == kARRAY) {
3035  type = cd.columnType.get_subtype();
3036  }
3037  switch (type) {
3038  case kSMALLINT:
3039  if (compression->get_encoding_param() != 8) {
3040  throw std::runtime_error(
3041  cd.columnName +
3042  ": Compression parameter for Fixed encoding on SMALLINT must be 8.");
3043  }
3044  break;
3045  case kINT:
3046  if (compression->get_encoding_param() != 8 &&
3047  compression->get_encoding_param() != 16) {
3048  throw std::runtime_error(
3049  cd.columnName +
3050  ": Compression parameter for Fixed encoding on INTEGER must be 8 or 16.");
3051  }
3052  break;
3053  case kBIGINT:
3054  if (compression->get_encoding_param() != 8 &&
3055  compression->get_encoding_param() != 16 &&
3056  compression->get_encoding_param() != 32) {
3057  throw std::runtime_error(cd.columnName +
3058  ": Compression parameter for Fixed encoding on "
3059  "BIGINT must be 8 or 16 or 32.");
3060  }
3061  break;
3062  case kTIMESTAMP:
3063  case kTIME:
3064  if (compression->get_encoding_param() != 32) {
3065  throw std::runtime_error(cd.columnName +
3066  ": Compression parameter for Fixed encoding on "
3067  "TIME or TIMESTAMP must be 32.");
3068  } else if (cd.columnType.is_high_precision_timestamp()) {
3069  throw std::runtime_error(
3070  "Fixed encoding is not supported for TIMESTAMP(3|6|9).");
3071  }
3072  break;
3073  case kDECIMAL:
3074  case kNUMERIC:
3075  if (compression->get_encoding_param() != 32 &&
3076  compression->get_encoding_param() != 16) {
3077  throw std::runtime_error(cd.columnName +
3078  ": Compression parameter for Fixed encoding on "
3079  "DECIMAL must be 16 or 32.");
3080  }
3081 
3082  if (compression->get_encoding_param() == 32 &&
3083  cd.columnType.get_precision() > 9) {
3084  throw std::runtime_error(
3085  cd.columnName + ": Precision too high for Fixed(32) encoding, max 9.");
3086  }
3087 
3088  if (compression->get_encoding_param() == 16 &&
3089  cd.columnType.get_precision() > 4) {
3090  throw std::runtime_error(
3091  cd.columnName + ": Precision too high for Fixed(16) encoding, max 4.");
3092  }
3093  break;
3094  case kDATE:
3095  if (compression->get_encoding_param() != 32 &&
3096  compression->get_encoding_param() != 16) {
3097  throw std::runtime_error(cd.columnName +
3098  ": Compression parameter for Fixed encoding on "
3099  "DATE must be 16 or 32.");
3100  }
3101  break;
3102  default:
3103  throw std::runtime_error(cd.columnName + ": Cannot apply FIXED encoding to " +
3104  t->to_string());
3105  }
3106  if (type == kDATE) {
3108  cd.columnType.set_comp_param(16);
3109  } else {
3111  cd.columnType.set_comp_param(compression->get_encoding_param());
3112  }
3113  } else if (boost::iequals(comp, "rl")) {
3114  // run length encoding
3116  cd.columnType.set_comp_param(0);
3117  // throw std::runtime_error("RL(Run Length) encoding not supported yet.");
3118  } else if (boost::iequals(comp, "diff")) {
3119  // differential encoding
3121  cd.columnType.set_comp_param(0);
3122  // throw std::runtime_error("DIFF(differential) encoding not supported yet.");
3123  } else if (boost::iequals(comp, "dict")) {
3124  if (!cd.columnType.is_string() && !cd.columnType.is_string_array()) {
3125  throw std::runtime_error(
3126  cd.columnName +
3127  ": Dictionary encoding is only supported on string or string array columns.");
3128  }
3129  if (compression->get_encoding_param() == 0) {
3130  comp_param = 32; // default to 32-bits
3131  } else {
3132  comp_param = compression->get_encoding_param();
3133  }
3134  if (cd.columnType.is_string_array() && comp_param != 32) {
3135  throw std::runtime_error(cd.columnName +
3136  ": Compression parameter for string arrays must be 32");
3137  }
3138  if (comp_param != 8 && comp_param != 16 && comp_param != 32) {
3139  throw std::runtime_error(
3140  cd.columnName +
3141  ": Compression parameter for Dictionary encoding must be 8 or 16 or 32.");
3142  }
3143  // diciontary encoding
3145  cd.columnType.set_comp_param(comp_param);
3146  } else if (boost::iequals(comp, "NONE")) {
3147  if (cd.columnType.is_geometry()) {
3149  cd.columnType.set_comp_param(64);
3150  } else {
3151  if (!cd.columnType.is_string() && !cd.columnType.is_string_array()) {
3152  throw std::runtime_error(
3153  cd.columnName +
3154  ": None encoding is only supported on string or string array columns.");
3155  }
3157  cd.columnType.set_comp_param(0);
3158  }
3159  } else if (boost::iequals(comp, "sparse")) {
3160  // sparse column encoding with mostly NULL values
3161  if (cd.columnType.get_notnull()) {
3162  throw std::runtime_error(
3163  cd.columnName + ": Cannot do sparse column encoding on a NOT NULL column.");
3164  }
3165  if (compression->get_encoding_param() == 0 ||
3166  compression->get_encoding_param() % 8 != 0 ||
3167  compression->get_encoding_param() > 48) {
3168  throw std::runtime_error(
3169  cd.columnName +
3170  "Must specify number of bits as 8, 16, 24, 32 or 48 as the parameter to "
3171  "sparse-column encoding.");
3172  }
3174  cd.columnType.set_comp_param(compression->get_encoding_param());
3175  // throw std::runtime_error("SPARSE encoding not supported yet.");
3176  } else if (boost::iequals(comp, "compressed")) {
3177  if (!cd.columnType.is_geometry() || cd.columnType.get_output_srid() != 4326) {
3178  throw std::runtime_error(
3179  cd.columnName +
3180  ": COMPRESSED encoding is only supported on WGS84 geo columns.");
3181  }
3182  if (compression->get_encoding_param() == 0) {
3183  comp_param = 32; // default to 32-bits
3184  } else {
3185  comp_param = compression->get_encoding_param();
3186  }
3187  if (comp_param != 32) {
3188  throw std::runtime_error(cd.columnName +
3189  ": only 32-bit COMPRESSED geo encoding is supported");
3190  }
3191  // encoding longitude/latitude as integers
3193  cd.columnType.set_comp_param(comp_param);
3194  } else if (boost::iequals(comp, "days")) {
3195  // days encoding for dates
3196  if (cd.columnType.get_type() != kDATE) {
3197  throw std::runtime_error(cd.columnName +
3198  ": Days encoding is only supported for DATE columns.");
3199  }
3200  if (compression->get_encoding_param() != 32 &&
3201  compression->get_encoding_param() != 16) {
3202  throw std::runtime_error(cd.columnName +
3203  ": Compression parameter for Days encoding on "
3204  "DATE must be 16 or 32.");
3205  }
3207  cd.columnType.set_comp_param((compression->get_encoding_param() == 16) ? 16 : 0);
3208  } else {
3209  throw std::runtime_error(cd.columnName + ": Invalid column compression scheme " +
3210  comp);
3211  }
3212  }
3213  if (cd.columnType.is_string_array() &&
3215  throw std::runtime_error(
3216  cd.columnName +
3217  ": Array of strings must be dictionary encoded. Specify ENCODING DICT");
3218  }
3219  if (t->get_is_array()) {
3220  int s = -1;
3221  auto array_size = t->get_array_size();
3222  if (array_size > 0) {
3223  auto sti = cd.columnType.get_elem_type();
3224  s = array_size * sti.get_size();
3225  if (s <= 0) {
3226  throw std::runtime_error(cd.columnName + ": Unexpected fixed length array size");
3227  }
3228  }
3229  cd.columnType.set_size(s);
3230 
3231  } else {
3233  }
3234  cd.isSystemCol = false;
3235  cd.isVirtualCol = false;
3236 }
3237 
3238 void AddColumnStmt::check_executable(const Catalog_Namespace::SessionInfo& session) {
3239  auto& catalog = session.getCatalog();
3240  const TableDescriptor* td = catalog.getMetadataForTable(*table);
3241  if (nullptr == td) {
3242  throw std::runtime_error("Table " + *table + " does not exist.");
3243  } else {
3244  if (td->isView) {
3245  throw std::runtime_error("Expecting a table , found view " + *table);
3246  }
3247  };
3248 
3249  check_alter_table_privilege(session, td);
3250 
3251  if (0 == coldefs.size()) {
3252  coldefs.push_back(std::move(coldef));
3253  }
3254 
3255  for (const auto& coldef : coldefs) {
3256  auto& new_column_name = *coldef->get_column_name();
3257  if (catalog.getMetadataForColumn(td->tableId, new_column_name) != nullptr) {
3258  throw std::runtime_error("Column " + new_column_name + " already exists.");
3259  }
3260  if (reserved_keywords.find(boost::to_upper_copy<std::string>(new_column_name)) !=
3261  reserved_keywords.end()) {
3262  throw std::runtime_error("Cannot add column with reserved keyword '" +
3263  new_column_name + "'");
3264  }
3265  }
3266 }
3267 
3268 void AddColumnStmt::execute(const Catalog_Namespace::SessionInfo& session) {
3269  auto& catalog = session.getCatalog();
3270  const TableDescriptor* td = catalog.getMetadataForTable(*table);
3271  check_executable(session);
3272 
3273  catalog.getSqliteConnector().query("BEGIN TRANSACTION");
3274  try {
3275  std::map<const std::string, const ColumnDescriptor> cds;
3276  std::map<const int, const ColumnDef*> cid_coldefs;
3277  for (const auto& coldef : coldefs) {
3278  ColumnDescriptor cd;
3279  setColumnDescriptor(cd, coldef.get());
3280  catalog.addColumn(*td, cd);
3281  cds.emplace(*coldef->get_column_name(), cd);
3282  cid_coldefs.emplace(cd.columnId, coldef.get());
3283 
3284  // expand geo column to phy columns
3285  if (cd.columnType.is_geometry()) {
3286  std::list<ColumnDescriptor> phy_geo_columns;
3287  catalog.expandGeoColumn(cd, phy_geo_columns);
3288  for (auto& cd : phy_geo_columns) {
3289  catalog.addColumn(*td, cd);
3290  cds.emplace(cd.columnName, cd);
3291  cid_coldefs.emplace(cd.columnId, nullptr);
3292  }
3293  }
3294  }
3295 
3296  std::unique_ptr<Importer_NS::Loader> loader(new Importer_NS::Loader(catalog, td));
3297  std::vector<std::unique_ptr<Importer_NS::TypedImportBuffer>> import_buffers;
3298 
3299  // a call of Catalog_Namespace::MapDHandler::prepare_columnar_loader
3301  td->tableName,
3302  td->nColumns - 1,
3303  &loader,
3304  &import_buffers);
3305  loader->setReplicating(true);
3306 
3307  // set_geo_physical_import_buffer below needs a sorted import_buffers
3308  std::sort(import_buffers.begin(),
3309  import_buffers.end(),
3310  [](decltype(import_buffers[0])& a, decltype(import_buffers[0])& b) {
3311  return a->getColumnDesc()->columnId < b->getColumnDesc()->columnId;
3312  });
3313 
3314  size_t nrows = td->fragmenter->getNumRows();
3315  // if sharded, get total nrows from all sharded tables
3316  if (td->nShards > 0) {
3317  const auto physical_tds = catalog.getPhysicalTablesDescriptors(td);
3318  nrows = 0;
3319  std::for_each(physical_tds.begin(), physical_tds.end(), [&nrows](const auto& td) {
3320  nrows += td->fragmenter->getNumRows();
3321  });
3322  }
3323  if (nrows > 0) {
3324  int skip_physical_cols = 0;
3325  for (const auto cit : cid_coldefs) {
3326  const auto cd = catalog.getMetadataForColumn(td->tableId, cit.first);
3327  const auto coldef = cit.second;
3328  const auto column_constraint = coldef ? coldef->get_column_constraint() : nullptr;
3329  std::string defaultval = "";
3330  if (column_constraint) {
3331  auto defaultlp = column_constraint->get_defaultval();
3332  auto defaultsp = dynamic_cast<const StringLiteral*>(defaultlp);
3333  defaultval = defaultsp ? *defaultsp->get_stringval()
3334  : defaultlp ? defaultlp->to_string() : "";
3335  }
3336  bool isnull = column_constraint ? (0 == defaultval.size()) : true;
3337  if (boost::to_upper_copy<std::string>(defaultval) == "NULL") {
3338  isnull = true;
3339  }
3340 
3341  // TODO: remove is_geometry below once if null is allowed for geo
3342  if (isnull) {
3343  if (cd->columnType.is_geometry() ||
3344  (column_constraint && column_constraint->get_notnull())) {
3345  throw std::runtime_error("Default value required for column " +
3346  cd->columnName + " (NULL value not supported)");
3347  }
3348  }
3349 
3350  for (auto it = import_buffers.begin(); it < import_buffers.end(); ++it) {
3351  auto& import_buffer = *it;
3352  if (cd->columnId == import_buffer->getColumnDesc()->columnId) {
3353  if (coldef != nullptr ||
3354  skip_physical_cols-- <= 0) { // skip non-null phy col
3355  import_buffer->add_value(
3356  cd, defaultval, isnull, Importer_NS::CopyParams(), nrows);
3357  // tedious non-null geo default value ...
3358  if (cd->columnType.is_geometry() && !isnull) {
3359  std::vector<double> coords, bounds;
3360  std::vector<int> ring_sizes, poly_rings;
3361  int render_group = 0;
3362  SQLTypeInfo tinfo;
3364  tinfo,
3365  coords,
3366  bounds,
3367  ring_sizes,
3368  poly_rings,
3369  false)) {
3370  throw std::runtime_error("Bad geometry data: '" + defaultval + "'");
3371  }
3372  size_t col_idx = 1 + std::distance(import_buffers.begin(), it);
3374  cd,
3375  import_buffers,
3376  col_idx,
3377  coords,
3378  bounds,
3379  ring_sizes,
3380  poly_rings,
3381  render_group,
3382  nrows);
3383  // skip following phy cols
3384  skip_physical_cols = cd->columnType.get_physical_cols();
3385  }
3386  }
3387  break;
3388  }
3389  }
3390  }
3391  }
3392 
3393  if (!loader->loadNoCheckpoint(import_buffers, nrows)) {
3394  throw std::runtime_error("loadNoCheckpoint failed!");
3395  }
3396  loader->checkpoint();
3397  } catch (...) {
3398  catalog.roll(false);
3399  catalog.getSqliteConnector().query("ROLLBACK TRANSACTION");
3400  throw;
3401  }
3402  catalog.getSqliteConnector().query("END TRANSACTION");
3403  catalog.roll(true);
3404 }
3405 
3406 void RenameColumnStmt::execute(const Catalog_Namespace::SessionInfo& session) {
3407  auto& catalog = session.getCatalog();
3408  const TableDescriptor* td = catalog.getMetadataForTable(*table);
3409  if (td == nullptr) {
3410  throw std::runtime_error("Table " + *table + " does not exist.");
3411  }
3412  check_alter_table_privilege(session, td);
3413  const ColumnDescriptor* cd = catalog.getMetadataForColumn(td->tableId, *column);
3414  if (cd == nullptr) {
3415  throw std::runtime_error("Column " + *column + " does not exist.");
3416  }
3417  if (catalog.getMetadataForColumn(td->tableId, *new_column_name) != nullptr) {
3418  throw std::runtime_error("Column " + *new_column_name + " already exists.");
3419  }
3420  if (reserved_keywords.find(boost::to_upper_copy<std::string>(*new_column_name)) !=
3421  reserved_keywords.end()) {
3422  throw std::runtime_error("Cannot create column with reserved keyword '" +
3423  *new_column_name + "'");
3424  }
3425  catalog.renameColumn(td, cd, *new_column_name);
3426 }
3427 
3428 void CopyTableStmt::execute(const Catalog_Namespace::SessionInfo& session) {
3429  auto importer_factory = [](Catalog_Namespace::Catalog& catalog,
3430  const TableDescriptor* td,
3431  const std::string& file_path,
3432  const Importer_NS::CopyParams& copy_params) {
3433  return boost::make_unique<Importer_NS::Importer>(catalog, td, file_path, copy_params);
3434  };
3435  return execute(session, importer_factory);
3436 }
3437 
3438 void CopyTableStmt::execute(const Catalog_Namespace::SessionInfo& session,
3439  const std::function<std::unique_ptr<Importer_NS::Importer>(
3441  const TableDescriptor*,
3442  const std::string&,
3443  const Importer_NS::CopyParams&)>& importer_factory) {
3444  size_t rows_completed = 0;
3445  size_t rows_rejected = 0;
3446  size_t total_time = 0;
3447  bool load_truncated = false;
3448 
3449  auto& catalog = session.getCatalog();
3450  const TableDescriptor* td = catalog.getMetadataForTable(*table);
3451 
3452  // if the table already exists, it's locked, so check access privileges
3453  if (td) {
3454  std::vector<DBObject> privObjects;
3455  DBObject dbObject(*table, TableDBObjectType);
3456  dbObject.loadKey(catalog);
3458  privObjects.push_back(dbObject);
3459  if (!SysCatalog::instance().checkPrivileges(session.get_currentUser(), privObjects)) {
3460  throw std::runtime_error("Violation of access privileges: user " +
3461  session.get_currentUser().userName +
3462  " has no insert privileges for table " + *table + ".");
3463  }
3464  }
3465 
3466  // since we'll have not only posix file names but also s3/hdfs/... url
3467  // we do not expand wildcard or check file existence here.
3468  // from here on, file_path contains something which may be a url
3469  // or a wildcard of file names;
3470  std::string file_path = *file_pattern;
3471  Importer_NS::CopyParams copy_params;
3472  if (!options.empty()) {
3473  for (auto& p : options) {
3474  if (boost::iequals(*p->get_name(), "max_reject")) {
3475  const IntLiteral* int_literal = dynamic_cast<const IntLiteral*>(p->get_value());
3476  if (int_literal == nullptr) {
3477  throw std::runtime_error("max_reject option must be an integer.");
3478  }
3479  copy_params.max_reject = int_literal->get_intval();
3480  } else if (boost::iequals(*p->get_name(), "buffer_size")) {
3481  const IntLiteral* int_literal = dynamic_cast<const IntLiteral*>(p->get_value());
3482  if (int_literal == nullptr) {
3483  throw std::runtime_error("buffer_size option must be an integer.");
3484  }
3485  copy_params.buffer_size = std::max<size_t>(1 << 20, int_literal->get_intval());
3486  } else if (boost::iequals(*p->get_name(), "threads")) {
3487  const IntLiteral* int_literal = dynamic_cast<const IntLiteral*>(p->get_value());
3488  if (int_literal == nullptr) {
3489  throw std::runtime_error("Threads option must be an integer.");
3490  }
3491  copy_params.threads = int_literal->get_intval();
3492  } else if (boost::iequals(*p->get_name(), "delimiter")) {
3493  const StringLiteral* str_literal =
3494  dynamic_cast<const StringLiteral*>(p->get_value());
3495  if (str_literal == nullptr) {
3496  throw std::runtime_error("Delimiter option must be a string.");
3497  } else if (str_literal->get_stringval()->length() != 1) {
3498  throw std::runtime_error("Delimiter must be a single character string.");
3499  }
3500  copy_params.delimiter = (*str_literal->get_stringval())[0];
3501  } else if (boost::iequals(*p->get_name(), "nulls")) {
3502  const StringLiteral* str_literal =
3503  dynamic_cast<const StringLiteral*>(p->get_value());
3504  if (str_literal == nullptr) {
3505  throw std::runtime_error("Nulls option must be a string.");
3506  }
3507  copy_params.null_str = *str_literal->get_stringval();
3508  } else if (boost::iequals(*p->get_name(), "header")) {
3509  const StringLiteral* str_literal =
3510  dynamic_cast<const StringLiteral*>(p->get_value());
3511  if (str_literal == nullptr) {
3512  throw std::runtime_error("Header option must be a boolean.");
3513  }
3514  copy_params.has_header = bool_from_string_literal(str_literal)
3517 #ifdef ENABLE_IMPORT_PARQUET
3518  } else if (boost::iequals(*p->get_name(), "parquet")) {
3519  const StringLiteral* str_literal =
3520  dynamic_cast<const StringLiteral*>(p->get_value());
3521  if (str_literal == nullptr) {
3522  throw std::runtime_error("Parquet option must be a boolean.");
3523  }
3524  if (bool_from_string_literal(str_literal)) {
3525  // not sure a parquet "table" type is proper, but to make code
3526  // look consistent in some places, let's set "table" type too
3527  copy_params.file_type = Importer_NS::FileType::PARQUET;
3528  }
3529 #endif // ENABLE_IMPORT_PARQUET
3530  } else if (boost::iequals(*p->get_name(), "s3_access_key")) {
3531  const StringLiteral* str_literal =
3532  dynamic_cast<const StringLiteral*>(p->get_value());
3533  if (str_literal == nullptr) {
3534  throw std::runtime_error("Option s3_access_key must be a string.");
3535  }
3536  copy_params.s3_access_key = *str_literal->get_stringval();
3537  } else if (boost::iequals(*p->get_name(), "s3_secret_key")) {
3538  const StringLiteral* str_literal =
3539  dynamic_cast<const StringLiteral*>(p->get_value());
3540  if (str_literal == nullptr) {
3541  throw std::runtime_error("Option s3_secret_key must be a string.");
3542  }
3543  copy_params.s3_secret_key = *str_literal->get_stringval();
3544  } else if (boost::iequals(*p->get_name(), "s3_region")) {
3545  const StringLiteral* str_literal =
3546  dynamic_cast<const StringLiteral*>(p->get_value());
3547  if (str_literal == nullptr) {
3548  throw std::runtime_error("Option s3_region must be a string.");
3549  }
3550  copy_params.s3_region = *str_literal->get_stringval();
3551  } else if (boost::iequals(*p->get_name(), "s3_endpoint")) {
3552  const StringLiteral* str_literal =
3553  dynamic_cast<const StringLiteral*>(p->get_value());
3554  if (str_literal == nullptr) {
3555  throw std::runtime_error("Option s3_endpoint must be a string.");
3556  }
3557  copy_params.s3_endpoint = *str_literal->get_stringval();
3558  } else if (boost::iequals(*p->get_name(), "quote")) {
3559  const StringLiteral* str_literal =
3560  dynamic_cast<const StringLiteral*>(p->get_value());
3561  if (str_literal == nullptr) {
3562  throw std::runtime_error("Quote option must be a string.");
3563  } else if (str_literal->get_stringval()->length() != 1) {
3564  throw std::runtime_error("Quote must be a single character string.");
3565  }
3566  copy_params.quote = (*str_literal->get_stringval())[0];
3567  } else if (boost::iequals(*p->get_name(), "escape")) {
3568  const StringLiteral* str_literal =
3569  dynamic_cast<const StringLiteral*>(p->get_value());
3570  if (str_literal == nullptr) {
3571  throw std::runtime_error("Escape option must be a string.");
3572  } else if (str_literal->get_stringval()->length() != 1) {
3573  throw std::runtime_error("Escape must be a single character string.");
3574  }
3575  copy_params.escape = (*str_literal->get_stringval())[0];
3576  } else if (boost::iequals(*p->get_name(), "line_delimiter")) {
3577  const StringLiteral* str_literal =
3578  dynamic_cast<const StringLiteral*>(p->get_value());
3579  if (str_literal == nullptr) {
3580  throw std::runtime_error("Line_delimiter option must be a string.");
3581  } else if (str_literal->get_stringval()->length() != 1) {
3582  throw std::runtime_error("Line_delimiter must be a single character string.");
3583  }
3584  copy_params.line_delim = (*str_literal->get_stringval())[0];
3585  } else if (boost::iequals(*p->get_name(), "quoted")) {
3586  const StringLiteral* str_literal =
3587  dynamic_cast<const StringLiteral*>(p->get_value());
3588  if (str_literal == nullptr) {
3589  throw std::runtime_error("Quoted option must be a boolean.");
3590  }
3591  copy_params.quoted = bool_from_string_literal(str_literal);
3592  } else if (boost::iequals(*p->get_name(), "plain_text")) {
3593  const StringLiteral* str_literal =
3594  dynamic_cast<const StringLiteral*>(p->get_value());
3595  if (str_literal == nullptr) {
3596  throw std::runtime_error("plain_text option must be a boolean.");
3597  }
3598  copy_params.plain_text = bool_from_string_literal(str_literal);
3599  } else if (boost::iequals(*p->get_name(), "array_marker")) {
3600  const StringLiteral* str_literal =
3601  dynamic_cast<const StringLiteral*>(p->get_value());
3602  if (str_literal == nullptr) {
3603  throw std::runtime_error("Array Marker option must be a string.");
3604  } else if (str_literal->get_stringval()->length() != 2) {
3605  throw std::runtime_error(
3606  "Array Marker option must be exactly two characters. Default is {}.");
3607  }
3608  copy_params.array_begin = (*str_literal->get_stringval())[0];
3609  copy_params.array_end = (*str_literal->get_stringval())[1];
3610  } else if (boost::iequals(*p->get_name(), "array_delimiter")) {
3611  const StringLiteral* str_literal =
3612  dynamic_cast<const StringLiteral*>(p->get_value());
3613  if (str_literal == nullptr) {
3614  throw std::runtime_error("Array Delimiter option must be a string.");
3615  } else if (str_literal->get_stringval()->length() != 1) {
3616  throw std::runtime_error("Array Delimiter must be a single character string.");
3617  }
3618  copy_params.array_delim = (*str_literal->get_stringval())[0];
3619  } else if (boost::iequals(*p->get_name(), "lonlat")) {
3620  const StringLiteral* str_literal =
3621  dynamic_cast<const StringLiteral*>(p->get_value());
3622  if (str_literal == nullptr) {
3623  throw std::runtime_error("Lonlat option must be a boolean.");
3624  }
3625  copy_params.lonlat = bool_from_string_literal(str_literal);
3626  } else if (boost::iequals(*p->get_name(), "geo")) {
3627  const StringLiteral* str_literal =
3628  dynamic_cast<const StringLiteral*>(p->get_value());
3629  if (str_literal == nullptr) {
3630  throw std::runtime_error("Geo option must be a boolean.");
3631  }
3632  copy_params.file_type = bool_from_string_literal(str_literal)
3635  } else if (boost::iequals(*p->get_name(), "geo_coords_type")) {
3636  const StringLiteral* str_literal =
3637  dynamic_cast<const StringLiteral*>(p->get_value());
3638  if (str_literal == nullptr) {
3639  throw std::runtime_error("'geo_coords_type' option must be a string");
3640  }
3641  const std::string* s = str_literal->get_stringval();
3642  if (boost::iequals(*s, "geography")) {
3643  throw std::runtime_error(
3644  "GEOGRAPHY coords type not yet supported. Please use GEOMETRY.");
3645  // copy_params.geo_coords_type = kGEOGRAPHY;
3646  } else if (boost::iequals(*s, "geometry")) {
3647  copy_params.geo_coords_type = kGEOMETRY;
3648  } else {
3649  throw std::runtime_error(
3650  "Invalid string for 'geo_coords_type' option (must be 'GEOGRAPHY' or "
3651  "'GEOMETRY'): " +
3652  *s);
3653  }
3654  } else if (boost::iequals(*p->get_name(), "geo_coords_encoding")) {
3655  const StringLiteral* str_literal =
3656  dynamic_cast<const StringLiteral*>(p->get_value());
3657  if (str_literal == nullptr) {
3658  throw std::runtime_error("'geo_coords_encoding' option must be a string");
3659  }
3660  const std::string* s = str_literal->get_stringval();
3661  if (boost::iequals(*s, "none")) {
3662  copy_params.geo_coords_encoding = kENCODING_NONE;
3663  copy_params.geo_coords_comp_param = 0;
3664  } else if (boost::iequals(*s, "compressed(32)")) {
3665  copy_params.geo_coords_encoding = kENCODING_GEOINT;
3666  copy_params.geo_coords_comp_param = 32;
3667  } else {
3668  throw std::runtime_error(
3669  "Invalid string for 'geo_coords_encoding' option (must be 'NONE' or "
3670  "'COMPRESSED(32)'): " +
3671  *s);
3672  }
3673  } else if (boost::iequals(*p->get_name(), "geo_coords_srid")) {
3674  const IntLiteral* int_literal = dynamic_cast<const IntLiteral*>(p->get_value());
3675  if (int_literal == nullptr) {
3676  throw std::runtime_error("'geo_coords_srid' option must be an integer");
3677  }
3678  const int srid = int_literal->get_intval();
3679  if (srid == 4326 || srid == 3857 || srid == 900913) {
3680  copy_params.geo_coords_srid = srid;
3681  } else {
3682  throw std::runtime_error(
3683  "Invalid value for 'geo_coords_srid' option (must be 4326, 3857, or "
3684  "900913): " +
3685  std::to_string(srid));
3686  }
3687  } else if (boost::iequals(*p->get_name(), "geo_layer_name")) {
3688  const StringLiteral* str_literal =
3689  dynamic_cast<const StringLiteral*>(p->get_value());
3690  if (str_literal == nullptr) {
3691  throw std::runtime_error("'geo_layer_name' option must be a string");
3692  }
3693  const std::string* layer_name = str_literal->get_stringval();
3694  if (layer_name) {
3695  copy_params.geo_layer_name = *layer_name;
3696  } else {
3697  throw std::runtime_error("Invalid value for 'geo_layer_name' option");
3698  }
3699  } else if (boost::iequals(*p->get_name(), "partitions")) {
3700  if (copy_params.file_type == Importer_NS::FileType::POLYGON) {
3701  const auto partitions =
3702  static_cast<const StringLiteral*>(p->get_value())->get_stringval();
3703  CHECK(partitions);
3704  const auto partitions_uc = boost::to_upper_copy<std::string>(*partitions);
3705  if (partitions_uc != "REPLICATED") {
3706  throw std::runtime_error("PARTITIONS must be REPLICATED for geo COPY");
3707  }
3708  _geo_copy_from_partitions = partitions_uc;
3709  } else {
3710  throw std::runtime_error("PARTITIONS option not supported for non-geo COPY: " +
3711  *p->get_name());
3712  }
3713  } else if (boost::iequals(*p->get_name(), "geo_assign_render_groups")) {
3714  const StringLiteral* str_literal =
3715  dynamic_cast<const StringLiteral*>(p->get_value());
3716  if (str_literal == nullptr) {
3717  throw std::runtime_error("geo_assign_render_groups option must be a boolean.");
3718  }
3719  copy_params.geo_assign_render_groups = bool_from_string_literal(str_literal);
3720  } else if (boost::iequals(*p->get_name(), "geo_explode_collections")) {
3721  const StringLiteral* str_literal =
3722  dynamic_cast<const StringLiteral*>(p->get_value());
3723  if (str_literal == nullptr) {
3724  throw std::runtime_error("geo_explode_collections option must be a boolean.");
3725  }
3726  copy_params.geo_explode_collections = bool_from_string_literal(str_literal);
3727  } else {
3728  throw std::runtime_error("Invalid option for COPY: " + *p->get_name());
3729  }
3730  }
3731  }
3732 
3733  std::string tr;
3734  if (copy_params.file_type == Importer_NS::FileType::POLYGON) {
3735  // geo import
3736  // we do nothing here, except stash the parameters so we can
3737  // do the import when we unwind to the top of the handler
3738  _geo_copy_from_file_name = file_path;
3739  _geo_copy_from_copy_params = copy_params;
3740  _was_geo_copy_from = true;
3741 
3742  // the result string
3743  // @TODO simon.eves put something more useful in here
3744  // except we really can't because we haven't done the import yet!
3745  if (td) {
3746  tr = std::string("Appending geo to table '") + *table + std::string("'...");
3747  } else {
3748  tr = std::string("Creating table '") + *table +
3749  std::string("' and importing geo...");
3750  }
3751  } else {
3752  if (td) {
3753  // regular import
3754  auto importer = importer_factory(catalog, td, file_path, copy_params);
3755  auto ms = measure<>::execution([&]() {
3756  auto res = importer->import();
3757  rows_completed += res.rows_completed;
3758  rows_rejected += res.rows_rejected;
3759  load_truncated = res.load_truncated;
3760  });
3761  total_time += ms;
3762 
3763  // results
3764  if (load_truncated || rows_rejected > copy_params.max_reject) {
3765  LOG(ERROR) << "COPY exited early due to reject records count during multi file "
3766  "processing ";
3767  // if we have crossed the truncated load threshold
3768  load_truncated = true;
3769  }
3770  if (!load_truncated) {
3771  tr = std::string("Loaded: " + std::to_string(rows_completed) +
3772  " recs, Rejected: " + std::to_string(rows_rejected) +
3773  " recs in " + std::to_string((double)total_time / 1000.0) +
3774  " secs");
3775  } else {
3776  tr = std::string("Loader truncated due to reject count. Processed : " +
3777  std::to_string(rows_completed) + " recs, Rejected: " +
3778  std::to_string(rows_rejected) + " recs in " +
3779  std::to_string((double)total_time / 1000.0) + " secs");
3780  }
3781  } else {
3782  throw std::runtime_error("Table '" + *table + "' must exist before COPY FROM");
3783  }
3784  }
3785 
3786  return_message.reset(new std::string(tr));
3787  LOG(INFO) << tr;
3788 }
3789 
3790 // CREATE ROLE payroll_dept_role;
3791 void CreateRoleStmt::execute(const Catalog_Namespace::SessionInfo& session) {
3792  const auto& currentUser = session.get_currentUser();
3793  if (!currentUser.isSuper) {
3794  throw std::runtime_error("CREATE ROLE " + get_role() +
3795  " failed. It can only be executed by super user.");
3796  }
3797  SysCatalog::instance().createRole(get_role());
3798 }
3799 
3800 // DROP ROLE payroll_dept_role;
3801 void DropRoleStmt::execute(const Catalog_Namespace::SessionInfo& session) {
3802  const auto& currentUser = session.get_currentUser();
3803  if (!currentUser.isSuper) {
3804  throw std::runtime_error("DROP ROLE " + get_role() +
3805  " failed. It can only be executed by super user.");
3806  }
3807  auto* rl = SysCatalog::instance().getRoleGrantee(get_role());
3808  if (!rl) {
3809  throw std::runtime_error("DROP ROLE " + get_role() +
3810  " failed because role with this name does not exist.");
3811  }
3812  SysCatalog::instance().dropRole(get_role());
3813 }
3814 
3815 std::vector<std::string> splitObjectHierName(const std::string& hierName) {
3816  std::vector<std::string> componentNames;
3817  boost::split(componentNames, hierName, boost::is_any_of("."));
3818  return componentNames;
3819 }
3820 
3821 std::string extractObjectNameFromHierName(const std::string& objectHierName,
3822  const std::string& objectType,
3823  const Catalog_Namespace::Catalog& cat) {
3824  std::string objectName;
3825  std::vector<std::string> componentNames = splitObjectHierName(objectHierName);
3826  if (objectType.compare("DATABASE") == 0) {
3827  if (componentNames.size() == 1) {
3828  objectName = componentNames[0];
3829  } else {
3830  throw std::runtime_error("DB object name is not correct " + objectHierName);
3831  }
3832  } else {
3833  if (objectType.compare("TABLE") == 0 || objectType.compare("DASHBOARD") == 0 ||
3834  objectType.compare("VIEW") == 0) {
3835  switch (componentNames.size()) {
3836  case (1): {
3837  objectName = componentNames[0];
3838  break;
3839  }
3840  case (2): {
3841  objectName = componentNames[1];
3842  break;
3843  }
3844  default: {
3845  throw std::runtime_error("DB object name is not correct " + objectHierName);
3846  }
3847  }
3848  } else {
3849  throw std::runtime_error("DB object type " + objectType + " is not supported.");
3850  }
3851  }
3852  return objectName;
3853 }
3854 
3855 static std::pair<AccessPrivileges, DBObjectType> parseStringPrivs(
3856  const std::string& privs,
3857  const DBObjectType& objectType,
3858  const std::string& object_name) {
3859  static const std::map<std::pair<const std::string, const DBObjectType>,
3860  std::pair<const AccessPrivileges, const DBObjectType>>
3861  privileges_lookup{
3862  {{"ALL"s, DatabaseDBObjectType},
3865  {{"ALL"s, DashboardDBObjectType},
3868 
3869  {{"CREATE TABLE"s, DatabaseDBObjectType},
3871  {{"CREATE"s, DatabaseDBObjectType},
3873  {{"SELECT"s, DatabaseDBObjectType},
3875  {{"INSERT"s, DatabaseDBObjectType},
3877  {{"TRUNCATE"s, DatabaseDBObjectType},
3879  {{"UPDATE"s, DatabaseDBObjectType},
3881  {{"DELETE"s, DatabaseDBObjectType},
3883  {{"DROP"s, DatabaseDBObjectType},
3885  {{"ALTER"s, DatabaseDBObjectType},
3887 
3888  {{"SELECT"s, TableDBObjectType},
3890  {{"INSERT"s, TableDBObjectType},
3892  {{"TRUNCATE"s, TableDBObjectType},
3894  {{"UPDATE"s, TableDBObjectType},
3896  {{"DELETE"s, TableDBObjectType},
3898  {{"DROP"s, TableDBObjectType},
3900  {{"ALTER"s, TableDBObjectType},
3902 
3903  {{"CREATE VIEW"s, DatabaseDBObjectType},
3905  {{"SELECT VIEW"s, DatabaseDBObjectType},
3907  {{"DROP VIEW"s, DatabaseDBObjectType},
3909  {{"SELECT"s, ViewDBObjectType},
3912 
3913  {{"CREATE DASHBOARD"s, DatabaseDBObjectType},
3915  {{"EDIT DASHBOARD"s, DatabaseDBObjectType},
3917  {{"VIEW DASHBOARD"s, DatabaseDBObjectType},
3919  {{"DELETE DASHBOARD"s, DatabaseDBObjectType},
3921  {{"VIEW"s, DashboardDBObjectType},
3923  {{"EDIT"s, DashboardDBObjectType},
3925  {{"DELETE"s, DashboardDBObjectType},
3927 
3928  {{"VIEW SQL EDITOR"s, DatabaseDBObjectType},
3930  {{"ACCESS"s, DatabaseDBObjectType},
3932 
3933  auto result = privileges_lookup.find(std::make_pair(privs, objectType));
3934  if (result == privileges_lookup.end()) {
3935  throw std::runtime_error("Privileges " + privs + " on DB object " + object_name +
3936  " are not correct.");
3937  }
3938  return result->second;
3939 }
3940 
3941 static DBObject createObject(const std::string& objectName, DBObjectType objectType) {
3942  if (objectType == DashboardDBObjectType) {
3943  int32_t dashboard_id = -1;
3944  if (!objectName.empty()) {
3945  try {
3946  dashboard_id = stoi(objectName);
3947  } catch (const std::exception&) {
3948  throw std::runtime_error(
3949  "Privileges on dashboards should be changed via integer dashboard ID");
3950  }
3951  }
3952  return DBObject(dashboard_id, objectType);
3953  } else {
3954  return DBObject(objectName, objectType);
3955  }
3956 }
3957 
3958 // GRANT SELECT/INSERT/CREATE ON TABLE payroll_table TO payroll_dept_role;
3959 void GrantPrivilegesStmt::execute(const Catalog_Namespace::SessionInfo& session) {
3960  auto& catalog = session.getCatalog();
3961  const auto& currentUser = session.get_currentUser();
3962  const auto parserObjectType = boost::to_upper_copy<std::string>(get_object_type());
3963  const auto objectName =
3964  extractObjectNameFromHierName(get_object(), parserObjectType, catalog);
3965  auto objectType = DBObjectTypeFromString(parserObjectType);
3966  DBObject dbObject = createObject(objectName, objectType);
3967  /* verify object ownership if not suser */
3968  if (!currentUser.isSuper) {
3969  if (!SysCatalog::instance().verifyDBObjectOwnership(currentUser, dbObject, catalog)) {
3970  throw std::runtime_error(
3971  "GRANT failed. It can only be executed by super user or owner of the object.");
3972  }
3973  }
3974  /* set proper values of privileges & grant them to the object */
3975  std::vector<DBObject> objects(get_privs().size(), dbObject);
3976  for (size_t i = 0; i < get_privs().size(); ++i) {
3977  std::pair<AccessPrivileges, DBObjectType> priv = parseStringPrivs(
3978  boost::to_upper_copy<std::string>(get_privs()[i]), objectType, get_object());
3979  objects[i].setPrivileges(priv.first);
3980  objects[i].setPermissionType(priv.second);
3981  }
3982  SysCatalog::instance().grantDBObjectPrivilegesBatch(grantees, objects, catalog);
3983 }
3984 
3985 // REVOKE SELECT/INSERT/CREATE ON TABLE payroll_table FROM payroll_dept_role;
3986 void RevokePrivilegesStmt::execute(const Catalog_Namespace::SessionInfo& session) {
3987  auto& catalog = session.getCatalog();
3988  const auto& currentUser = session.get_currentUser();
3989  const auto parserObjectType = boost::to_upper_copy<std::string>(get_object_type());
3990  const auto objectName =
3991  extractObjectNameFromHierName(get_object(), parserObjectType, catalog);
3992  auto objectType = DBObjectTypeFromString(parserObjectType);
3993  DBObject dbObject = createObject(objectName, objectType);
3994  /* verify object ownership if not suser */
3995  if (!currentUser.isSuper) {
3996  if (!SysCatalog::instance().verifyDBObjectOwnership(currentUser, dbObject, catalog)) {
3997  throw std::runtime_error(
3998  "REVOKE failed. It can only be executed by super user or owner of the object.");
3999  }
4000  }
4001  /* set proper values of privileges & grant them to the object */
4002  std::vector<DBObject> objects(get_privs().size(), dbObject);
4003  for (size_t i = 0; i < get_privs().size(); ++i) {
4004  std::pair<AccessPrivileges, DBObjectType> priv = parseStringPrivs(
4005  boost::to_upper_copy<std::string>(get_privs()[i]), objectType, get_object());
4006  objects[i].setPrivileges(priv.first);
4007  objects[i].setPermissionType(priv.second);
4008  }
4009  SysCatalog::instance().revokeDBObjectPrivilegesBatch(grantees, objects, catalog);
4010 }
4011 
4012 // NOTE: not used currently, will we ever use it?
4013 // SHOW ON TABLE payroll_table FOR payroll_dept_role;
4014 void ShowPrivilegesStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4015  auto& catalog = session.getCatalog();
4016  const auto& currentUser = session.get_currentUser();
4017  const auto parserObjectType = boost::to_upper_copy<std::string>(get_object_type());
4018  const auto objectName =
4019  extractObjectNameFromHierName(get_object(), parserObjectType, catalog);
4020  auto objectType = DBObjectTypeFromString(parserObjectType);
4021  DBObject dbObject = createObject(objectName, objectType);
4022  /* verify object ownership if not suser */
4023  if (!currentUser.isSuper) {
4024  if (!SysCatalog::instance().verifyDBObjectOwnership(currentUser, dbObject, catalog)) {
4025  throw std::runtime_error(
4026  "SHOW ON " + get_object() + " FOR " + get_role() +
4027  " failed. It can only be executed by super user or owner of the object.");
4028  }
4029  }
4030  /* get values of privileges for the object and report them */
4031  SysCatalog::instance().getDBObjectPrivileges(get_role(), dbObject, catalog);
4032  AccessPrivileges privs = dbObject.getPrivileges();
4033  printf("\nPRIVILEGES ON %s FOR %s ARE SET AS FOLLOWING: ",
4034  get_object().c_str(),
4035  get_role().c_str());
4036 
4037  if (objectType == DBObjectType::DatabaseDBObjectType) {
4039  printf(" CREATE");
4040  }
4042  printf(" DROP");
4043  }
4044  } else if (objectType == DBObjectType::TableDBObjectType) {
4046  printf(" CREATE");
4047  }
4049  printf(" DROP");
4050  }
4052  printf(" SELECT");
4053  }
4055  printf(" INSERT");
4056  }
4058  printf(" UPDATE");
4059  }
4061  printf(" DELETE");
4062  }
4064  printf(" TRUNCATE");
4065  }
4067  printf(" ALTER");
4068  }
4069  } else if (objectType == DBObjectType::DashboardDBObjectType) {
4071  printf(" CREATE");
4072  }
4074  printf(" DELETE");
4075  }
4077  printf(" VIEW");
4078  }
4080  printf(" EDIT");
4081  }
4082  } else if (objectType == DBObjectType::ViewDBObjectType) {
4084  printf(" CREATE");
4085  }
4087  printf(" DROP");
4088  }
4090  printf(" SELECT");
4091  }
4093  printf(" INSERT");
4094  }
4096  printf(" UPDATE");
4097  }
4099  printf(" DELETE");
4100  }
4101  }
4102  printf(".\n");
4103 }
4104 
4105 // GRANT payroll_dept_role TO joe;
4106 void GrantRoleStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4107  const auto& currentUser = session.get_currentUser();
4108  if (!currentUser.isSuper) {
4109  throw std::runtime_error(
4110  "GRANT failed, because it can only be executed by super user.");
4111  }
4112  if (std::find(get_grantees().begin(), get_grantees().end(), OMNISCI_ROOT_USER) !=
4113  get_grantees().end()) {
4114  throw std::runtime_error(
4115  "Request to grant role failed because mapd root user has all privileges by "
4116  "default.");
4117  }
4118  SysCatalog::instance().grantRoleBatch(get_roles(), get_grantees());
4119 }
4120 
4121 // REVOKE payroll_dept_role FROM joe;get_users
4122 void RevokeRoleStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4123  const auto& currentUser = session.get_currentUser();
4124  if (!currentUser.isSuper) {
4125  throw std::runtime_error(
4126  "REVOKE failed, because it can only be executed by super user.");
4127  }
4128  if (std::find(get_grantees().begin(), get_grantees().end(), OMNISCI_ROOT_USER) !=
4129  get_grantees().end()) {
4130  throw std::runtime_error(
4131  "Request to revoke role failed because privileges can not be revoked from mapd "
4132  "root user.");
4133  }
4134  SysCatalog::instance().revokeRoleBatch(get_roles(), get_grantees());
4135 }
4136 
4137 using dbl = std::numeric_limits<double>;
4138 
4139 void ExportQueryStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4140  auto session_copy = session;
4141  auto session_ptr = std::shared_ptr<Catalog_Namespace::SessionInfo>(
4142  &session_copy, boost::null_deleter());
4143  auto query_state = query_state::QueryState::create(session_ptr, *select_stmt);
4144  auto stdlog = STDLOG(query_state);
4145  if (SysCatalog::instance().isAggregator()) {
4146  // allow copy to statement for stand alone leafs
4147  throw std::runtime_error("Distributed export not supported yet");
4148  }
4149  auto& catalog = session.getCatalog();
4150  Importer_NS::CopyParams copy_params;
4151  if (!options.empty()) {
4152  for (auto& p : options) {
4153  if (boost::iequals(*p->get_name(), "delimiter")) {
4154  const StringLiteral* str_literal =
4155  dynamic_cast<const StringLiteral*>(p->get_value());
4156  if (str_literal == nullptr) {
4157  throw std::runtime_error("Delimiter option must be a string.");
4158  } else if (str_literal->get_stringval()->length() != 1) {
4159  throw std::runtime_error("Delimiter must be a single character string.");
4160  }
4161  copy_params.delimiter = (*str_literal->get_stringval())[0];
4162  } else if (boost::iequals(*p->get_name(), "nulls")) {
4163  const StringLiteral* str_literal =
4164  dynamic_cast<const StringLiteral*>(p->get_value());
4165  if (str_literal == nullptr) {
4166  throw std::runtime_error("Nulls option must be a string.");
4167  }
4168  copy_params.null_str = *str_literal->get_stringval();
4169  } else if (boost::iequals(*p->get_name(), "header")) {
4170  const StringLiteral* str_literal =
4171  dynamic_cast<const StringLiteral*>(p->get_value());
4172  if (str_literal == nullptr) {
4173  throw std::runtime_error("Header option must be a boolean.");
4174  }
4175  copy_params.has_header = bool_from_string_literal(str_literal)
4178  } else if (boost::iequals(*p->get_name(), "quote")) {
4179  const StringLiteral* str_literal =
4180  dynamic_cast<const StringLiteral*>(p->get_value());
4181  if (str_literal == nullptr) {
4182  throw std::runtime_error("Quote option must be a string.");
4183  } else if (str_literal->get_stringval()->length() != 1) {
4184  throw std::runtime_error("Quote must be a single character string.");
4185  }
4186  copy_params.quote = (*str_literal->get_stringval())[0];
4187  } else if (boost::iequals(*p->get_name(), "escape")) {
4188  const StringLiteral* str_literal =
4189  dynamic_cast<const StringLiteral*>(p->get_value());
4190  if (str_literal == nullptr) {
4191  throw std::runtime_error("Escape option must be a string.");
4192  } else if (str_literal->get_stringval()->length() != 1) {
4193  throw std::runtime_error("Escape must be a single character string.");
4194  }
4195  copy_params.escape = (*str_literal->get_stringval())[0];
4196  } else if (boost::iequals(*p->get_name(), "line_delimiter")) {
4197  const StringLiteral* str_literal =
4198  dynamic_cast<const StringLiteral*>(p->get_value());
4199  if (str_literal == nullptr) {
4200  throw std::runtime_error("Line_delimiter option must be a string.");
4201  } else if (str_literal->get_stringval()->length() != 1) {
4202  throw std::runtime_error("Line_delimiter must be a single character string.");
4203  }
4204  copy_params.line_delim = (*str_literal->get_stringval())[0];
4205  } else if (boost::iequals(*p->get_name(), "quoted")) {
4206  const StringLiteral* str_literal =
4207  dynamic_cast<const StringLiteral*>(p->get_value());
4208  if (str_literal == nullptr) {
4209  throw std::runtime_error("Quoted option must be a boolean.");
4210  }
4211  copy_params.quoted = bool_from_string_literal(str_literal);
4212  } else {
4213  throw std::runtime_error("Invalid option for COPY: " + *p->get_name());
4214  }
4215  }
4216  }
4217  std::vector<TargetMetaInfo> targets;
4218  const auto results =
4219  getResultSet(query_state->createQueryStateProxy(), *select_stmt, targets);
4220  TargetMetaInfo* td = targets.data();
4221 
4222  std::ofstream outfile;
4223  if (file_path->empty() || !boost::filesystem::path(*file_path).is_absolute()) {
4224  std::string file_name;
4225  if (file_path->empty()) {
4226  file_name = session.get_session_id() + ".txt";
4227  } else {
4228  file_name = boost::filesystem::path(*file_path).filename().string();
4229  }
4230  *file_path = catalog.getBasePath() + "/mapd_export/" + session.get_session_id() + "/";
4231  if (!boost::filesystem::exists(*file_path)) {
4232  if (!boost::filesystem::create_directories(*file_path)) {
4233  throw std::runtime_error("Directory " + *file_path + " cannot be created.");
4234  }
4235  }
4236  *file_path += file_name;
4237  }
4238  outfile.open(*file_path);
4239  if (!outfile) {
4240  throw std::runtime_error("Cannot open file: " + *file_path);
4241  }
4243  bool not_first = false;
4244  size_t i = 0;
4245  for (const auto& target : targets) {
4246  std::string col_name = target.get_resname();
4247  if (col_name.empty()) {
4248  col_name = "result_" + std::to_string(i + 1);
4249  }
4250  if (not_first) {
4251  outfile << copy_params.delimiter;
4252  } else {
4253  not_first = true;
4254  }
4255  outfile << col_name;
4256  ++i;
4257  }
4258  outfile << copy_params.line_delim;
4259  }
4260  while (true) {
4261  const auto crt_row = results->getNextRow(true, true);
4262  if (crt_row.empty()) {
4263  break;
4264  }
4265  bool not_first = false;
4266  for (size_t i = 0; i < results->colCount(); ++i) {
4267  bool is_null;
4268  const auto tv = crt_row[i];
4269  const auto scalar_tv = boost::get<ScalarTargetValue>(&tv);
4270  if (not_first) {
4271  outfile << copy_params.delimiter;
4272  } else {
4273  not_first = true;
4274  }
4275  if (copy_params.quoted) {
4276  outfile << copy_params.quote;
4277  }
4278  const auto& ti = td[i].get_type_info();
4279  if (!scalar_tv) {
4280  outfile << datum_to_string(crt_row[i], ti, " | ");
4281  if (copy_params.quoted) {
4282  outfile << copy_params.quote;
4283  }
4284  continue;
4285  }
4286  if (boost::get<int64_t>(scalar_tv)) {
4287  auto int_val = *(boost::get<int64_t>(scalar_tv));
4288  switch (ti.get_type()) {
4289  case kBOOLEAN:
4290  is_null = (int_val == NULL_BOOLEAN);
4291  break;
4292  case kTINYINT:
4293  is_null = (int_val == NULL_TINYINT);
4294  break;
4295  case kSMALLINT:
4296  is_null = (int_val == NULL_SMALLINT);
4297  break;
4298  case kINT:
4299  is_null = (int_val == NULL_INT);
4300  break;
4301  case kBIGINT:
4302  is_null = (int_val == NULL_BIGINT);
4303  break;
4304  case kTIME:
4305  case kTIMESTAMP:
4306  case kDATE:
4307  is_null = (int_val == NULL_BIGINT);
4308  break;
4309  default:
4310  is_null = false;
4311  }
4312  if (is_null) {
4313  outfile << copy_params.null_str;
4314  } else if (ti.get_type() == kTIME) {
4315  const auto t = static_cast<time_t>(int_val);
4316  std::tm tm_struct;
4317  gmtime_r(&t, &tm_struct);
4318  char buf[9];
4319  strftime(buf, 9, "%T", &tm_struct);
4320  outfile << buf;
4321  } else {
4322  outfile << int_val;
4323  }
4324  } else if (boost::get<double>(scalar_tv)) {
4325  auto real_val = *(boost::get<double>(scalar_tv));
4326  if (ti.get_type() == kFLOAT) {
4327  is_null = (real_val == NULL_FLOAT);
4328  } else {
4329  is_null = (real_val == NULL_DOUBLE);
4330  }
4331  if (is_null) {
4332  outfile << copy_params.null_str;
4333  } else if (ti.get_type() == kNUMERIC) {
4334  outfile << std::setprecision(ti.get_precision()) << real_val;
4335  } else {
4336  outfile << std::setprecision(std::numeric_limits<double>::digits10 + 1)
4337  << real_val;
4338  }
4339  } else if (boost::get<float>(scalar_tv)) {
4340  CHECK_EQ(kFLOAT, ti.get_type());
4341  auto real_val = *(boost::get<float>(scalar_tv));
4342  if (real_val == NULL_FLOAT) {
4343  outfile << copy_params.null_str;
4344  } else {
4345  outfile << std::setprecision(std::numeric_limits<float>::digits10 + 1)
4346  << real_val;
4347  }
4348  } else {
4349  auto s = boost::get<NullableString>(scalar_tv);
4350  is_null = !s || boost::get<void*>(s);
4351  if (is_null) {
4352  outfile << copy_params.null_str;
4353  } else {
4354  auto s_notnull = boost::get<std::string>(s);
4355  CHECK(s_notnull);
4356  if (!copy_params.quoted) {
4357  outfile << *s_notnull;
4358  } else {
4359  size_t q = s_notnull->find(copy_params.quote);
4360  if (q == std::string::npos) {
4361  outfile << *s_notnull;
4362  } else {
4363  std::string str(*s_notnull);
4364  while (q != std::string::npos) {
4365  str.insert(q, 1, copy_params.escape);
4366  q = str.find(copy_params.quote, q + 2);
4367  }
4368  outfile << str;
4369  }
4370  }
4371  }
4372  }
4373  if (copy_params.quoted) {
4374  outfile << copy_params.quote;
4375  }
4376  }
4377  outfile << copy_params.line_delim;
4378  }
4379  outfile.close();
4380 }
4381 
4382 void CreateViewStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4383  auto session_copy = session;
4384  auto session_ptr = std::shared_ptr<Catalog_Namespace::SessionInfo>(
4385  &session_copy, boost::null_deleter());
4386  auto query_state = query_state::QueryState::create(session_ptr, select_query_);
4387  auto stdlog = STDLOG(query_state);
4388  auto& catalog = session.getCatalog();
4389 
4390  if (catalog.getMetadataForTable(view_name_) != nullptr) {
4391  if (if_not_exists_) {
4392  return;
4393  }
4394  throw std::runtime_error("Table or View " + view_name_ + " already exists.");
4395  }
4396 
4399  throw std::runtime_error("View " + view_name_ +
4400  " will not be created. User has no create view privileges.");
4401  }
4402 
4403  const auto query_after_shim = pg_shim(select_query_);
4404 
4405  // this now also ensures that access permissions are checked
4406  catalog.getCalciteMgr()->process(
4407  query_state->createQueryStateProxy(), query_after_shim, {}, true, false, false);
4408  TableDescriptor td;
4409  td.tableName = view_name_;
4410  td.userId = session.get_currentUser().userId;
4411  td.nColumns = 0;
4412  td.isView = true;
4413  td.viewSQL = query_after_shim;
4414  td.fragmenter = nullptr;
4416  td.maxFragRows =
4417  DEFAULT_FRAGMENT_ROWS; // @todo this stuff should not be InsertOrderFragmenter
4418  td.maxChunkSize =
4419  DEFAULT_MAX_CHUNK_SIZE; // @todo this stuff should not be InsertOrderFragmenter
4422  catalog.createTable(td, {}, {}, true);
4423 
4424  // TODO (max): It's transactionally unsafe, should be fixed: we may create object w/o
4425  // privileges
4426  SysCatalog::instance().createDBObject(
4427  session.get_currentUser(), view_name_, ViewDBObjectType, catalog);
4428 }
4429 
4430 void DropViewStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4431  auto& catalog = session.getCatalog();
4432  const TableDescriptor* td = catalog.getMetadataForTable(*view_name);
4433  if (td == nullptr) {
4434  if (if_exists) {
4435  return;
4436  }
4437  throw std::runtime_error("View " + *view_name + " does not exist.");
4438  }
4439 
4440  if (!session.checkDBAccessPrivileges(
4442  throw std::runtime_error("View " + *view_name +
4443  " will not be dropped. User has no drop view privileges.");
4444  }
4445 
4446  if (!td->isView) {
4447  throw std::runtime_error(*view_name + " is a table. Use DROP TABLE.");
4448  }
4449  catalog.dropTable(td);
4450 }
4451 
4452 static void checkStringLiteral(const std::string& option_name,
4453  const std::unique_ptr<NameValueAssign>& p) {
4454  CHECK(p);
4455  if (!dynamic_cast<const StringLiteral*>(p->get_value())) {
4456  throw std::runtime_error(option_name + " option must be a string literal.");
4457  }
4458 }
4459 
4460 void CreateDBStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4461  if (!session.get_currentUser().isSuper) {
4462  throw std::runtime_error(
4463  "CREATE DATABASE command can only be executed by super user.");
4464  }
4466  if (SysCatalog::instance().getMetadataForDB(*db_name, db_meta) && if_not_exists_) {
4467  return;
4468  }
4469  int ownerId = session.get_currentUser().userId;
4470  if (!name_value_list.empty()) {
4471  for (auto& p : name_value_list) {
4472  if (boost::iequals(*p->get_name(), "owner")) {
4473  checkStringLiteral("Owner name", p);
4474  const std::string* str =
4475  static_cast<const StringLiteral*>(p->get_value())->get_stringval();
4477  if (!SysCatalog::instance().getMetadataForUser(*str, user)) {
4478  throw std::runtime_error("User " + *str + " does not exist.");
4479  }
4480  ownerId = user.userId;
4481  } else {
4482  throw std::runtime_error("Invalid CREATE DATABASE option " + *p->get_name() +
4483  ". Only OWNER supported.");
4484  }
4485  }
4486  }
4487  SysCatalog::instance().createDatabase(*db_name, ownerId);
4488 }
4489 
4490 void DropDBStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4491  if (!session.get_currentUser().isSuper) {
4492  throw std::runtime_error("DROP DATABASE command can only be executed by super user.");
4493  }
4495  if (!SysCatalog::instance().getMetadataForDB(*db_name, db)) {
4496  if (if_exists_) {
4497  return;
4498  }
4499  throw std::runtime_error("Database " + *db_name + " does not exist.");
4500  }
4501 
4502  if (!session.get_currentUser().isSuper &&
4503  session.get_currentUser().userId != db.dbOwner) {
4504  throw std::runtime_error("Only the super user or the owner can drop database.");
4505  }
4506 
4507  SysCatalog::instance().dropDatabase(db);
4508 }
4509 
4510 static bool readBooleanLiteral(const std::string& option_name,
4511  const std::unique_ptr<NameValueAssign>& p) {
4512  CHECK(p);
4513  const std::string* str =
4514  static_cast<const StringLiteral*>(p->get_value())->get_stringval();
4515  if (boost::iequals(*str, "true")) {
4516  return true;
4517  } else if (boost::iequals(*str, "false")) {
4518  return false;
4519  } else {
4520  throw std::runtime_error("Value to " + option_name + " must be TRUE or FALSE.");
4521  }
4522 }
4523 
4524 void CreateUserStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4525  std::string passwd;
4526  bool is_super = false;
4527  std::string default_db;
4528  bool can_login = true;
4529  for (auto& p : name_value_list) {
4530  if (boost::iequals(*p->get_name(), "password")) {
4531  checkStringLiteral("Password", p);
4532  passwd = *static_cast<const StringLiteral*>(p->get_value())->get_stringval();
4533  } else if (boost::iequals(*p->get_name(), "is_super")) {
4534  checkStringLiteral("IS_SUPER", p);
4535  is_super = readBooleanLiteral("IS_SUPER", p);
4536  } else if (boost::iequals(*p->get_name(), "default_db")) {
4537  checkStringLiteral("DEFAULT_DB", p);
4538  default_db = *static_cast<const StringLiteral*>(p->get_value())->get_stringval();
4539  } else if (boost::iequals(*p->get_name(), "can_login")) {
4540  checkStringLiteral("CAN_LOGIN", p);
4541  can_login = readBooleanLiteral("can_login", p);
4542  } else {
4543  throw std::runtime_error("Invalid CREATE USER option " + *p->get_name() +
4544  ". Should be PASSWORD, IS_SUPER, CAN_LOGIN"
4545  " or DEFAULT_DB.");
4546  }
4547  }
4548  if (!session.get_currentUser().isSuper) {
4549  throw std::runtime_error("Only super user can create new users.");
4550  }
4551  SysCatalog::instance().createUser(*user_name, passwd, is_super, default_db, can_login);
4552 }
4553 
4554 void AlterUserStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4555  // Parse the statement
4556  const std::string* passwd = nullptr;
4557  bool is_super = false;
4558  bool* is_superp = nullptr;
4559  const std::string* default_db = nullptr;
4560  bool can_login = true;
4561  bool* can_loginp = nullptr;
4562  for (auto& p : name_value_list) {
4563  if (boost::iequals(*p->get_name(), "password")) {
4564  checkStringLiteral("Password", p);
4565  passwd = static_cast<const StringLiteral*>(p->get_value())->get_stringval();
4566  } else if (boost::iequals(*p->get_name(), "is_super")) {
4567  checkStringLiteral("IS_SUPER", p);
4568  is_super = readBooleanLiteral("IS_SUPER", p);
4569  is_superp = &is_super;
4570  } else if (boost::iequals(*p->get_name(), "default_db")) {
4571  if (dynamic_cast<const StringLiteral*>(p->get_value())) {
4572  default_db = static_cast<const StringLiteral*>(p->get_value())->get_stringval();
4573  } else if (dynamic_cast<const NullLiteral*>(p->get_value())) {
4574  static std::string blank;
4575  default_db = &blank;
4576  } else {
4577  throw std::runtime_error(
4578  "DEFAULT_DB option must be either a string literal or a NULL literal.");
4579  }
4580  } else if (boost::iequals(*p->get_name(), "can_login")) {
4581  checkStringLiteral("CAN_LOGIN", p);
4582  can_login = readBooleanLiteral("CAN_LOGIN", p);
4583  can_loginp = &can_login;
4584  } else {
4585  throw std::runtime_error("Invalid ALTER USER option " + *p->get_name() +
4586  ". Should be PASSWORD, DEFAULT_DB, CAN_LOGIN"
4587  " or IS_SUPER.");
4588  }
4589  }
4590 
4591  // Check if the user is authorized to execute ALTER USER statement
4593  if (!SysCatalog::instance().getMetadataForUser(*user_name, user)) {
4594  throw std::runtime_error("User " + *user_name + " does not exist.");
4595  }
4596  if (!session.get_currentUser().isSuper) {
4597  if (session.get_currentUser().userId != user.userId) {
4598  throw std::runtime_error("Only super user can change another user's attributes.");
4599  } else if (is_superp || can_loginp) {
4600  throw std::runtime_error(
4601  "A user can only update their own password or default database.");
4602  }
4603  }
4604 
4605  if (passwd || is_superp || default_db || can_loginp) {
4606  SysCatalog::instance().alterUser(
4607  user.userId, passwd, is_superp, default_db, can_loginp);
4608  }
4609 }
4610 
4611 void DropUserStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4612  if (!session.get_currentUser().isSuper) {
4613  throw std::runtime_error("Only super user can drop users.");
4614  }
4615  SysCatalog::instance().dropUser(*user_name);
4616 }
4617 
4618 void DumpTableStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4619  // check access privileges
4620  if (!session.checkDBAccessPrivileges(
4622  throw std::runtime_error("Table " + *table +
4623  " will not be dumped. User has no select privileges.");
4624  }
4627  throw std::runtime_error("Table " + *table +
4628  " will not be dumped. User has no create privileges.");
4629  }
4630  auto& catalog = session.getCatalog();
4631  const TableDescriptor* td = catalog.getMetadataForTable(*table);
4632  catalog.dumpTable(td, *path, compression);
4633 }
4634 
4635 void RestoreTableStmt::execute(const Catalog_Namespace::SessionInfo& session) {
4636  auto& catalog = session.getCatalog();
4637  const TableDescriptor* td = catalog.getMetadataForTable(*table, false);
4638  if (td) {
4639  // TODO: v1.0 simply throws to avoid accidentally overwrite target table.
4640  // Will add a REPLACE TABLE to explictly replace target table.
4641  // catalog.restoreTable(session, td, *path, compression);
4642  throw std::runtime_error("Table " + *table + " exists.");
4643  } else {
4644  // check access privileges
4647  throw std::runtime_error("Table " + *table +
4648  " will not be restored. User has no create privileges.");
4649  }
4650  catalog.restoreTable(session, *table, *path, compression);
4651  }
4652 }
4653 
4654 } // namespace Parser
4655 
4656 // this is a non-clustered version of MapDHandler::prepare_columnar_loader,
4657 // exists for non-thrift builds, specifically for test cases and bin/initdb.
4659  const std::string& session,
4660  const std::string& table_name,
4661  size_t num_cols,
4662  std::unique_ptr<Importer_NS::Loader>* loader,
4663  std::vector<std::unique_ptr<Importer_NS::TypedImportBuffer>>* import_buffers) {
4664  auto col_descs = (*loader)->get_column_descs();
4665  for (auto cd : col_descs) {
4666  import_buffers->push_back(std::unique_ptr<Importer_NS::TypedImportBuffer>(
4667  new Importer_NS::TypedImportBuffer(cd, (*loader)->getStringDict(cd))));
4668  }
4669 }
int8_t tinyintval
Definition: sqltypes.h:126
const ColumnConstraintDef * get_column_constraint() const
Definition: ParserNode.h:818
decltype(auto) get_max_rows_def(TableDescriptor &td, const NameValueAssign *p, const std::list< ColumnDescriptor > &columns)
static std::set< std::string > reserved_keywords
static const AccessPrivileges VIEW_SQL_EDITOR
Definition: DBObject.h:145
decltype(auto) get_max_chunk_size_def(TableDescriptor &td, const NameValueAssign *p, const std::list< ColumnDescriptor > &columns)
bool is_boolean() const
Definition: sqltypes.h:484
static const std::map< const std::string, const TableDefFuncPtr > tableDefFuncMap
SQLAgg
Definition: sqldefs.h:71
#define CHECK_EQ(x, y)
Definition: Logger.h:201
size_t shard_column_index(const std::string &name, const std::list< ColumnDescriptor > &columns)
const std::string & get_column() const
Definition: ParserNode.h:939
std::string partitions
std::vector< int > ChunkKey
Definition: types.h:35
HOST DEVICE int get_output_srid() const
Definition: sqltypes.h:332
MapDHandler * get_mapdHandler() const
Definition: SessionInfo.h:89
static const AccessPrivileges VIEW_DASHBOARD
Definition: DBObject.h:164
#define NULL_DOUBLE
Definition: sqltypes.h:179
static const int32_t DROP_VIEW
Definition: DBObject.h:114
SQLType * get_column_type() const
Definition: ParserNode.h:816
static const int32_t SELECT_FROM_VIEW
Definition: DBObject.h:115
const int8_t const int64_t * num_rows
Definition: Analyzer.h:1487
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:81
std::string null_str
Definition: CopyParams.h:47
Definition: sqltypes.h:52
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:334
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:335
SQLTypes
Definition: sqltypes.h:41
static void checkStringLiteral(const std::string &option_name, const std::unique_ptr< NameValueAssign > &p)
static const int32_t UPDATE_IN_VIEW
Definition: DBObject.h:117
std::string tableName
const std::vector< TargetMetaInfo > targets_meta
bool g_cluster
void add_rte(RangeTableEntry *rte)
Definition: Analyzer.cpp:1274
static const AccessPrivileges ALL_DATABASE
Definition: DBObject.h:144
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
decltype(auto) get_vacuum_def(TableDescriptor &td, const NameValueAssign *p, const std::list< ColumnDescriptor > &columns)
static const AccessPrivileges ALTER_TABLE
Definition: DBObject.h:158
std::string to_string() const
DBObjectType
Definition: DBObject.h:42
static const int32_t CREATE_VIEW
Definition: DBObject.h:113
static std::shared_ptr< Executor > getExecutor(const int db_id, const std::string &debug_dir="", const std::string &debug_file="", const MapDParameters mapd_parameters=MapDParameters())
Definition: Execute.cpp:127
virtual void prepare_columnar_loader(const std::string &session, const std::string &table_name, size_t num_cols, std::unique_ptr< Importer_NS::Loader > *loader, std::vector< std::unique_ptr< Importer_NS::TypedImportBuffer >> *import_buffers)
#define NULL_BIGINT
Definition: sqltypes.h:177
void expand_star_in_targetlist(const Catalog_Namespace::Catalog &catalog, std::vector< std::shared_ptr< TargetEntry >> &tlist, int rte_idx)
static const AccessPrivileges TRUNCATE_TABLE
Definition: DBObject.h:157
SQLQualifier
Definition: sqldefs.h:69
#define LOG(tag)
Definition: Logger.h:188
std::string datum_to_string(const TargetValue &tv, const SQLTypeInfo &ti, const std::string &delim)
std::string parse_to_ra(query_state::QueryStateProxy query_state_proxy, const std::string &query_str)
Definition: LockMgr.cpp:77
SQLOps
Definition: sqldefs.h:29
Definition: sqldefs.h:35
void set_size(int s)
Definition: sqltypes.h:424
static const AccessPrivileges INSERT_INTO_TABLE
Definition: DBObject.h:154
HOST DEVICE int get_scale() const
Definition: sqltypes.h:331
std::shared_ptr< Analyzer::Expr > deep_copy() const override
Definition: Analyzer.cpp:80
Definition: sqldefs.h:36
static const AccessPrivileges CREATE_DASHBOARD
Definition: DBObject.h:163
static std::shared_ptr< QueryState > create(ARGS &&...args)
Definition: QueryState.h:140
bool get_is_null() const
Definition: Analyzer.h:328
std::vector< uint8_t > compress_coords(std::vector< double > &coords, const SQLTypeInfo &ti)
Definition: Importer.cpp:1422
void set_input_srid(int d)
Definition: sqltypes.h:420
std::shared_ptr< ResultSet > getResultSet(QueryStateProxy query_state_proxy, const std::string select_stmt, std::vector< TargetMetaInfo > &targets, bool validate_only=false)
HOST DEVICE int get_size() const
Definition: sqltypes.h:336
Definition: sqldefs.h:49
#define DEFAULT_MAX_ROWS
HOST DEVICE void set_type(SQLTypes t)
Definition: sqltypes.h:416
void set_order_by(std::list< OrderEntry > *o)
Definition: Analyzer.h:1554
std::string extractObjectNameFromHierName(const std::string &objectHierName, const std::string &objectType, const Catalog_Namespace::Catalog &cat)
void set_fixed_size()
Definition: sqltypes.h:425
static const int32_t ALTER_TABLE
Definition: DBObject.h:94
void set_result_col_list(const std::list< int > &col_list)
Definition: Analyzer.h:1548
int get_num_aggs() const
Definition: Analyzer.h:1529
void set_dimension(int d)
Definition: sqltypes.h:418
void setPrivileges(const AccessPrivileges &privs)
Definition: DBObject.h:203
void set_offset(int64_t o)
Definition: Analyzer.h:1566
std::vector< std::string > splitObjectHierName(const std::string &hierName)
static const AccessPrivileges SELECT_FROM_TABLE
Definition: DBObject.h:153
void set_scale(int s)
Definition: sqltypes.h:421
const ColumnDescriptor * getShardColumnMetadataForTable(const TableDescriptor *td) const
Definition: Catalog.cpp:3026
void set_compression(EncodingType c)
Definition: sqltypes.h:426
bool bool_from_string_literal(const Parser::StringLiteral *str_literal)
Definition: ParserNode.cpp:892
int32_t intval
Definition: sqltypes.h:128
int32_t get_table_id() const
std::string to_string(char const *&&v)
void set_notnull(bool n)
Definition: sqltypes.h:423
void set_output_srid(int s)
Definition: sqltypes.h:422
int get_rte_idx(const std::string &range_var_name) const
Definition: Analyzer.cpp:1263
bool is_number() const
Definition: sqltypes.h:482
static const AccessPrivileges ALL_VIEW
Definition: DBObject.h:170
std::string get_type_name() const
Definition: sqltypes.h:429
EncodingType geo_coords_encoding
Definition: CopyParams.h:73
std::shared_ptr< StringDictionary > stringDict
int tableId
identifies the database into which the data is being inserted
Definition: Fragmenter.h:61
DBObjectType DBObjectTypeFromString(const std::string &type)
Definition: DBObject.cpp:88
Definition: sqldefs.h:73
size_t numRows
a vector of column ids for the row(s) being inserted
Definition: Fragmenter.h:63
float floatval
Definition: sqltypes.h:130
std::numeric_limits< double > dbl
bool get_is_array() const
Definition: ParserNode.h:81
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:417
bool g_use_date_in_days_default_encoding
Definition: ParserNode.cpp:67
static SQLTypeInfo common_string_type(const SQLTypeInfo &type1, const SQLTypeInfo &type2)
Definition: Analyzer.cpp:387
void set_precision(int d)
Definition: sqltypes.h:419
const std::string * get_stringval() const
Definition: ParserNode.h:149
CHECK(cgen_state)
bool is_array() const
Definition: sqltypes.h:485
Classes representing a parse tree.
std::string s3_access_key
Definition: CopyParams.h:62
bool is_time() const
Definition: sqltypes.h:483
size_t g_leaf_count
Definition: ParserNode.cpp:66
int get_logical_size() const
Definition: sqltypes.h:337
RangeTableEntry * get_rte(int rte_idx) const
Definition: Analyzer.h:1560
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:176
QueryState & getQueryState()
Definition: QueryState.h:172
static const int32_t DROP_DATABASE
Definition: DBObject.h:80
#define NULL_TINYINT
Definition: sqltypes.h:174
static const AccessPrivileges DROP_TABLE
Definition: DBObject.h:152
const std::string * get_encoding_name() const
Definition: ParserNode.h:799
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:326
decltype(auto) get_shard_count_def(TableDescriptor &td, const NameValueAssign *p, const std::list< ColumnDescriptor > &columns)
boost::function< void(TableDescriptor &, const NameValueAssign *, const std::list< ColumnDescriptor > &columns)> TableDefFuncPtr
Definition: ParserNode.cpp:76
int64_t bigintval
Definition: sqltypes.h:129
WhichRow get_which_row() const
Definition: Analyzer.h:273
#define NULL_FLOAT
Definition: sqltypes.h:178
std::list< ColumnDescriptor > getColumnDescriptors(AggregatedResult &result, bool for_create)
int64_t get_intval() const
Definition: ParserNode.h:168
static const int32_t DELETE_FROM_TABLE
Definition: DBObject.h:92
bool get_is_distinct() const
Definition: Analyzer.h:1528
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:333
static SQLTypeInfo common_numeric_type(const SQLTypeInfo &type1, const SQLTypeInfo &type2)
Definition: Analyzer.cpp:427
Definition: sqldefs.h:37
Definition: sqldefs.h:75
static const int32_t TRUNCATE_TABLE
Definition: DBObject.h:93
const std::string * get_column_name() const
Definition: ParserNode.h:815
#define DEFAULT_PAGE_SIZE
int16_t smallintval
Definition: sqltypes.h:127
int get_varno() const
Definition: Analyzer.h:275
static const AccessPrivileges DELETE_FROM_TABLE
Definition: DBObject.h:156
ImportHeaderRow has_header
Definition: CopyParams.h:48
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1377
specifies the content in-memory of a row in the column metadata table
#define NULL_INT
Definition: sqltypes.h:176
bool expr_is_null(const Analyzer::Expr *expr)
Definition: ParserNode.cpp:884
static const int32_t EDIT_DASHBOARD
Definition: DBObject.h:105
static const int32_t DELETE_DASHBOARD
Definition: DBObject.h:103
static const AccessPrivileges CREATE_TABLE
Definition: DBObject.h:151
static const int32_t INSERT_INTO_TABLE
Definition: DBObject.h:90
SQLTypeInfoCore< ArrayContextTypeSizer, ExecutorTypePackaging, DateTimeFacilities > SQLTypeInfo
Definition: sqltypes.h:852
std::string keyMetainfo
void set_num_aggs(int a)
Definition: Analyzer.h:1558
void set_group_by(std::list< std::shared_ptr< Analyzer::Expr >> &g)
Definition: Analyzer.h:1552
const SQLTypeInfo & get_type_info() const
Definition: Analyzer.h:78
bool is_geometry() const
Definition: sqltypes.h:489
void set_next_query(Query *q)
Definition: Analyzer.h:1555
std::string * stringval
Definition: sqltypes.h:134
int get_result_table_id() const
Definition: Analyzer.h:1546
static void set_geo_physical_import_buffer(const Catalog_Namespace::Catalog &catalog, const ColumnDescriptor *cd, std::vector< std::unique_ptr< TypedImportBuffer >> &import_buffers, size_t &col_idx, std::vector< double > &coords, std::vector< double > &bounds, std::vector< int > &ring_sizes, std::vector< int > &poly_rings, int render_group, const int64_t replicate_count=0)
Definition: Importer.cpp:1457
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1608
std::string s3_endpoint
Definition: CopyParams.h:65
void set_having_predicate(std::shared_ptr< Analyzer::Expr > p)
Definition: Analyzer.h:1553
std::string get_session_id() const
Definition: SessionInfo.h:98
void set_result_table_id(int id)
Definition: Analyzer.h:1549
void loadKey()
Definition: DBObject.cpp:165
const AccessPrivileges & getPrivileges() const
Definition: DBObject.h:202
void validate_shard_column_type(const ColumnDescriptor &cd)
Definition: sqltypes.h:55
Definition: sqltypes.h:56
void check_alter_table_privilege(const Catalog_Namespace::SessionInfo &session, const TableDescriptor *td)
Definition: sqldefs.h:69
#define TRANSIENT_DICT_ID
Definition: sqltypes.h:189
const std::string OMNISCI_ROOT_USER
Definition: SysCatalog.h:60
int64_t const int32_t sz assert(dest)
void set_is_distinct(bool d)
Definition: Analyzer.h:1550
static const AccessPrivileges SELECT_FROM_VIEW
Definition: DBObject.h:173
bool is_null(const T &v, const SQLTypeInfo &t)
const CompressDef * get_compression() const
Definition: ParserNode.h:817
std::vector< std::shared_ptr< TargetEntry > > & get_targetlist_nonconst()
Definition: Analyzer.h:1533
Catalog & getCatalog() const
Definition: SessionInfo.h:90
bool is_integer() const
Definition: sqltypes.h:479
Datum StringToDatum(const std::string &s, SQLTypeInfo &ti)
Definition: Datum.cpp:90
std::string s3_region
Definition: CopyParams.h:64
Datum get_constval() const
Definition: Analyzer.h:329
bool g_enable_experimental_string_functions
Fragmenter_Namespace::FragmenterType fragType
Definition: sqldefs.h:76
Data_Namespace::MemoryLevel persistenceLevel
SQLTypeInfoCore get_array_type() const
Definition: sqltypes.h:663
void set_is_unionall(bool u)
Definition: Analyzer.h:1556
static const int32_t CREATE_DATABASE
Definition: DBObject.h:79
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:328
std::vector< std::string > split(const std::string &str, const std::string &delim)
split apart a string into a vector of substrings
void insertData(const Catalog_Namespace::SessionInfo &session_info, InsertData &insert_data)
decltype(auto) get_sort_column_def(TableDescriptor &td, const NameValueAssign *p, const std::list< ColumnDescriptor > &columns)
static const AccessPrivileges ALL_DASHBOARD
Definition: DBObject.h:162
static bool getGeoColumns(const std::string &wkt, SQLTypeInfo &ti, std::vector< double > &coords, std::vector< double > &bounds, std::vector< int > &ring_sizes, std::vector< int > &poly_rings, const bool promote_poly_to_mpoly=false)
Definition: geo_types.cpp:459
static const AccessPrivileges ACCESS
Definition: DBObject.h:146
std::string geo_layer_name
Definition: CopyParams.h:78
static const int32_t VIEW_DASHBOARD
Definition: DBObject.h:104
static std::pair< AccessPrivileges, DBObjectType > parseStringPrivs(const std::string &privs, const DBObjectType &objectType, const std::string &object_name)
Definition: sqltypes.h:44
virtual std::shared_ptr< Analyzer::Expr > add_cast(const SQLTypeInfo &new_type_info)
Definition: Analyzer.cpp:671
static const AccessPrivileges ALL_TABLE
Definition: DBObject.h:150
AggregatedResult query(QueryStateProxy, std::string &sql_query_string, bool validate_only)
decltype(auto) get_frag_size_def(TableDescriptor &td, const NameValueAssign *p, const std::list< ColumnDescriptor > &columns)
#define DEFAULT_FRAGMENT_ROWS
void set_comp_param(int p)
Definition: sqltypes.h:427
void set_where_predicate(std::shared_ptr< Analyzer::Expr > p)
Definition: Analyzer.h:1551
int get_param1() const
Definition: ParserNode.h:79
void set_stmt_type(SQLStmtType t)
Definition: Analyzer.h:1557
const ColumnDescriptor * get_column_desc(const Catalog_Namespace::Catalog &catalog, const std::string &name)
int get_array_size() const
Definition: ParserNode.h:83
Definition: sqldefs.h:53
static const int32_t DROP_TABLE
Definition: DBObject.h:88
static const int32_t INSERT_INTO_VIEW
Definition: DBObject.h:116
#define NULL_SMALLINT
Definition: sqltypes.h:175
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:327
virtual std::shared_ptr< Analyzer::Expr > analyze(const Catalog_Namespace::Catalog &catalog, Analyzer::Query &query, TlistRefType allow_tlist_ref=TLIST_NONE) const =0
static const AccessPrivileges DROP_VIEW
Definition: DBObject.h:172
bool is_string() const
Definition: sqltypes.h:477
std::shared_ptr< ResultSet > rs
static bool readBooleanLiteral(const std::string &option_name, const std::unique_ptr< NameValueAssign > &p)
static const int32_t DELETE_FROM_VIEW
Definition: DBObject.h:118
static const AccessPrivileges CREATE_VIEW
Definition: DBObject.h:171
static const int32_t CREATE_TABLE
Definition: DBObject.h:87
void get_table_definitions(TableDescriptor &td, const std::unique_ptr< NameValueAssign > &p, const std::list< ColumnDescriptor > &columns)
decltype(auto) get_partions_def(TableDescriptor &td, const NameValueAssign *p, const std::list< ColumnDescriptor > &columns)
SQLTypes get_type() const
Definition: ParserNode.h:78
static const int32_t CREATE_DASHBOARD
Definition: DBObject.h:102
decltype(auto) get_page_size_def(TableDescriptor &td, const NameValueAssign *p, const std::list< ColumnDescriptor > &columns)
int64_t inline_fixed_encoding_null_val(const SQL_TYPE_INFO &ti)
decltype(auto) get_property_value(const NameValueAssign *p, ASSIGNMENT op, VALIDATE validate=VALIDATE())
const std::list< std::shared_ptr< Analyzer::Expr > > & get_group_by() const
Definition: Analyzer.h:1538
The data to be inserted using the fragment manager.
Definition: Fragmenter.h:59
static DBObject createObject(const std::string &objectName, DBObjectType objectType)
#define DEFAULT_MAX_CHUNK_SIZE
std::string viewSQL
static const AccessPrivileges DELETE_DASHBOARD
Definition: DBObject.h:166
int get_precision() const
Definition: sqltypes.h:329
Definition: sqltypes.h:48
static const int32_t SELECT_FROM_TABLE
Definition: DBObject.h:89
SQLTypeInfoCore get_elem_type() const
Definition: sqltypes.h:659
SQLTypeInfo columnType
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
FileType file_type
Definition: