27 #include <boost/algorithm/string.hpp>
28 #include <boost/core/null_deleter.hpp>
29 #include <boost/filesystem.hpp>
30 #include <boost/function.hpp>
32 #include <rapidjson/document.h>
33 #include <rapidjson/stringbuffer.h>
34 #include <rapidjson/writer.h>
43 #include <type_traits>
76 #include "gen-cpp/CalciteServer.h"
84 #ifdef ENABLE_IMPORT_PARQUET
85 bool g_enable_legacy_parquet_import{
false};
92 using namespace std::string_literals;
96 const std::list<ColumnDescriptor>& columns)>;
101 const std::list<ColumnDescriptor>& columns)>;
109 mapd_shared_lock<mapd_shared_mutex> session_read_lock(executor->getSessionLock());
110 return executor->checkIsQuerySessionInterrupted(query_session, session_read_lock);
117 std::vector<int> table_chunk_key_prefix;
120 table_chunk_key_prefix = td->
fragmenter->getFragmentsForQuery().chunkKeyPrefix;
123 table_chunk_key_prefix.push_back(td->
tableId);
126 return table_chunk_key_prefix;
129 std::shared_ptr<Analyzer::Expr> NullLiteral::analyze(
133 return makeExpr<Analyzer::Constant>(
kNULLT,
true);
136 std::shared_ptr<Analyzer::Expr> StringLiteral::analyze(
140 return analyzeValue(*stringval_,
false);
143 std::shared_ptr<Analyzer::Expr> StringLiteral::analyzeValue(
const std::string& stringval,
148 d.
stringval =
new std::string(stringval);
149 return makeExpr<Analyzer::Constant>(ti,
false, d);
152 return makeExpr<Analyzer::Constant>(
kVARCHAR,
true);
155 std::shared_ptr<Analyzer::Expr> IntLiteral::analyze(
159 return analyzeValue(intval_);
162 std::shared_ptr<Analyzer::Expr> IntLiteral::analyzeValue(
const int64_t intval) {
165 if (intval >= INT16_MIN && intval <= INT16_MAX) {
168 }
else if (intval >= INT32_MIN && intval <= INT32_MAX) {
170 d.
intval = (int32_t)intval;
175 return makeExpr<Analyzer::Constant>(t,
false, d);
178 std::shared_ptr<Analyzer::Expr> FixedPtLiteral::analyze(
184 return makeExpr<Analyzer::Constant>(ti,
false, d);
187 std::shared_ptr<Analyzer::Expr> FixedPtLiteral::analyzeValue(
const int64_t numericval,
189 const int precision) {
195 return makeExpr<Analyzer::Constant>(ti,
false, d);
198 std::shared_ptr<Analyzer::Expr> FloatLiteral::analyze(
204 return makeExpr<Analyzer::Constant>(
kFLOAT,
false, d);
207 std::shared_ptr<Analyzer::Expr> DoubleLiteral::analyze(
213 return makeExpr<Analyzer::Constant>(
kDOUBLE,
false, d);
216 std::shared_ptr<Analyzer::Expr> TimestampLiteral::analyze(
220 return get(timestampval_);
223 std::shared_ptr<Analyzer::Expr> TimestampLiteral::get(
const int64_t timestampval) {
226 return makeExpr<Analyzer::Constant>(
kTIMESTAMP,
false, d);
229 std::shared_ptr<Analyzer::Expr> UserLiteral::analyze(
234 return makeExpr<Analyzer::Constant>(
kTEXT,
false, d);
237 std::shared_ptr<Analyzer::Expr> UserLiteral::get(
const std::string& user) {
240 return makeExpr<Analyzer::Constant>(
kTEXT,
false, d);
243 std::shared_ptr<Analyzer::Expr> ArrayLiteral::analyze(
248 bool set_subtype =
true;
249 std::list<std::shared_ptr<Analyzer::Expr>> value_exprs;
250 for (
auto& p : value_list_) {
251 auto e = p->analyze(catalog, query, allow_tlist_ref);
254 if (c !=
nullptr && c->get_is_null()) {
255 value_exprs.push_back(c);
261 }
else if (set_subtype) {
265 value_exprs.push_back(e);
267 std::shared_ptr<Analyzer::Expr>
result =
268 makeExpr<Analyzer::Constant>(ti,
false, value_exprs);
273 std::string str =
"{";
274 bool notfirst =
false;
275 for (
auto& p : value_list_) {
281 str += p->to_string();
287 std::shared_ptr<Analyzer::Expr> OperExpr::analyze(
291 auto left_expr = left_->analyze(catalog, query, allow_tlist_ref);
292 const auto& left_type = left_expr->get_type_info();
293 if (right_ ==
nullptr) {
294 return makeExpr<Analyzer::UOper>(
295 left_type, left_expr->get_contains_agg(), optype_, left_expr->decompress());
298 if (left_type.get_type() !=
kARRAY) {
299 throw std::runtime_error(left_->to_string() +
" is not of array type.");
301 auto right_expr = right_->analyze(catalog, query, allow_tlist_ref);
302 const auto& right_type = right_expr->get_type_info();
303 if (!right_type.is_integer()) {
304 throw std::runtime_error(right_->to_string() +
" is not of integer type.");
306 return makeExpr<Analyzer::BinOper>(
307 left_type.get_elem_type(),
false,
kARRAY_AT,
kONE, left_expr, right_expr);
309 auto right_expr = right_->analyze(catalog, query, allow_tlist_ref);
310 return normalize(optype_, opqualifier_, left_expr, right_expr);
314 const std::shared_ptr<Analyzer::Expr>& rhs_expr) {
315 std::set<int> lhs_rte_idx;
316 lhs_expr->collect_rte_idx(lhs_rte_idx);
317 CHECK(!lhs_rte_idx.empty());
318 std::set<int> rhs_rte_idx;
319 rhs_expr->collect_rte_idx(rhs_rte_idx);
320 CHECK(!rhs_rte_idx.empty());
321 return lhs_rte_idx.size() == 1UL && lhs_rte_idx == rhs_rte_idx;
326 const Executor* executor) {
333 CHECK_NE(lhs_comp_param, rhs_comp_param);
335 return rhs_type_info;
338 return lhs_type_info;
342 const auto lhs_sdp = executor->getStringDictionaryProxy(lhs_comp_param,
true);
343 const auto rhs_sdp = executor->getStringDictionaryProxy(rhs_comp_param,
true);
344 return lhs_sdp->entryCount() >= rhs_sdp->entryCount() ? lhs_type_info : rhs_type_info;
349 const Executor* executor) {
355 if (lhs_comp_param == rhs_comp_param ||
357 return lhs_comp_param <= rhs_comp_param ? lhs_type_info : rhs_type_info;
364 if (ret_ti.is_none_encoded_string()) {
371 std::shared_ptr<Analyzer::Expr> OperExpr::normalize(
374 std::shared_ptr<Analyzer::Expr> left_expr,
375 std::shared_ptr<Analyzer::Expr> right_expr,
376 const Executor* executor) {
377 if (left_expr->get_type_info().is_date_in_days() ||
378 right_expr->get_type_info().is_date_in_days()) {
380 left_expr = left_expr->decompress();
381 right_expr = right_expr->decompress();
383 const auto& left_type = left_expr->get_type_info();
384 auto right_type = right_expr->get_type_info();
387 CHECK(!std::dynamic_pointer_cast<Analyzer::Subquery>(right_expr));
388 if (right_type.get_type() !=
kARRAY) {
389 throw std::runtime_error(
390 "Existential or universal qualifiers can only be used in front of a subquery "
392 "expression of array type.");
394 right_type = right_type.get_elem_type();
399 optype, left_type, right_type, &new_left_type, &new_right_type);
400 if (result_type.is_timeinterval()) {
401 return makeExpr<Analyzer::BinOper>(
402 result_type,
false, optype, qual, left_expr, right_expr);
404 if (left_type != new_left_type) {
405 left_expr = left_expr->add_cast(new_left_type);
407 if (right_type != new_right_type) {
409 right_expr = right_expr->add_cast(new_right_type);
411 right_expr = right_expr->add_cast(new_right_type.
get_array_type());
418 throw std::runtime_error(
419 "Comparison operators are not yet supported for geospatial types.");
425 if (optype ==
kEQ || optype ==
kNE) {
431 const bool should_translate_strings =
433 if (should_translate_strings && (optype ==
kEQ || optype ==
kNE)) {
439 auto& expr_to_cast = ti == new_left_type ? right_expr : left_expr;
442 expr_to_cast = expr_to_cast->add_cast(ti);
447 left_expr = left_expr->decompress();
448 right_expr = right_expr->decompress();
454 left_expr = left_expr->decompress();
455 right_expr = right_expr->decompress();
458 if (!(optype ==
kEQ || optype ==
kNE)) {
461 left_expr = left_expr->decompress();
462 right_expr = right_expr->decompress();
475 right_expr = right_expr->add_cast(ti);
482 left_expr = left_expr->add_cast(ti);
484 left_expr = left_expr->decompress();
485 right_expr = right_expr->decompress();
489 left_expr = left_expr->decompress();
490 right_expr = right_expr->decompress();
492 bool has_agg = (left_expr->get_contains_agg() || right_expr->get_contains_agg());
493 return makeExpr<Analyzer::BinOper>(
494 result_type, has_agg, optype, qual, left_expr, right_expr);
497 std::shared_ptr<Analyzer::Expr> SubqueryExpr::analyze(
501 throw std::runtime_error(
"Subqueries are not supported yet.");
505 std::shared_ptr<Analyzer::Expr> IsNullExpr::analyze(
509 auto arg_expr = arg_->analyze(catalog, query, allow_tlist_ref);
517 std::shared_ptr<Analyzer::Expr> InSubquery::analyze(
521 throw std::runtime_error(
"Subqueries are not supported yet.");
525 std::shared_ptr<Analyzer::Expr> InValues::analyze(
529 auto arg_expr = arg_->analyze(catalog, query, allow_tlist_ref);
532 std::list<std::shared_ptr<Analyzer::Expr>> value_exprs;
533 for (
auto& p : value_list_) {
534 auto e = p->analyze(catalog, query, allow_tlist_ref);
535 if (ti != e->get_type_info()) {
536 if (ti.
is_string() && e->get_type_info().is_string()) {
539 }
else if (ti.
is_number() && e->get_type_info().is_number()) {
542 throw std::runtime_error(
"IN expressions must contain compatible types.");
546 value_exprs.push_back(e->add_cast(arg_expr->get_type_info()));
548 value_exprs.push_back(e);
552 arg_expr = arg_expr->decompress();
553 arg_expr = arg_expr->add_cast(ti);
554 std::list<std::shared_ptr<Analyzer::Expr>> cast_vals;
555 for (
auto p : value_exprs) {
556 cast_vals.push_back(p->add_cast(ti));
558 value_exprs.swap(cast_vals);
560 std::shared_ptr<Analyzer::Expr>
result =
561 makeExpr<Analyzer::InValues>(arg_expr, value_exprs);
568 std::shared_ptr<Analyzer::Expr> BetweenExpr::analyze(
572 auto arg_expr = arg_->analyze(catalog, query, allow_tlist_ref);
573 auto lower_expr = lower_->analyze(catalog, query, allow_tlist_ref);
574 auto upper_expr = upper_->analyze(catalog, query, allow_tlist_ref);
577 arg_expr->get_type_info(),
578 lower_expr->get_type_info(),
582 makeExpr<Analyzer::BinOper>(
kBOOLEAN,
585 arg_expr->add_cast(new_left_type)->decompress(),
586 lower_expr->add_cast(new_right_type)->decompress());
588 arg_expr->get_type_info(),
589 lower_expr->get_type_info(),
592 auto upper_pred = makeExpr<Analyzer::BinOper>(
596 arg_expr->deep_copy()->add_cast(new_left_type)->decompress(),
597 upper_expr->add_cast(new_right_type)->decompress());
598 std::shared_ptr<Analyzer::Expr>
result =
606 std::shared_ptr<Analyzer::Expr> CharLengthExpr::analyze(
610 auto arg_expr = arg_->analyze(catalog, query, allow_tlist_ref);
611 if (!arg_expr->get_type_info().is_string()) {
612 throw std::runtime_error(
613 "expression in char_length clause must be of a string type.");
615 std::shared_ptr<Analyzer::Expr>
result =
616 makeExpr<Analyzer::CharLengthExpr>(arg_expr->decompress(), calc_encoded_length_);
620 std::shared_ptr<Analyzer::Expr> CardinalityExpr::analyze(
624 auto arg_expr = arg_->analyze(catalog, query, allow_tlist_ref);
625 if (!arg_expr->get_type_info().is_array()) {
626 throw std::runtime_error(
627 "expression in cardinality clause must be of an array type.");
629 std::shared_ptr<Analyzer::Expr>
result =
630 makeExpr<Analyzer::CardinalityExpr>(arg_expr->decompress());
634 void LikeExpr::check_like_expr(
const std::string& like_str,
char escape_char) {
635 if (like_str.back() == escape_char) {
636 throw std::runtime_error(
"LIKE pattern must not end with escape character.");
640 bool LikeExpr::test_is_simple_expr(
const std::string& like_str,
char escape_char) {
642 if (like_str.size() < 2 || like_str[0] !=
'%' || like_str[like_str.size() - 1] !=
'%') {
646 if (like_str[like_str.size() - 2] == escape_char &&
647 like_str[like_str.size() - 3] != escape_char) {
650 for (
size_t i = 1; i < like_str.size() - 1; i++) {
651 if (like_str[i] ==
'%' || like_str[i] ==
'_' || like_str[i] ==
'[' ||
652 like_str[i] ==
']') {
653 if (like_str[i - 1] != escape_char) {
661 void LikeExpr::erase_cntl_chars(std::string& like_str,
char escape_char) {
662 char prev_char =
'\0';
667 for (
char& cur_char : like_str) {
668 if (cur_char ==
'%' || cur_char == escape_char) {
669 if (prev_char != escape_char) {
670 prev_char = cur_char;
674 new_str.push_back(cur_char);
675 prev_char = cur_char;
680 std::shared_ptr<Analyzer::Expr> LikeExpr::analyze(
684 auto arg_expr = arg_->analyze(catalog, query, allow_tlist_ref);
685 auto like_expr = like_string_->analyze(catalog, query, allow_tlist_ref);
686 auto escape_expr = escape_string_ ==
nullptr
688 : escape_string_->analyze(catalog, query, allow_tlist_ref);
689 return LikeExpr::get(arg_expr, like_expr, escape_expr, is_ilike_, is_not_);
692 std::shared_ptr<Analyzer::Expr> LikeExpr::get(std::shared_ptr<Analyzer::Expr> arg_expr,
693 std::shared_ptr<Analyzer::Expr> like_expr,
694 std::shared_ptr<Analyzer::Expr> escape_expr,
697 if (!arg_expr->get_type_info().is_string()) {
698 throw std::runtime_error(
"expression before LIKE must be of a string type.");
700 if (!like_expr->get_type_info().is_string()) {
701 throw std::runtime_error(
"expression after LIKE must be of a string type.");
703 char escape_char =
'\\';
704 if (escape_expr !=
nullptr) {
705 if (!escape_expr->get_type_info().is_string()) {
706 throw std::runtime_error(
"expression after ESCAPE must be of a string type.");
708 if (!escape_expr->get_type_info().is_string()) {
709 throw std::runtime_error(
"expression after ESCAPE must be of a string type.");
712 if (c !=
nullptr && c->get_constval().stringval->length() > 1) {
713 throw std::runtime_error(
"String after ESCAPE must have a single character.");
715 escape_char = (*c->get_constval().stringval)[0];
718 bool is_simple =
false;
722 std::transform(pattern.begin(), pattern.end(), pattern.begin(), ::tolower);
724 check_like_expr(pattern, escape_char);
725 is_simple = test_is_simple_expr(pattern, escape_char);
727 erase_cntl_chars(pattern, escape_char);
730 std::shared_ptr<Analyzer::Expr>
result = makeExpr<Analyzer::LikeExpr>(
731 arg_expr->decompress(), like_expr, escape_expr, is_ilike, is_simple);
738 void RegexpExpr::check_pattern_expr(
const std::string& pattern_str,
char escape_char) {
739 if (pattern_str.back() == escape_char) {
740 throw std::runtime_error(
"REGEXP pattern must not end with escape character.");
744 bool RegexpExpr::translate_to_like_pattern(std::string& pattern_str,
char escape_char) {
745 char prev_char =
'\0';
746 char prev_prev_char =
'\0';
747 std::string like_str;
748 for (
char& cur_char : pattern_str) {
749 if (prev_char == escape_char || isalnum(cur_char) || cur_char ==
' ' ||
751 like_str.push_back((cur_char ==
'.') ?
'_' : cur_char);
752 prev_prev_char = prev_char;
753 prev_char = cur_char;
756 if (prev_char ==
'.' && prev_prev_char != escape_char) {
757 if (cur_char ==
'*' || cur_char ==
'+') {
758 if (cur_char ==
'*') {
763 like_str.push_back(
'%');
764 prev_prev_char = prev_char;
765 prev_char = cur_char;
771 pattern_str = like_str;
775 std::shared_ptr<Analyzer::Expr> RegexpExpr::analyze(
779 auto arg_expr = arg_->analyze(catalog, query, allow_tlist_ref);
780 auto pattern_expr = pattern_string_->analyze(catalog, query, allow_tlist_ref);
781 auto escape_expr = escape_string_ ==
nullptr
783 : escape_string_->analyze(catalog, query, allow_tlist_ref);
784 return RegexpExpr::get(arg_expr, pattern_expr, escape_expr, is_not_);
787 std::shared_ptr<Analyzer::Expr> RegexpExpr::get(
788 std::shared_ptr<Analyzer::Expr> arg_expr,
789 std::shared_ptr<Analyzer::Expr> pattern_expr,
790 std::shared_ptr<Analyzer::Expr> escape_expr,
792 if (!arg_expr->get_type_info().is_string()) {
793 throw std::runtime_error(
"expression before REGEXP must be of a string type.");
795 if (!pattern_expr->get_type_info().is_string()) {
796 throw std::runtime_error(
"expression after REGEXP must be of a string type.");
798 char escape_char =
'\\';
799 if (escape_expr !=
nullptr) {
800 if (!escape_expr->get_type_info().is_string()) {
801 throw std::runtime_error(
"expression after ESCAPE must be of a string type.");
803 if (!escape_expr->get_type_info().is_string()) {
804 throw std::runtime_error(
"expression after ESCAPE must be of a string type.");
807 if (c !=
nullptr && c->get_constval().stringval->length() > 1) {
808 throw std::runtime_error(
"String after ESCAPE must have a single character.");
810 escape_char = (*c->get_constval().stringval)[0];
811 if (escape_char !=
'\\') {
812 throw std::runtime_error(
"Only supporting '\\' escape character.");
818 if (translate_to_like_pattern(pattern, escape_char)) {
819 return LikeExpr::get(arg_expr, pattern_expr, escape_expr,
false, is_not);
822 std::shared_ptr<Analyzer::Expr>
result =
823 makeExpr<Analyzer::RegexpExpr>(arg_expr->decompress(), pattern_expr, escape_expr);
830 std::shared_ptr<Analyzer::Expr> LikelihoodExpr::analyze(
834 auto arg_expr = arg_->analyze(catalog, query, allow_tlist_ref);
835 return LikelihoodExpr::get(arg_expr, likelihood_, is_not_);
838 std::shared_ptr<Analyzer::Expr> LikelihoodExpr::get(
839 std::shared_ptr<Analyzer::Expr> arg_expr,
842 if (!arg_expr->get_type_info().is_boolean()) {
843 throw std::runtime_error(
"likelihood expression expects boolean type.");
845 std::shared_ptr<Analyzer::Expr>
result = makeExpr<Analyzer::LikelihoodExpr>(
846 arg_expr->decompress(), is_not ? 1 - likelihood : likelihood);
850 std::shared_ptr<Analyzer::Expr> WidthBucketExpr::analyze(
854 auto target_value = target_value_->analyze(catalog, query, allow_tlist_ref);
855 auto lower_bound = lower_bound_->analyze(catalog, query, allow_tlist_ref);
856 auto upper_bound = upper_bound_->analyze(catalog, query, allow_tlist_ref);
857 auto partition_count = partition_count_->analyze(catalog, query, allow_tlist_ref);
861 std::shared_ptr<Analyzer::Expr> WidthBucketExpr::get(
862 std::shared_ptr<Analyzer::Expr> target_value,
865 std::shared_ptr<Analyzer::Expr> partition_count) {
866 std::shared_ptr<Analyzer::Expr>
result = makeExpr<Analyzer::WidthBucketExpr>(
871 std::shared_ptr<Analyzer::Expr> ExistsExpr::analyze(
875 throw std::runtime_error(
"Subqueries are not supported yet.");
879 std::shared_ptr<Analyzer::Expr> ColumnRef::analyze(
886 if (column_ ==
nullptr) {
887 throw std::runtime_error(
"invalid column name *.");
889 if (table_ !=
nullptr) {
892 throw std::runtime_error(
"range variable or table name " + *table_ +
898 throw std::runtime_error(
"Column name " + *column_ +
" does not exist.");
905 cd = rte->get_column_desc(catalog, *column_);
906 if (cd !=
nullptr && !found) {
909 table_id = rte->get_table_id();
910 }
else if (cd !=
nullptr && found) {
911 throw std::runtime_error(
"Column name " + *column_ +
" is ambiguous.");
915 if (cd ==
nullptr && allow_tlist_ref != TlistRefType::TLIST_NONE) {
920 std::shared_ptr<Analyzer::TargetEntry> tle;
922 if (*column_ == p->get_resname() && !found) {
926 }
else if (*column_ == p->get_resname() && found) {
927 throw std::runtime_error(
"Output alias " + *column_ +
" is ambiguous.");
932 if (dynamic_cast<Analyzer::Var*>(tle->get_expr())) {
938 if (allow_tlist_ref == TlistRefType::TLIST_COPY) {
939 return tle->get_expr()->deep_copy();
941 return makeExpr<Analyzer::Var>(
947 throw std::runtime_error(
"Column name " + *column_ +
" does not exist.");
950 return makeExpr<Analyzer::ColumnVar>(cd->columnType, table_id, cd->columnId, rte_idx);
953 std::shared_ptr<Analyzer::Expr> FunctionRef::analyze(
959 std::shared_ptr<Analyzer::Expr> arg_expr;
961 if (boost::iequals(*name_,
"count")) {
965 arg_expr = arg_->analyze(catalog, query, allow_tlist_ref);
968 throw std::runtime_error(
969 "Strings must be dictionary-encoded in COUNT(DISTINCT).");
972 throw std::runtime_error(
"Only COUNT(DISTINCT) is supported on arrays.");
975 is_distinct = distinct_;
978 throw std::runtime_error(
"Cannot compute " + *name_ +
" with argument '*'.");
980 if (boost::iequals(*name_,
"min")) {
982 arg_expr = arg_->analyze(catalog, query, allow_tlist_ref);
983 arg_expr = arg_expr->decompress();
984 result_type = arg_expr->get_type_info();
985 }
else if (boost::iequals(*name_,
"max")) {
987 arg_expr = arg_->analyze(catalog, query, allow_tlist_ref);
988 arg_expr = arg_expr->decompress();
989 result_type = arg_expr->get_type_info();
990 }
else if (boost::iequals(*name_,
"avg")) {
992 arg_expr = arg_->analyze(catalog, query, allow_tlist_ref);
993 if (!arg_expr->get_type_info().is_number()) {
994 throw std::runtime_error(
"Cannot compute AVG on non-number-type arguments.");
996 arg_expr = arg_expr->decompress();
998 }
else if (boost::iequals(*name_,
"sum")) {
1000 arg_expr = arg_->analyze(catalog, query, allow_tlist_ref);
1001 if (!arg_expr->get_type_info().is_number()) {
1002 throw std::runtime_error(
"Cannot compute SUM on non-number-type arguments.");
1004 arg_expr = arg_expr->decompress();
1006 : arg_expr->get_type_info();
1007 }
else if (boost::iequals(*name_,
"unnest")) {
1008 arg_expr = arg_->analyze(catalog, query, allow_tlist_ref);
1009 const SQLTypeInfo& arg_ti = arg_expr->get_type_info();
1011 throw std::runtime_error(arg_->to_string() +
" is not of array type.");
1015 throw std::runtime_error(
"invalid function name: " + *name_);
1017 if (arg_expr->get_type_info().is_string() ||
1018 arg_expr->get_type_info().get_type() ==
kARRAY) {
1019 throw std::runtime_error(
1020 "Only COUNT(DISTINCT ) aggregate is supported on strings and arrays.");
1025 return makeExpr<Analyzer::AggExpr>(
1026 result_type, agg_type, arg_expr,
is_distinct,
nullptr);
1029 std::shared_ptr<Analyzer::Expr> CastExpr::analyze(
1033 target_type_->check_type();
1034 auto arg_expr = arg_->analyze(catalog, query, allow_tlist_ref);
1036 target_type_->get_param1(),
1037 target_type_->get_param2(),
1038 arg_expr->get_type_info().get_notnull());
1039 if (arg_expr->get_type_info().get_type() != target_type_->get_type() &&
1041 arg_expr->decompress();
1043 return arg_expr->add_cast(ti);
1046 std::shared_ptr<Analyzer::Expr> CaseExpr::analyze(
1051 std::list<std::pair<std::shared_ptr<Analyzer::Expr>, std::shared_ptr<Analyzer::Expr>>>
1053 for (
auto& p : when_then_list_) {
1054 auto e1 = p->get_expr1()->analyze(catalog, query, allow_tlist_ref);
1055 if (e1->get_type_info().get_type() !=
kBOOLEAN) {
1056 throw std::runtime_error(
"Only boolean expressions can be used after WHEN.");
1058 auto e2 = p->get_expr2()->analyze(catalog, query, allow_tlist_ref);
1059 expr_pair_list.emplace_back(e1, e2);
1062 else_expr_ ? else_expr_->analyze(catalog, query, allow_tlist_ref) :
nullptr;
1063 return normalize(expr_pair_list, else_e);
1070 if (*s ==
"t" || *s ==
"true" || *s ==
"T" || *s ==
"True") {
1072 }
else if (*s ==
"f" || *s ==
"false" || *s ==
"F" || *s ==
"False") {
1075 throw std::runtime_error(
"Invalid string for boolean " + *s);
1081 std::vector<std::string>& warnings,
1082 std::string& deferred_copy_from_partitions_) {
1083 if (!options_.empty()) {
1084 for (
auto& p : options_) {
1085 if (boost::iequals(*p->get_name(),
"max_reject")) {
1087 if (int_literal ==
nullptr) {
1088 throw std::runtime_error(
"max_reject option must be an integer.");
1091 }
else if (boost::iequals(*p->get_name(),
"buffer_size")) {
1093 if (int_literal ==
nullptr) {
1094 throw std::runtime_error(
"buffer_size option must be an integer.");
1097 }
else if (boost::iequals(*p->get_name(),
"threads")) {
1099 if (int_literal ==
nullptr) {
1100 throw std::runtime_error(
"Threads option must be an integer.");
1103 }
else if (boost::iequals(*p->get_name(),
"delimiter")) {
1106 if (str_literal ==
nullptr) {
1107 throw std::runtime_error(
"Delimiter option must be a string.");
1109 throw std::runtime_error(
"Delimiter must be a single character string.");
1112 }
else if (boost::iequals(*p->get_name(),
"nulls")) {
1115 if (str_literal ==
nullptr) {
1116 throw std::runtime_error(
"Nulls option must be a string.");
1119 }
else if (boost::iequals(*p->get_name(),
"header")) {
1122 if (str_literal ==
nullptr) {
1123 throw std::runtime_error(
"Header option must be a boolean.");
1128 #ifdef ENABLE_IMPORT_PARQUET
1129 }
else if (boost::iequals(*p->get_name(),
"parquet")) {
1131 "Deprecation Warning: COPY FROM WITH (parquet='true') is deprecated. Use "
1132 "WITH (source_type='parquet_file') instead.");
1135 if (str_literal ==
nullptr) {
1136 throw std::runtime_error(
"'parquet' option must be a boolean.");
1143 #endif // ENABLE_IMPORT_PARQUET
1144 }
else if (boost::iequals(*p->get_name(),
"s3_access_key")) {
1147 if (str_literal ==
nullptr) {
1148 throw std::runtime_error(
"Option s3_access_key must be a string.");
1151 }
else if (boost::iequals(*p->get_name(),
"s3_secret_key")) {
1154 if (str_literal ==
nullptr) {
1155 throw std::runtime_error(
"Option s3_secret_key must be a string.");
1158 }
else if (boost::iequals(*p->get_name(),
"s3_session_token")) {
1161 if (str_literal ==
nullptr) {
1162 throw std::runtime_error(
"Option s3_session_token must be a string.");
1165 }
else if (boost::iequals(*p->get_name(),
"s3_region")) {
1168 if (str_literal ==
nullptr) {
1169 throw std::runtime_error(
"Option s3_region must be a string.");
1172 }
else if (boost::iequals(*p->get_name(),
"s3_endpoint")) {
1175 if (str_literal ==
nullptr) {
1176 throw std::runtime_error(
"Option s3_endpoint must be a string.");
1179 }
else if (boost::iequals(*p->get_name(),
"s3_max_concurrent_downloads")) {
1181 if (int_literal ==
nullptr) {
1182 throw std::runtime_error(
1183 "'s3_max_concurrent_downloads' option must be an integer");
1185 const int s3_max_concurrent_downloads = int_literal->
get_intval();
1186 if (s3_max_concurrent_downloads > 0) {
1189 throw std::runtime_error(
1190 "Invalid value for 's3_max_concurrent_downloads' option (must be > 0): " +
1193 }
else if (boost::iequals(*p->get_name(),
"quote")) {
1196 if (str_literal ==
nullptr) {
1197 throw std::runtime_error(
"Quote option must be a string.");
1199 throw std::runtime_error(
"Quote must be a single character string.");
1202 }
else if (boost::iequals(*p->get_name(),
"escape")) {
1205 if (str_literal ==
nullptr) {
1206 throw std::runtime_error(
"Escape option must be a string.");
1208 throw std::runtime_error(
"Escape must be a single character string.");
1211 }
else if (boost::iequals(*p->get_name(),
"line_delimiter")) {
1214 if (str_literal ==
nullptr) {
1215 throw std::runtime_error(
"Line_delimiter option must be a string.");
1217 throw std::runtime_error(
"Line_delimiter must be a single character string.");
1220 }
else if (boost::iequals(*p->get_name(),
"quoted")) {
1223 if (str_literal ==
nullptr) {
1224 throw std::runtime_error(
"Quoted option must be a boolean.");
1227 }
else if (boost::iequals(*p->get_name(),
"plain_text")) {
1230 if (str_literal ==
nullptr) {
1231 throw std::runtime_error(
"plain_text option must be a boolean.");
1234 }
else if (boost::iequals(*p->get_name(),
"array_marker")) {
1237 if (str_literal ==
nullptr) {
1238 throw std::runtime_error(
"Array Marker option must be a string.");
1240 throw std::runtime_error(
1241 "Array Marker option must be exactly two characters. Default is {}.");
1245 }
else if (boost::iequals(*p->get_name(),
"array_delimiter")) {
1248 if (str_literal ==
nullptr) {
1249 throw std::runtime_error(
"Array Delimiter option must be a string.");
1251 throw std::runtime_error(
"Array Delimiter must be a single character string.");
1254 }
else if (boost::iequals(*p->get_name(),
"lonlat")) {
1257 if (str_literal ==
nullptr) {
1258 throw std::runtime_error(
"Lonlat option must be a boolean.");
1261 }
else if (boost::iequals(*p->get_name(),
"geo")) {
1263 "Deprecation Warning: COPY FROM WITH (geo='true') is deprecated. Use WITH "
1264 "(source_type='geo_file') instead.");
1267 if (str_literal ==
nullptr) {
1268 throw std::runtime_error(
"'geo' option must be a boolean.");
1273 }
else if (boost::iequals(*p->get_name(),
"source_type")) {
1276 if (str_literal ==
nullptr) {
1277 throw std::runtime_error(
"'source_type' option must be a string.");
1280 if (boost::iequals(*s,
"delimited_file")) {
1282 }
else if (boost::iequals(*s,
"geo_file")) {
1284 #if ENABLE_IMPORT_PARQUET
1285 }
else if (boost::iequals(*s,
"parquet_file")) {
1288 }
else if (boost::iequals(*s,
"raster_file")) {
1290 }
else if (boost::iequals(*s,
"regex_parsed_file")) {
1293 throw std::runtime_error(
1294 "Invalid string for 'source_type' option (must be 'GEO_FILE', 'RASTER_FILE'"
1295 #
if ENABLE_IMPORT_PARQUET
1298 ", 'REGEX_PARSED_FILE'"
1299 " or 'DELIMITED_FILE'): " +
1302 }
else if (boost::iequals(*p->get_name(),
"geo_coords_type")) {
1305 if (str_literal ==
nullptr) {
1306 throw std::runtime_error(
"'geo_coords_type' option must be a string");
1309 if (boost::iequals(*s,
"geography")) {
1310 throw std::runtime_error(
1311 "GEOGRAPHY coords type not yet supported. Please use GEOMETRY.");
1313 }
else if (boost::iequals(*s,
"geometry")) {
1316 throw std::runtime_error(
1317 "Invalid string for 'geo_coords_type' option (must be 'GEOGRAPHY' or "
1321 }
else if (boost::iequals(*p->get_name(),
"raster_point_type")) {
1324 if (str_literal ==
nullptr) {
1325 throw std::runtime_error(
"'raster_point_type' option must be a string");
1328 if (boost::iequals(*s,
"none")) {
1330 }
else if (boost::iequals(*s,
"auto")) {
1332 }
else if (boost::iequals(*s,
"smallint")) {
1334 }
else if (boost::iequals(*s,
"int")) {
1336 }
else if (boost::iequals(*s,
"float")) {
1338 }
else if (boost::iequals(*s,
"double")) {
1340 }
else if (boost::iequals(*s,
"point")) {
1343 throw std::runtime_error(
1344 "Invalid string for 'raster_point_type' option (must be 'NONE', 'AUTO', "
1345 "'SMALLINT', 'INT', 'FLOAT', 'DOUBLE' or 'POINT'): " +
1348 }
else if (boost::iequals(*p->get_name(),
"raster_point_transform")) {
1351 if (str_literal ==
nullptr) {
1352 throw std::runtime_error(
"'raster_point_transform' option must be a string");
1355 if (boost::iequals(*s,
"none")) {
1357 }
else if (boost::iequals(*s,
"auto")) {
1359 }
else if (boost::iequals(*s,
"file")) {
1361 }
else if (boost::iequals(*s,
"world")) {
1365 throw std::runtime_error(
1366 "Invalid string for 'raster_point_transform' option (must be 'NONE', "
1367 "'AUTO', 'FILE' or 'WORLD'): " +
1370 }
else if (boost::iequals(*p->get_name(),
"raster_import_bands")) {
1373 if (str_literal ==
nullptr) {
1374 throw std::runtime_error(
"'raster_import_bands' option must be a string");
1376 const std::string* raster_import_bands = str_literal->
get_stringval();
1377 if (raster_import_bands) {
1380 throw std::runtime_error(
"Invalid value for 'raster_import_bands' option");
1382 }
else if (boost::iequals(*p->get_name(),
"raster_import_dimensions")) {
1385 if (str_literal ==
nullptr) {
1386 throw std::runtime_error(
"'raster_import_dimensions' option must be a string");
1388 const std::string* raster_import_dimensions = str_literal->
get_stringval();
1389 if (raster_import_dimensions) {
1392 throw std::runtime_error(
"Invalid value for 'raster_import_dimensions' option");
1394 }
else if (boost::iequals(*p->get_name(),
"geo_coords_encoding")) {
1397 if (str_literal ==
nullptr) {
1398 throw std::runtime_error(
"'geo_coords_encoding' option must be a string");
1401 if (boost::iequals(*s,
"none")) {
1404 }
else if (boost::iequals(*s,
"compressed(32)")) {
1408 throw std::runtime_error(
1409 "Invalid string for 'geo_coords_encoding' option (must be 'NONE' or "
1410 "'COMPRESSED(32)'): " +
1413 }
else if (boost::iequals(*p->get_name(),
"raster_scanlines_per_thread")) {
1415 if (int_literal ==
nullptr) {
1416 throw std::runtime_error(
1417 "'raster_scanlines_per_thread' option must be an integer");
1419 const int raster_scanlines_per_thread = int_literal->
get_intval();
1420 if (raster_scanlines_per_thread < 0) {
1421 throw std::runtime_error(
1422 "'raster_scanlines_per_thread' option must be >= 0, with 0 denoting auto "
1426 }
else if (boost::iequals(*p->get_name(),
"geo_coords_srid")) {
1428 if (int_literal ==
nullptr) {
1429 throw std::runtime_error(
"'geo_coords_srid' option must be an integer");
1432 if (srid == 4326 || srid == 3857 || srid == 900913) {
1435 throw std::runtime_error(
1436 "Invalid value for 'geo_coords_srid' option (must be 4326, 3857, or "
1440 }
else if (boost::iequals(*p->get_name(),
"geo_layer_name")) {
1443 if (str_literal ==
nullptr) {
1444 throw std::runtime_error(
"'geo_layer_name' option must be a string");
1446 const std::string* layer_name = str_literal->
get_stringval();
1450 throw std::runtime_error(
"Invalid value for 'geo_layer_name' option");
1452 }
else if (boost::iequals(*p->get_name(),
"partitions")) {
1453 const auto partitions =
1454 static_cast<const StringLiteral*
>(p->get_value())->get_stringval();
1456 const auto partitions_uc = boost::to_upper_copy<std::string>(*partitions);
1457 if (partitions_uc !=
"REPLICATED") {
1458 throw std::runtime_error(
1459 "Invalid value for 'partitions' option. Must be 'REPLICATED'.");
1461 deferred_copy_from_partitions_ = partitions_uc;
1462 }
else if (boost::iequals(*p->get_name(),
"geo_assign_render_groups")) {
1465 if (str_literal ==
nullptr) {
1466 throw std::runtime_error(
"geo_assign_render_groups option must be a boolean.");
1469 }
else if (boost::iequals(*p->get_name(),
"geo_explode_collections")) {
1472 if (str_literal ==
nullptr) {
1473 throw std::runtime_error(
"geo_explode_collections option must be a boolean.");
1476 }
else if (boost::iequals(*p->get_name(),
"source_srid")) {
1478 if (int_literal ==
nullptr) {
1479 throw std::runtime_error(
"'source_srid' option must be an integer");
1485 throw std::runtime_error(
1486 "'source_srid' option can only be used on csv/tsv files");
1488 }
else if (boost::iequals(*p->get_name(),
"regex_path_filter")) {
1491 if (str_literal ==
nullptr) {
1492 throw std::runtime_error(
"Option regex_path_filter must be a string.");
1496 string_val.empty() ? std::nullopt : std::optional<std::string>{string_val};
1497 }
else if (boost::iequals(*p->get_name(),
"file_sort_order_by")) {
1500 if (str_literal ==
nullptr) {
1501 throw std::runtime_error(
"Option file_sort_order_by must be a string.");
1505 string_val.empty() ? std::nullopt : std::optional<std::string>{string_val};
1506 }
else if (boost::iequals(*p->get_name(),
"file_sort_regex")) {
1509 if (str_literal ==
nullptr) {
1510 throw std::runtime_error(
"Option file_sort_regex must be a string.");
1514 string_val.empty() ? std::nullopt : std::optional<std::string>{string_val};
1515 }
else if (boost::iequals(*p->get_name(),
"raster_point_compute_angle")) {
1518 if (str_literal ==
nullptr) {
1519 throw std::runtime_error(
1520 "'raster_point_compute_angle' option must be a boolean.");
1525 }
else if (boost::iequals(*p->get_name(),
"odbc_sql_order_by")) {
1526 if (
auto str_literal = dynamic_cast<const StringLiteral*>(p->get_value())) {
1529 throw std::runtime_error(
"Option odbc_sql_order_by must be a string.");
1531 }
else if (boost::iequals(*p->get_name(),
"odbc_username")) {
1534 if (str_literal ==
nullptr) {
1535 throw std::runtime_error(
"Option odbc_username must be a string.");
1539 }
else if (boost::iequals(*p->get_name(),
"odbc_password")) {
1542 if (str_literal ==
nullptr) {
1543 throw std::runtime_error(
"Option odbc_password must be a string.");
1547 }
else if (boost::iequals(*p->get_name(),
"odbc_credential_string")) {
1550 if (str_literal ==
nullptr) {
1551 throw std::runtime_error(
"Option odbc_credential_string must be a string.");
1555 }
else if (boost::iequals(*p->get_name(),
"odbc_dsn")) {
1558 if (str_literal ==
nullptr) {
1559 throw std::runtime_error(
"Option odbc_dsn must be a string.");
1563 }
else if (boost::iequals(*p->get_name(),
"odbc_connection_string")) {
1566 if (str_literal ==
nullptr) {
1567 throw std::runtime_error(
"Option odbc_connection_string must be a string.");
1571 }
else if (boost::iequals(*p->get_name(),
"line_start_regex")) {
1574 if (str_literal ==
nullptr) {
1575 throw std::runtime_error(
"Option line_start_regex must be a string.");
1579 }
else if (boost::iequals(*p->get_name(),
"line_regex")) {
1582 if (str_literal ==
nullptr) {
1583 throw std::runtime_error(
"Option line_regex must be a string.");
1587 }
else if (boost::iequals(*p->get_name(),
"add_metadata_columns") &&
1591 if (str_literal ==
nullptr) {
1592 throw std::runtime_error(
"'add_metadata_columns' option must be a string.");
1596 throw std::runtime_error(
"Invalid option for COPY: " + *p->get_name());
1612 std::shared_ptr<Analyzer::Expr> CaseExpr::normalize(
1613 const std::list<std::pair<std::shared_ptr<Analyzer::Expr>,
1614 std::shared_ptr<Analyzer::Expr>>>& expr_pair_list,
1615 const std::shared_ptr<Analyzer::Expr> else_e_in,
1616 const Executor* executor) {
1618 bool has_agg =
false;
1629 for (
auto& p : expr_pair_list) {
1631 CHECK(e1->get_type_info().is_boolean());
1633 if (e2->get_contains_agg()) {
1636 const auto& e2_ti = e2->get_type_info();
1637 if (e2_ti.is_string() && !e2_ti.is_dict_encoded_string() &&
1639 CHECK(e2_ti.is_none_encoded_string());
1640 none_encoded_literal_ti =
1648 }
else if (e2_ti.get_type() ==
kNULLT) {
1650 e2->set_type_info(ti);
1651 }
else if (ti != e2_ti) {
1652 if (ti.
is_string() && e2_ti.is_string()) {
1656 }
else if (ti.
is_number() && e2_ti.is_number()) {
1658 }
else if (ti.
is_boolean() && e2_ti.is_boolean()) {
1661 throw std::runtime_error(
1662 "Expressions in THEN clause must be of the same or compatible types.");
1666 auto else_e = else_e_in;
1667 const auto& else_ti = else_e->get_type_info();
1669 if (else_e->get_contains_agg()) {
1672 if (else_ti.is_string() && !else_ti.is_dict_encoded_string() &&
1674 CHECK(else_ti.is_none_encoded_string());
1675 none_encoded_literal_ti =
1684 else_e->set_type_info(ti);
1685 }
else if (ti != else_ti) {
1687 if (ti.
is_string() && else_ti.is_string()) {
1691 }
else if (ti.
is_number() && else_ti.is_number()) {
1693 }
else if (ti.
is_boolean() && else_ti.is_boolean()) {
1696 throw std::runtime_error(
1698 "Expressions in ELSE clause must be of the same or compatible types as "
1699 "those in the THEN clauses.");
1709 ti = none_encoded_literal_ti;
1712 std::list<std::pair<std::shared_ptr<Analyzer::Expr>, std::shared_ptr<Analyzer::Expr>>>
1713 cast_expr_pair_list;
1714 for (
auto p : expr_pair_list) {
1716 cast_expr_pair_list.emplace_back(p.first, p.second->add_cast(ti));
1718 if (else_e !=
nullptr) {
1719 else_e = else_e->add_cast(ti);
1724 else_e = makeExpr<Analyzer::Constant>(ti,
true, d);
1727 throw std::runtime_error(
1728 "Cannot deduce the type for case expressions, all branches null");
1731 auto case_expr = makeExpr<Analyzer::CaseExpr>(ti, has_agg, cast_expr_pair_list, else_e);
1736 std::string str(
"CASE ");
1737 for (
auto& p : when_then_list_) {
1738 str +=
"WHEN " + p->get_expr1()->to_string() +
" THEN " +
1739 p->get_expr2()->to_string() +
" ";
1741 if (else_expr_ !=
nullptr) {
1742 str +=
"ELSE " + else_expr_->to_string();
1750 left_->analyze(catalog, query);
1752 right_->analyze(catalog, *right_query);
1759 std::shared_ptr<Analyzer::Expr> p;
1760 if (having_clause_ !=
nullptr) {
1761 p = having_clause_->analyze(catalog, query, Expr::TlistRefType::TLIST_COPY);
1762 if (p->get_type_info().get_type() !=
kBOOLEAN) {
1763 throw std::runtime_error(
"Only boolean expressions can be in HAVING clause.");
1772 std::list<std::shared_ptr<Analyzer::Expr>> groupby;
1773 if (!groupby_clause_.empty()) {
1775 std::shared_ptr<Analyzer::Expr> gexpr;
1776 const std::vector<std::shared_ptr<Analyzer::TargetEntry>>& tlist =
1778 for (
auto& c : groupby_clause_) {
1780 if (dynamic_cast<Literal*>(c.get())) {
1783 throw std::runtime_error(
"Invalid literal in GROUP BY clause.");
1786 if (varno <= 0 || varno > static_cast<int>(tlist.size())) {
1787 throw std::runtime_error(
"Invalid ordinal number in GROUP BY clause.");
1789 if (tlist[varno - 1]->get_expr()->get_contains_agg()) {
1790 throw std::runtime_error(
1791 "Ordinal number in GROUP BY cannot reference an expression containing "
1795 gexpr = makeExpr<Analyzer::Var>(
1798 gexpr = c->analyze(catalog, query, Expr::TlistRefType::TLIST_REF);
1801 bool set_new_type =
false;
1804 set_new_type =
true;
1809 std::shared_ptr<Analyzer::Var> v;
1810 if (std::dynamic_pointer_cast<Analyzer::Var>(gexpr)) {
1812 int n = v->get_varno();
1813 gexpr = tlist[n - 1]->get_own_expr();
1815 if (cv !=
nullptr) {
1820 v->set_varno(gexpr_no);
1821 tlist[n - 1]->set_expr(v);
1825 groupby.push_back(new_e);
1827 v->set_type_info(new_e->get_type_info());
1830 groupby.push_back(gexpr);
1837 auto e = t->get_expr();
1838 e->check_group_by(groupby);
1846 if (where_clause_ ==
nullptr) {
1850 auto p = where_clause_->analyze(catalog, query, Expr::TlistRefType::TLIST_COPY);
1851 if (p->get_type_info().get_type() !=
kBOOLEAN) {
1852 throw std::runtime_error(
"Only boolean expressions can be in WHERE clause.");
1859 std::vector<std::shared_ptr<Analyzer::TargetEntry>>& tlist =
1861 if (select_clause_.empty()) {
1865 rte->expand_star_in_targetlist(catalog, tlist, rte_idx++);
1868 for (
auto& p : select_clause_) {
1869 const Parser::Expr* select_expr = p->get_select_expr();
1871 if (
typeid(*select_expr) ==
typeid(
ColumnRef) &&
1872 dynamic_cast<const ColumnRef*>(select_expr)->get_column() ==
nullptr) {
1873 const std::string* range_var_name =
1874 dynamic_cast<const ColumnRef*
>(select_expr)->get_table();
1877 throw std::runtime_error(
"invalid range variable name: " + *range_var_name);
1882 auto e = select_expr->
analyze(catalog, query);
1883 std::string resname;
1885 if (p->get_alias() !=
nullptr) {
1886 resname = *p->get_alias();
1887 }
else if (std::dynamic_pointer_cast<Analyzer::ColumnVar>(e) &&
1888 !std::dynamic_pointer_cast<Analyzer::Var>(e)) {
1891 colvar->get_table_id(), colvar->get_column_id());
1894 if (e->get_type_info().get_type() ==
kNULLT) {
1895 throw std::runtime_error(
1896 "Untyped NULL in SELECT clause. Use CAST to specify a type.");
1899 bool unnest = (o !=
nullptr && o->get_optype() ==
kUNNEST);
1900 auto tle = std::make_shared<Analyzer::TargetEntry>(resname, e, unnest);
1901 tlist.push_back(tle);
1910 for (
auto& p : from_clause_) {
1913 if (table_desc ==
nullptr) {
1914 throw std::runtime_error(
"Table " + *p->get_table_name() +
" does not exist.");
1916 std::string range_var;
1917 if (p->get_range_var() ==
nullptr) {
1918 range_var = *p->get_table_name();
1920 range_var = *p->get_range_var();
1930 analyze_from_clause(catalog, query);
1931 analyze_select_clause(catalog, query);
1932 analyze_where_clause(catalog, query);
1933 analyze_group_by(catalog, query);
1934 analyze_having_clause(catalog, query);
1941 boost::replace_all(s,
"\\\\t",
"\t");
1942 boost::replace_all(s,
"\\t",
"\t");
1943 boost::replace_all(s,
"\\\\n",
"\n");
1944 boost::replace_all(s,
"\\n",
"\n");
1950 std::regex e1(
"(\\\\x[0-9A-Fa-f][0-9A-Fa-f])");
1951 while (std::regex_search(s, m, e1)) {
1952 std::string original(m[0].first, m[0].second);
1953 std::string replacement;
1954 long val = strtol(original.substr(2, 2).c_str(), NULL, 16);
1955 replacement.push_back(val);
1956 boost::replace_all(s, original, replacement);
1960 std::regex e2(
"(\\\\u[0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f])");
1961 while (std::regex_search(s, m, e2)) {
1962 std::string original(m[0].first, m[0].second);
1963 std::string replacement;
1964 long val = strtol(original.substr(2, 4).c_str(), NULL, 16);
1965 replacement.push_back(val);
1966 boost::replace_all(s, original, replacement);
1973 std::list<std::unique_ptr<NameValueAssign>>& nameValueList,
1974 bool stringToNull =
false,
1975 bool stringToInteger =
false) {
1976 if (payload.HasMember(
"options") && payload[
"options"].IsObject()) {
1977 for (
const auto& option : payload[
"options"].GetObject()) {
1978 auto option_name = std::make_unique<std::string>(option.name.GetString());
1979 std::unique_ptr<Literal> literal_value;
1980 if (option.value.IsString()) {
1981 std::string str = option.value.GetString();
1982 if (stringToNull && str ==
"") {
1983 literal_value = std::make_unique<NullLiteral>();
1984 }
else if (stringToInteger && std::all_of(str.begin(), str.end(), ::isdigit)) {
1985 int iVal = std::stoi(str);
1986 literal_value = std::make_unique<IntLiteral>(iVal);
1990 auto unique_literal_string = std::make_unique<std::string>(
unescape(str));
1992 std::make_unique<StringLiteral>(unique_literal_string.release());
1994 }
else if (option.value.IsInt() || option.value.IsInt64()) {
1995 literal_value = std::make_unique<IntLiteral>(
json_i64(option.value));
1996 }
else if (option.value.IsNull()) {
1997 literal_value = std::make_unique<NullLiteral>();
1999 throw std::runtime_error(
"Unable to handle literal for " + *option_name);
2001 CHECK(literal_value);
2003 nameValueList.emplace_back(std::make_unique<NameValueAssign>(
2004 option_name.release(), literal_value.release()));
2015 throw std::runtime_error(
"OFFSET cannot be negative.");
2018 query_expr_->analyze(catalog, query);
2023 const std::vector<std::shared_ptr<Analyzer::TargetEntry>>& tlist =
2025 std::list<Analyzer::OrderEntry>* order_by =
new std::list<Analyzer::OrderEntry>();
2026 if (!orderby_clause_.empty()) {
2027 for (
auto& p : orderby_clause_) {
2028 int tle_no = p->get_colno();
2032 const std::string*
name = p->get_column()->get_column();
2035 for (
auto tle : tlist) {
2036 if (tle->get_resname() == *
name) {
2043 throw std::runtime_error(
"invalid name in order by: " + *name);
2046 order_by->push_back(
2052 for (
int i = 1; i <= static_cast<int>(tlist.size()); i++) {
2053 bool in_orderby =
false;
2054 std::for_each(order_by->begin(),
2057 in_orderby = in_orderby || (i == oe.tle_no);
2068 std::string str = select_expr_->to_string();
2069 if (alias_ !=
nullptr) {
2070 str +=
" AS " + *alias_;
2076 std::string str = *table_name_;
2077 if (range_var_ !=
nullptr) {
2078 str +=
" " + *range_var_;
2085 if (table_ ==
nullptr) {
2087 }
else if (column_ ==
nullptr) {
2088 str = *table_ +
".*";
2090 str = *table_ +
"." + *column_;
2096 std::string op_str[] = {
2097 "=",
"===",
"<>",
"<",
">",
"<=",
">=",
" AND ",
" OR ",
"NOT",
"-",
"+",
"*",
"/"};
2100 str =
"-(" + left_->to_string() +
")";
2101 }
else if (optype_ ==
kNOT) {
2102 str =
"NOT (" + left_->to_string() +
")";
2104 str = left_->to_string() +
"[" + right_->to_string() +
"]";
2105 }
else if (optype_ ==
kUNNEST) {
2106 str =
"UNNEST(" + left_->to_string() +
")";
2107 }
else if (optype_ ==
kIN) {
2108 str =
"(" + left_->to_string() +
" IN " + right_->to_string() +
")";
2110 str =
"(" + left_->to_string() + op_str[optype_] + right_->to_string() +
")";
2116 std::string str = arg_->to_string();
2126 return "EXISTS (" + query_->to_string() +
")";
2132 str += query_->to_string();
2138 std::string str = arg_->to_string();
2140 str +=
" IS NOT NULL";
2149 str += subquery_->to_string();
2155 bool notfirst =
false;
2156 for (
auto& p : value_list_) {
2162 str += p->to_string();
2169 std::string str = arg_->to_string();
2171 str +=
" NOT BETWEEN ";
2175 str += lower_->to_string() +
" AND " + upper_->to_string();
2181 if (calc_encoded_length_) {
2182 str =
"CHAR_LENGTH (" + arg_->to_string() +
")";
2184 str =
"LENGTH (" + arg_->to_string() +
")";
2190 std::string str =
"CARDINALITY(" + arg_->to_string() +
")";
2195 std::string str = arg_->to_string();
2197 str +=
" NOT LIKE ";
2201 str += like_string_->to_string();
2202 if (escape_string_ !=
nullptr) {
2203 str +=
" ESCAPE " + escape_string_->to_string();
2209 std::string str = arg_->to_string();
2211 str +=
" NOT REGEXP ";
2215 str += pattern_string_->to_string();
2216 if (escape_string_ !=
nullptr) {
2217 str +=
" ESCAPE " + escape_string_->to_string();
2223 std::string str =
" WIDTH_BUCKET ";
2224 str += target_value_->to_string();
2226 str += lower_bound_->to_string();
2228 str += upper_bound_->to_string();
2230 str += partition_count_->to_string();
2236 std::string str =
" LIKELIHOOD ";
2237 str += arg_->to_string();
2239 str += boost::lexical_cast<std::string>(is_not_ ? 1.0 - likelihood_ : likelihood_);
2244 std::string str = *name_ +
"(";
2248 if (arg_ ==
nullptr) {
2251 str += arg_->to_string() +
")";
2257 std::string query_str =
"SELECT ";
2259 query_str +=
"DISTINCT ";
2261 if (select_clause_.empty()) {
2264 bool notfirst =
false;
2265 for (
auto& p : select_clause_) {
2271 query_str += p->to_string();
2274 query_str +=
" FROM ";
2275 bool notfirst =
false;
2276 for (
auto& p : from_clause_) {
2282 query_str += p->to_string();
2284 if (where_clause_) {
2285 query_str +=
" WHERE " + where_clause_->to_string();
2287 if (!groupby_clause_.empty()) {
2288 query_str +=
" GROUP BY ";
2289 bool notfirst =
false;
2290 for (
auto& p : groupby_clause_) {
2296 query_str += p->to_string();
2299 if (having_clause_) {
2300 query_str +=
" HAVING " + having_clause_->to_string();
2310 if (td ==
nullptr) {
2311 throw std::runtime_error(
"Table " + *table_ +
" does not exist.");
2314 throw std::runtime_error(
"Insert to views is not supported yet.");
2318 std::list<int> result_col_list;
2319 if (column_list_.empty()) {
2320 const std::list<const ColumnDescriptor*> all_cols =
2322 for (
auto cd : all_cols) {
2323 result_col_list.push_back(cd->columnId);
2326 for (
auto& c : column_list_) {
2328 if (cd ==
nullptr) {
2329 throw std::runtime_error(
"Column " + *c +
" does not exist.");
2331 result_col_list.push_back(cd->
columnId);
2333 if (col_ti.get_physical_cols() > 0) {
2335 for (
auto i = 1; i <= col_ti.get_physical_cols(); i++) {
2338 if (pcd ==
nullptr) {
2339 throw std::runtime_error(
"Column " + *c +
"'s metadata is incomplete.");
2341 result_col_list.push_back(pcd->
columnId);
2351 CHECK(literal.IsObject());
2352 CHECK(literal.HasMember(
"literal"));
2353 CHECK(literal.HasMember(
"type"));
2355 if (type ==
"NULL") {
2357 }
else if (type ==
"CHAR" || type ==
"BOOLEAN") {
2358 auto* val =
new std::string(
json_str(literal[
"literal"]));
2360 }
else if (type ==
"DECIMAL") {
2361 CHECK(literal.HasMember(
"scale"));
2362 CHECK(literal.HasMember(
"precision"));
2363 auto scale =
json_i64(literal[
"scale"]);
2364 auto precision =
json_i64(literal[
"precision"]);
2366 auto int_val = std::stol(
json_str(literal[
"literal"]));
2368 }
else if (precision > 18) {
2369 auto dbl_val = std::stod(
json_str(literal[
"literal"]));
2372 auto* val =
new std::string(
json_str(literal[
"literal"]));
2375 }
else if (type ==
"DOUBLE") {
2376 auto dbl_val = std::stod(
json_str(literal[
"literal"]));
2379 CHECK(
false) <<
"Unexpected calcite data type: " <<
type;
2385 CHECK(array.IsArray());
2386 auto json_elements = array.GetArray();
2387 auto* elements =
new std::list<Expr*>();
2388 for (
const auto& e : json_elements) {
2395 InsertValuesStmt::InsertValuesStmt(
const rapidjson::Value& payload)
2397 CHECK(payload.HasMember(
"name"));
2398 table_ = std::make_unique<std::string>(
json_str(payload[
"name"]));
2400 if (payload.HasMember(
"columns")) {
2401 CHECK(payload[
"columns"].IsArray());
2402 for (
auto& column : payload[
"columns"].GetArray()) {
2404 column_list_.emplace_back(std::make_unique<std::string>(s));
2408 CHECK(payload.HasMember(
"values") && payload[
"values"].IsArray());
2409 auto tuples = payload[
"values"].GetArray();
2410 if (tuples.Empty()) {
2411 throw std::runtime_error(
"Values statement cannot be empty");
2414 for (
const auto& json_tuple : tuples) {
2415 auto values_list = std::make_unique<ValuesList>();
2416 CHECK(json_tuple.IsArray());
2417 auto tuple = json_tuple.GetArray();
2418 for (
const auto& value : tuple) {
2419 CHECK(value.IsObject());
2420 if (value.HasMember(
"array")) {
2433 size_t list_size =
values_lists_[0]->get_value_list().size();
2436 throw std::runtime_error(
2437 "Numbers of columns and values don't match for the "
2442 const std::list<const ColumnDescriptor*> non_phys_cols =
2444 if (non_phys_cols.size() != list_size) {
2445 throw std::runtime_error(
2446 "Number of columns in table does not match the list of values given in the "
2450 std::vector<const ColumnDescriptor*> cds;
2460 const auto& values_list =
values_lists_[i]->get_value_list();
2461 if (values_list.size() != list_size) {
2462 throw std::runtime_error(
2463 "Insert values lists should be of the same size. Expected: " +
2466 auto& query_values_list = query_values_lists[i];
2468 for (
auto& v : values_list) {
2469 auto e = v->analyze(catalog, query);
2470 const auto* cd = cds[cds_id];
2471 const auto& col_ti = cd->columnType;
2472 if (col_ti.get_notnull()) {
2474 if (c !=
nullptr && c->get_is_null()) {
2475 throw std::runtime_error(
"Cannot insert NULL into column " + cd->columnName);
2478 e = e->add_cast(col_ti);
2482 if (col_ti.get_physical_cols() > 0) {
2483 CHECK(cd->columnType.is_geometry());
2487 if (uoper && uoper->get_optype() ==
kCAST) {
2492 std::string* geo_string{
nullptr};
2494 is_null = c->get_is_null();
2496 geo_string = c->get_constval().stringval;
2499 if (!is_null && !geo_string) {
2500 throw std::runtime_error(
"Expecting a WKT or WKB hex string for column " +
2503 std::vector<double> coords;
2504 std::vector<double> bounds;
2505 std::vector<int> ring_sizes;
2506 std::vector<int> poly_rings;
2512 *geo_string, import_ti, coords, bounds, ring_sizes, poly_rings)) {
2513 throw std::runtime_error(
"Cannot read geometry to insert into column " +
2516 if (coords.empty()) {
2520 if (cd->columnType.get_type() != import_ti.get_type()) {
2524 throw std::runtime_error(
2525 "Imported geometry doesn't match the type of column " + cd->columnName);
2530 if (cd->columnType.get_type() ==
kPOINT) {
2531 if (!coords.empty()) {
2532 throw std::runtime_error(
2533 "NULL POINT with unexpected coordinates in column " + cd->columnName);
2543 const auto* cd_coords = cds[cds_id];
2546 std::list<std::shared_ptr<Analyzer::Expr>> value_exprs;
2547 if (!is_null || cd->columnType.get_type() ==
kPOINT) {
2549 for (
auto cc : compressed_coords) {
2552 auto e = makeExpr<Analyzer::Constant>(
kTINYINT,
false, d);
2553 value_exprs.push_back(e);
2558 makeExpr<Analyzer::Constant>(cd_coords->columnType, is_null, value_exprs),
2562 if (cd->columnType.get_type() ==
kPOLYGON ||
2565 const auto* cd_ring_sizes = cds[cds_id];
2566 CHECK(cd_ring_sizes);
2568 CHECK_EQ(cd_ring_sizes->columnType.get_subtype(),
kINT);
2569 std::list<std::shared_ptr<Analyzer::Expr>> value_exprs;
2571 for (
auto c : ring_sizes) {
2574 auto e = makeExpr<Analyzer::Constant>(
kINT,
false, d);
2575 value_exprs.push_back(e);
2580 makeExpr<Analyzer::Constant>(
2581 cd_ring_sizes->columnType, is_null, value_exprs),
2587 const auto* cd_poly_rings = cds[cds_id];
2588 CHECK(cd_poly_rings);
2590 CHECK_EQ(cd_poly_rings->columnType.get_subtype(),
kINT);
2591 std::list<std::shared_ptr<Analyzer::Expr>> value_exprs;
2593 for (
auto c : poly_rings) {
2596 auto e = makeExpr<Analyzer::Constant>(
kINT,
false, d);
2597 value_exprs.push_back(e);
2602 makeExpr<Analyzer::Constant>(
2603 cd_poly_rings->columnType, is_null, value_exprs),
2610 cd->columnType.get_type() ==
kPOLYGON ||
2612 const auto* cd_bounds = cds[cds_id];
2616 std::list<std::shared_ptr<Analyzer::Expr>> value_exprs;
2618 for (
auto b : bounds) {
2621 auto e = makeExpr<Analyzer::Constant>(
kDOUBLE,
false, d);
2622 value_exprs.push_back(e);
2627 makeExpr<Analyzer::Constant>(cd_bounds->columnType, is_null, value_exprs),
2632 if (cd->columnType.get_type() ==
kPOLYGON ||
2635 const auto* cd_render_group = cds[cds_id];
2636 CHECK(cd_render_group);
2637 CHECK_EQ(cd_render_group->columnType.get_type(),
kINT);
2642 makeExpr<Analyzer::Constant>(cd_render_group->columnType, is_null, d),
2652 auto execute_read_lock = mapd_shared_lock<mapd_shared_mutex>(
2656 const auto td_with_lock =
2662 throw std::runtime_error(
"User has no insert privileges on " + *
table_ +
".");
2668 const auto insert_data_lock =
2674 auto td = td_with_lock();
2677 throw std::runtime_error(
"Singleton inserts on views is not supported.");
2690 ra_executor.executeSimpleInsert(query, insert_data_loader, session);
2694 }
catch (std::exception& e) {
2695 LOG(
ERROR) <<
"An error occurred during insert rollback attempt. Table id: "
2696 << td->tableId <<
", Error: " << e.what();
2700 if (!td->isTemporaryTable()) {
2707 throw std::runtime_error(
"UPDATE statement not supported yet.");
2712 throw std::runtime_error(
"DELETE statement not supported yet.");
2719 if (!col_ti.is_integer() && !col_ti.is_time() &&
2720 !(col_ti.is_string() && col_ti.get_compression() ==
kENCODING_DICT)) {
2721 throw std::runtime_error(
"Cannot shard on type " + col_ti.get_type_name() +
2722 ", encoding " + col_ti.get_compression_name());
2727 const std::list<ColumnDescriptor>& columns) {
2729 for (
const auto& cd : columns) {
2730 if (cd.columnName == name) {
2735 if (cd.columnType.is_geometry()) {
2736 index += cd.columnType.get_physical_cols();
2744 const std::list<ColumnDescriptor>& columns) {
2746 for (
const auto& cd : columns) {
2747 if (boost::to_upper_copy<std::string>(cd.columnName) == name) {
2751 if (cd.columnType.is_geometry()) {
2752 index += cd.columnType.get_physical_cols();
2760 const std::string& field_name,
2761 const std::string& field_value,
2762 rapidjson::Document& document) {
2763 rapidjson::Value field_name_json_str;
2764 field_name_json_str.SetString(
2765 field_name.c_str(), field_name.size(), document.GetAllocator());
2766 rapidjson::Value field_value_json_str;
2767 field_value_json_str.SetString(
2768 field_value.c_str(), field_value.size(), document.GetAllocator());
2769 obj.AddMember(field_name_json_str, field_value_json_str, document.GetAllocator());
2774 const std::vector<SharedDictionaryDef>& shared_dict_defs) {
2775 rapidjson::Document document;
2776 auto& allocator = document.GetAllocator();
2777 rapidjson::Value arr(rapidjson::kArrayType);
2778 if (shard_key_def) {
2779 rapidjson::Value shard_key_obj(rapidjson::kObjectType);
2782 arr.PushBack(shard_key_obj, allocator);
2784 for (
const auto& shared_dict_def : shared_dict_defs) {
2785 rapidjson::Value shared_dict_obj(rapidjson::kObjectType);
2787 set_string_field(shared_dict_obj,
"name", shared_dict_def.get_column(), document);
2789 shared_dict_obj,
"foreign_table", shared_dict_def.get_foreign_table(), document);
2792 shared_dict_def.get_foreign_column(),
2794 arr.PushBack(shared_dict_obj, allocator);
2796 rapidjson::StringBuffer buffer;
2797 rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
2799 return buffer.GetString();
2802 template <
typename LITERAL_TYPE,
2803 typename ASSIGNMENT,
2807 VALIDATE validate = VALIDATE()) {
2808 const auto val = validate(p);
2815 auto assignment = [&td](
const auto val) { td.storageType = val; };
2816 return get_property_value<StringLiteral, decltype(assignment), CaseSensitiveValidate>(
2823 return get_property_value<IntLiteral>(p,
2824 [&td](
const auto val) { td.maxFragRows = val; });
2830 return get_property_value<IntLiteral>(
2831 p, [&df_td](
const auto val) { df_td.maxFragRows = val; });
2837 return get_property_value<IntLiteral>(p,
2838 [&td](
const auto val) { td.maxChunkSize = val; });
2845 return get_property_value<IntLiteral>(
2846 p, [&df_td](
const auto val) { df_td.maxChunkSize = val; });
2852 return get_property_value<StringLiteral>(p, [&df_td](
const auto val) {
2853 if (val.size() != 1) {
2854 throw std::runtime_error(
"Length of DELIMITER must be equal to 1.");
2856 df_td.delimiter = val;
2863 return get_property_value<StringLiteral>(p, [&df_td](
const auto val) {
2864 if (val ==
"FALSE") {
2865 df_td.hasHeader =
false;
2866 }
else if (val ==
"TRUE") {
2867 df_td.hasHeader =
true;
2869 throw std::runtime_error(
"Option HEADER support only 'true' or 'false' values.");
2877 return get_property_value<IntLiteral>(p,
2878 [&td](
const auto val) { td.fragPageSize = val; });
2883 return get_property_value<IntLiteral>(p, [&td](
const auto val) { td.maxRows = val; });
2889 return get_property_value<IntLiteral>(
2890 p, [&df_td](
const auto val) { df_td.skipRows = val; });
2896 return get_property_value<StringLiteral>(p, [&td](
const auto partitions_uc) {
2897 if (partitions_uc !=
"SHARDED" && partitions_uc !=
"REPLICATED") {
2898 throw std::runtime_error(
"PARTITIONS must be SHARDED or REPLICATED");
2900 if (td.shardedColumnId != 0 && partitions_uc ==
"REPLICATED") {
2901 throw std::runtime_error(
2902 "A table cannot be sharded and replicated at the same time");
2904 td.partitions = partitions_uc;
2910 if (!td.shardedColumnId) {
2911 throw std::runtime_error(
"SHARD KEY must be defined.");
2913 return get_property_value<IntLiteral>(p, [&td](
const auto shard_count) {
2915 throw std::runtime_error(
2916 "SHARD_COUNT must be a multiple of the number of leaves in the cluster.");
2918 td.nShards = g_leaf_count ? shard_count / g_leaf_count : shard_count;
2919 if (!td.shardedColumnId && !td.nShards) {
2920 throw std::runtime_error(
2921 "Must specify the number of shards through the SHARD_COUNT option");
2929 return get_property_value<StringLiteral>(p, [&td](
const auto vacuum_uc) {
2930 if (vacuum_uc !=
"IMMEDIATE" && vacuum_uc !=
"DELAYED") {
2931 throw std::runtime_error(
"VACUUM must be IMMEDIATE or DELAYED");
2933 td.hasDeletedCol = boost::iequals(vacuum_uc,
"IMMEDIATE") ?
false :
true;
2940 return get_property_value<StringLiteral>(p, [&td, &columns](
const auto sort_upper) {
2942 if (!td.sortedColumnId) {
2943 throw std::runtime_error(
"Specified sort column " + sort_upper +
" doesn't exist");
2951 auto assignment = [&td](
const auto val) {
2952 td.maxRollbackEpochs =
2957 return get_property_value<IntLiteral, decltype(assignment), PositiveOrZeroValidate>(
2974 const std::unique_ptr<NameValueAssign>& p,
2975 const std::list<ColumnDescriptor>& columns) {
2976 const auto it = tableDefFuncMap.find(boost::to_lower_copy<std::string>(*p->get_name()));
2977 if (it == tableDefFuncMap.end()) {
2978 throw std::runtime_error(
2979 "Invalid CREATE TABLE option " + *p->get_name() +
2980 ". Should be FRAGMENT_SIZE, MAX_CHUNK_SIZE, PAGE_SIZE, MAX_ROLLBACK_EPOCHS, "
2982 "PARTITIONS, SHARD_COUNT, VACUUM, SORT_COLUMN, STORAGE_TYPE.");
2984 return it->second(td, p.get(), columns);
2988 const std::unique_ptr<NameValueAssign>& p,
2989 const std::list<ColumnDescriptor>& columns) {
2990 const auto it = tableDefFuncMap.find(boost::to_lower_copy<std::string>(*p->get_name()));
2991 if (it == tableDefFuncMap.end()) {
2992 throw std::runtime_error(
2993 "Invalid CREATE TABLE AS option " + *p->get_name() +
2994 ". Should be FRAGMENT_SIZE, MAX_CHUNK_SIZE, PAGE_SIZE, MAX_ROLLBACK_EPOCHS, "
2996 "PARTITIONS, SHARD_COUNT, VACUUM, SORT_COLUMN, STORAGE_TYPE or "
2997 "USE_SHARED_DICTIONARIES.");
2999 return it->second(td, p.get(), columns);
3010 const std::unique_ptr<NameValueAssign>& p,
3011 const std::list<ColumnDescriptor>& columns) {
3013 dataframeDefFuncMap.find(boost::to_lower_copy<std::string>(*p->get_name()));
3014 if (it == dataframeDefFuncMap.end()) {
3015 throw std::runtime_error(
3016 "Invalid CREATE DATAFRAME option " + *p->get_name() +
3017 ". Should be FRAGMENT_SIZE, MAX_CHUNK_SIZE, SKIP_ROWS, DELIMITER or HEADER.");
3019 return it->second(df_td, p.get(), columns);
3023 CHECK(element.HasMember(
"name"));
3024 auto col_name = std::make_unique<std::string>(
json_str(element[
"name"]));
3025 CHECK(element.HasMember(
"sqltype"));
3031 if (element.HasMember(
"precision")) {
3032 precision =
json_i64(element[
"precision"]);
3034 if (element.HasMember(
"scale")) {
3035 scale =
json_i64(element[
"scale"]);
3038 std::optional<int64_t> array_size;
3039 if (element.HasMember(
"arraySize")) {
3041 array_size =
json_i64(element[
"arraySize"]);
3043 std::unique_ptr<SQLType> sql_type;
3044 if (element.HasMember(
"subtype")) {
3045 CHECK(element.HasMember(
"coordinateSystem"));
3048 std::make_unique<SQLType>(subtype_sql_types,
3049 static_cast<int>(sql_types),
3050 static_cast<int>(
json_i64(element[
"coordinateSystem"])),
3052 }
else if (precision > 0 && scale > 0) {
3053 sql_type = std::make_unique<SQLType>(sql_types,
3056 array_size.has_value(),
3057 array_size ? *array_size : -1);
3058 }
else if (precision > 0) {
3059 sql_type = std::make_unique<SQLType>(sql_types,
3062 array_size.has_value(),
3063 array_size ? *array_size : -1);
3065 sql_type = std::make_unique<SQLType>(sql_types,
3066 array_size.has_value(),
3067 array_size ? *array_size : -1);
3071 CHECK(element.HasMember(
"nullable"));
3072 const auto nullable =
json_bool(element[
"nullable"]);
3073 std::unique_ptr<ColumnConstraintDef> constraint_def;
3075 if (element.HasMember(
"default") && !element[
"default"].IsNull()) {
3076 std::string* defaultval =
new std::string(
json_str(element[
"default"]));
3077 boost::algorithm::trim_if(*defaultval, boost::is_any_of(
" \"'`"));
3081 constraint_def = std::make_unique<ColumnConstraintDef>(!nullable,
3085 std::unique_ptr<CompressDef> compress_def;
3086 if (element.HasMember(
"encodingType") && !element[
"encodingType"].IsNull()) {
3087 std::string encoding_type =
json_str(element[
"encodingType"]);
3088 CHECK(element.HasMember(
"encodingSize"));
3089 auto encoding_name = std::make_unique<std::string>(
json_str(element[
"encodingType"]));
3090 compress_def = std::make_unique<CompressDef>(encoding_name.release(),
3091 json_i64(element[
"encodingSize"]));
3093 return std::make_unique<ColumnDef>(col_name.release(),
3095 compress_def ? compress_def.release() :
nullptr,
3096 constraint_def ? constraint_def.release() :
nullptr);
3100 std::string element_name,
3101 std::string& table_name,
3102 std::list<std::unique_ptr<TableElement>>& table_element_list) {
3103 const auto elements = payload[element_name].GetArray();
3104 for (
const auto& element : elements) {
3105 CHECK(element.IsObject());
3106 CHECK(element.HasMember(
"type"));
3107 if (
json_str(element[
"type"]) ==
"SQL_COLUMN_DECLARATION") {
3109 table_element_list.emplace_back(std::move(col_def));
3110 }
else if (
json_str(element[
"type"]) ==
"SQL_COLUMN_CONSTRAINT") {
3111 CHECK(element.HasMember(
"name"));
3112 if (
json_str(element[
"name"]) ==
"SHARD_KEY") {
3113 CHECK(element.HasMember(
"columns"));
3114 CHECK(element[
"columns"].IsArray());
3115 const auto& columns = element[
"columns"].GetArray();
3116 if (columns.Size() != size_t(1)) {
3117 throw std::runtime_error(
"Only one shard column is currently supported.");
3119 auto shard_key_def = std::make_unique<ShardKeyDef>(
json_str(columns[0]));
3120 table_element_list.emplace_back(std::move(shard_key_def));
3121 }
else if (
json_str(element[
"name"]) ==
"SHARED_DICT") {
3122 CHECK(element.HasMember(
"columns"));
3123 CHECK(element[
"columns"].IsArray());
3124 const auto& columns = element[
"columns"].GetArray();
3125 if (columns.Size() != size_t(1)) {
3126 throw std::runtime_error(
3127 R
"(Only one column per shared dictionary entry is currently supported. Use multiple SHARED DICT statements to share dictionaries from multiple columns.)");
3129 CHECK(element.HasMember("references") && element[
"references"].IsObject());
3130 const auto& references = element[
"references"].GetObject();
3131 std::string references_table_name;
3132 if (references.HasMember(
"table")) {
3133 references_table_name =
json_str(references[
"table"]);
3135 references_table_name = table_name;
3137 CHECK(references.HasMember(
"column"));
3139 auto shared_dict_def = std::make_unique<SharedDictionaryDef>(
3140 json_str(columns[0]), references_table_name,
json_str(references[
"column"]));
3141 table_element_list.emplace_back(std::move(shared_dict_def));
3144 LOG(
FATAL) <<
"Unsupported type for SQL_COLUMN_CONSTRAINT: "
3148 LOG(
FATAL) <<
"Unsupported element type for CREATE TABLE: "
3149 << element[
"type"].GetString();
3156 CHECK(payload.HasMember(
"name"));
3157 table_ = std::make_unique<std::string>(
json_str(payload[
"name"]));
3158 CHECK(payload.HasMember(
"elements"));
3159 CHECK(payload[
"elements"].IsArray());
3162 if (payload.HasMember(
"temporary")) {
3167 if (payload.HasMember(
"ifNotExists")) {
3178 std::list<ColumnDescriptor>& columns,
3179 std::vector<SharedDictionaryDef>& shared_dict_defs) {
3180 std::unordered_set<std::string> uc_col_names;
3184 if (dynamic_cast<SharedDictionaryDef*>(e.get())) {
3187 this, shared_dict_def, columns, shared_dict_defs, catalog);
3188 shared_dict_defs.push_back(*shared_dict_def);
3191 if (dynamic_cast<ShardKeyDef*>(e.get())) {
3192 if (shard_key_def) {
3193 throw std::runtime_error(
"Specified more than one shard key");
3195 shard_key_def =
static_cast<const ShardKeyDef*
>(e.get());
3198 if (!dynamic_cast<ColumnDef*>(e.get())) {
3199 throw std::runtime_error(
"Table constraints are not supported yet.");
3206 columns.push_back(cd);
3211 if (shard_key_def) {
3214 throw std::runtime_error(
"Specified shard column " + shard_key_def->get_column() +
3229 throw std::runtime_error(
"SHARD_COUNT needs to be specified with SHARD_KEY.");
3237 const auto execute_write_lock = mapd_unique_lock<mapd_shared_mutex>(
3244 throw std::runtime_error(
"Table " + *
table_ +
3245 " will not be created. User has no create privileges.");
3253 std::list<ColumnDescriptor> columns;
3254 std::vector<SharedDictionaryDef> shared_dict_defs;
3259 catalog.createShardedTable(td, columns, shared_dict_defs);
3262 SysCatalog::instance().createDBObject(
3267 CHECK(payload.HasMember(
"name"));
3268 table_ = std::make_unique<std::string>(
json_str(payload[
"name"]));
3270 CHECK(payload.HasMember(
"elementList"));
3273 CHECK(payload.HasMember(
"filePath"));
3274 std::string fs =
json_str(payload[
"filePath"]);
3276 boost::algorithm::trim_if(fs, boost::is_any_of(
" \"'`"));
3277 filename_ = std::make_unique<std::string>(fs);
3285 const auto execute_write_lock = mapd_unique_lock<mapd_shared_mutex>(
3292 throw std::runtime_error(
"Table " + *
table_ +
3293 " will not be created. User has no create privileges.");
3296 if (catalog.getMetadataForTable(*
table_) !=
nullptr) {
3297 throw std::runtime_error(
"Table " + *
table_ +
" already exists.");
3300 std::list<ColumnDescriptor> columns;
3301 std::vector<SharedDictionaryDef> shared_dict_defs;
3303 std::unordered_set<std::string> uc_col_names;
3305 if (dynamic_cast<SharedDictionaryDef*>(e.get())) {
3308 this, shared_dict_def, columns, shared_dict_defs, catalog);
3309 shared_dict_defs.push_back(*shared_dict_def);
3312 if (!dynamic_cast<ColumnDef*>(e.get())) {
3313 throw std::runtime_error(
"Table constraints are not supported yet.");
3318 const auto uc_col_name = boost::to_upper_copy<std::string>(cd.
columnName);
3319 const auto it_ok = uc_col_names.insert(uc_col_name);
3320 if (!it_ok.second) {
3321 throw std::runtime_error(
"Column '" + cd.
columnName +
"' defined more than once");
3324 columns.push_back(cd);
3346 catalog.createShardedTable(df_td, columns, shared_dict_defs);
3349 SysCatalog::instance().createDBObject(
3354 const std::string select_stmt,
3355 std::vector<TargetMetaInfo>& targets,
3356 bool validate_only =
false,
3357 std::vector<size_t> outer_fragment_indices = {},
3358 bool allow_interrupt =
false) {
3360 auto& catalog = session->getCatalog();
3364 const auto device_type = session->get_executor_device_type();
3368 auto calcite_mgr = catalog.getCalciteMgr();
3372 const auto calciteQueryParsingOption =
3373 calcite_mgr->getCalciteQueryParsingOption(
true,
false,
true);
3374 const auto calciteOptimizationOption =
3376 const auto query_ra = calcite_mgr
3377 ->process(query_state_proxy,
3379 calciteQueryParsingOption,
3380 calciteOptimizationOption)
3406 outer_fragment_indices};
3416 result = ra_executor.executeRelAlgQuery(co, eo,
false,
nullptr);
3417 targets =
result.getTargetsMeta();
3423 std::string& sql_query_string) {
3425 auto& catalog = session->getCatalog();
3429 const auto device_type = session->get_executor_device_type();
3433 auto calcite_mgr = catalog.getCalciteMgr();
3437 const auto calciteQueryParsingOption =
3438 calcite_mgr->getCalciteQueryParsingOption(
true,
false,
true);
3439 const auto calciteOptimizationOption =
3441 const auto query_ra = calcite_mgr
3442 ->process(query_state_proxy,
3444 calciteQueryParsingOption,
3445 calciteOptimizationOption)
3465 return ra_executor.getOuterFragmentCount(co, eo);
3469 std::string& sql_query_string,
3470 std::vector<size_t> outer_frag_indices,
3472 bool allow_interrupt) {
3474 std::string pg_shimmed_select_query =
pg_shim(sql_query_string);
3476 std::vector<TargetMetaInfo> target_metainfos;
3479 auto query_session = session ? session->get_session_id() :
"";
3481 if (allow_interrupt && !validate_only && !query_session.empty()) {
3482 executor->enrollQuerySession(query_session,
3484 query_submitted_time,
3486 QuerySessionStatus::QueryStatus::PENDING_EXECUTOR);
3500 std::string& sql_query_string,
3501 std::vector<size_t> outer_frag_indices,
3502 bool allow_interrupt) {
3504 query_state_proxy, sql_query_string, outer_frag_indices,
false, allow_interrupt);
3511 std::list<ColumnDescriptor> column_descriptors;
3512 std::list<ColumnDescriptor> column_descriptors_for_create;
3514 int rowid_suffix = 0;
3515 for (
const auto& target_metainfo : result.
targets_meta) {
3517 cd.
columnName = target_metainfo.get_resname();
3521 cd.
columnType = target_metainfo.get_physical_type_info();
3541 column_descriptors_for_create.push_back(cd_for_create);
3542 column_descriptors.push_back(cd);
3546 return column_descriptors_for_create;
3549 return column_descriptors;
3553 const rapidjson::Value& payload) {
3554 CHECK(payload.HasMember(
"name"));
3557 CHECK(payload.HasMember(
"query"));
3560 boost::replace_all(select_query_,
"\n",
" ");
3561 select_query_ =
"(" + select_query_ +
")";
3563 if (payload.HasMember(
"columns")) {
3564 CHECK(payload[
"columns"].IsArray());
3565 for (
auto& column : payload[
"columns"].GetArray()) {
3567 column_list_.emplace_back(std::unique_ptr<std::string>(
new std::string(s)));
3574 bool validate_table,
3577 auto& catalog = session->getCatalog();
3581 bool populate_table =
false;
3584 populate_table =
true;
3588 populate_table =
true;
3592 auto get_target_column_descriptors = [
this, &catalog](
const TableDescriptor* td) {
3593 std::vector<const ColumnDescriptor*> target_column_descriptors;
3595 auto list = catalog.getAllColumnMetadataForTable(td->
tableId,
false,
false,
false);
3596 target_column_descriptors = {std::begin(list), std::end(list)};
3600 if (cd ==
nullptr) {
3601 throw std::runtime_error(
"Column " + *c +
" does not exist.");
3603 target_column_descriptors.push_back(cd);
3607 return target_column_descriptors;
3612 if (validate_table) {
3615 throw std::runtime_error(
"Table " +
table_name_ +
" does not exist.");
3618 throw std::runtime_error(
"Insert to views is not supported yet.");
3624 throw std::runtime_error(
"User has no insert privileges on " +
table_name_ +
".");
3632 std::vector<const ColumnDescriptor*> target_column_descriptors =
3633 get_target_column_descriptors(td);
3635 if (source_column_descriptors.size() != target_column_descriptors.size()) {
3636 throw std::runtime_error(
"The number of source and target columns does not match.");
3639 for (
int i = 0; i < source_column_descriptors.size(); i++) {
3641 &(*std::next(source_column_descriptors.begin(), i));
3645 auto type_cannot_be_cast = [](
const auto& col_type) {
3646 return (col_type.is_time() || col_type.is_geometry() || col_type.is_array() ||
3647 col_type.is_boolean());
3650 if (type_cannot_be_cast(source_cd->
columnType) ||
3651 type_cannot_be_cast(target_cd->
columnType)) {
3652 throw std::runtime_error(
"Source '" + source_cd->
columnName +
" " +
3654 "' and target '" + target_cd->
columnName +
" " +
3656 "' column types do not match.");
3661 throw std::runtime_error(
"Source '" + source_cd->
columnName +
" " +
3663 "' and target '" + target_cd->
columnName +
" " +
3665 "' array column element types do not match.");
3680 throw std::runtime_error(
"Source '" + source_cd->
columnName +
" " +
3682 "' and target '" + target_cd->
columnName +
" " +
3684 "' decimal columns scales do not match.");
3690 throw std::runtime_error(
"Source '" + source_cd->
columnName +
" " +
3692 "' and target '" + target_cd->
columnName +
" " +
3694 "' column types do not match.");
3698 throw std::runtime_error(
"Source '" + source_cd->
columnName +
" " +
3700 "' and target '" + target_cd->
columnName +
" " +
3702 "' columns string encodings do not match.");
3709 throw std::runtime_error(
"Source '" + source_cd->
columnName +
" " +
3711 "' and target '" + target_cd->
columnName +
" " +
3713 "' timestamp column precisions do not match.");
3722 throw std::runtime_error(
"Source '" + source_cd->
columnName +
" " +
3724 "' and target '" + target_cd->
columnName +
" " +
3726 "' column encoding sizes do not match.");
3731 if (!populate_table) {
3735 int64_t total_row_count = 0;
3736 int64_t total_source_query_time_ms = 0;
3737 int64_t total_target_value_translate_time_ms = 0;
3738 int64_t total_data_load_time_ms = 0;
3741 auto target_column_descriptors = get_target_column_descriptors(td);
3742 auto outer_frag_count =
3745 size_t outer_frag_end = outer_frag_count == 0 ? 1 : outer_frag_count;
3746 auto query_session = session ? session->get_session_id() :
"";
3748 std::string work_type_str = for_CTAS ?
"CTAS" :
"ITAS";
3750 for (
size_t outer_frag_idx = 0; outer_frag_idx < outer_frag_end; outer_frag_idx++) {
3751 std::vector<size_t> allowed_outer_fragment_indices;
3753 if (outer_frag_count) {
3754 allowed_outer_fragment_indices.push_back(outer_frag_idx);
3758 std::vector<AggregatedResult> query_results =
3761 allowed_outer_fragment_indices,
3763 total_source_query_time_ms +=
timer_stop(query_clock_begin);
3766 auto query_str =
"INSERT_DATA for " + work_type_str;
3771 executor->enrollQuerySession(query_session,
3775 QuerySessionStatus::QueryStatus::RUNNING_IMPORTER);
3782 executor->clearQuerySessionStatus(query_session,
start_time);
3786 for (
auto&
res : query_results) {
3788 throw std::runtime_error(
3789 "Query execution has been interrupted while performing " + work_type_str);
3791 auto& result_rows =
res.rs;
3793 const auto num_rows = result_rows->rowCount();
3795 if (0 == num_rows) {
3799 total_row_count += num_rows;
3804 const size_t rows_per_block =
3805 std::max(std::min(num_rows / leaf_count,
size_t(64 * 1024)),
size_t(1));
3807 std::vector<std::unique_ptr<TargetValueConverter>> value_converters;
3811 const int num_worker_threads = std::thread::hardware_concurrency();
3813 std::vector<size_t> thread_start_idx(num_worker_threads),
3814 thread_end_idx(num_worker_threads);
3815 bool can_go_parallel = !result_rows->isTruncated() && rows_per_block > 20000;
3817 std::atomic<size_t> crt_row_idx{0};
3819 auto do_work = [&result_rows, &value_converters, &crt_row_idx](
3821 const size_t block_end,
3822 const size_t num_cols,
3824 bool& stop_convert) {
3825 const auto result_row = result_rows->getRowAtNoTranslations(idx);
3826 if (!result_row.empty()) {
3827 size_t target_row = crt_row_idx.fetch_add(1);
3828 if (target_row >= block_end) {
3829 stop_convert =
true;
3832 for (
unsigned int col = 0; col < num_cols; col++) {
3833 const auto& mapd_variant = result_row[col];
3834 value_converters[col]->convertToColumnarFormat(target_row, &mapd_variant);
3839 auto convert_function = [&thread_start_idx,
3845 &do_work](
const int thread_id,
const size_t block_end) {
3846 const int num_cols = value_converters.size();
3847 const size_t start = thread_start_idx[
thread_id];
3848 const size_t end = thread_end_idx[
thread_id];
3850 bool stop_convert =
false;
3852 size_t local_idx = 0;
3853 for (idx = start; idx < end; ++idx, ++local_idx) {
3854 if (
UNLIKELY((local_idx & 0xFFFF) == 0 &&
3856 throw std::runtime_error(
3857 "Query execution has been interrupted while performing " +
3860 do_work(idx, block_end, num_cols, thread_id, stop_convert);
3866 for (idx = start; idx < end; ++idx) {
3867 do_work(idx, block_end, num_cols, thread_id, stop_convert);
3876 auto single_threaded_value_converter =
3877 [&crt_row_idx, &value_converters, &result_rows](
const size_t idx,
3878 const size_t block_end,
3879 const size_t num_cols,
3880 bool& stop_convert) {
3881 size_t target_row = crt_row_idx.fetch_add(1);
3882 if (target_row >= block_end) {
3883 stop_convert =
true;
3886 const auto result_row = result_rows->getNextRow(
false,
false);
3887 CHECK(!result_row.empty());
3888 for (
unsigned int col = 0; col < num_cols; col++) {
3889 const auto& mapd_variant = result_row[col];
3890 value_converters[col]->convertToColumnarFormat(target_row, &mapd_variant);
3894 auto single_threaded_convert_function = [&value_converters,
3900 &single_threaded_value_converter](
3902 const size_t block_end) {
3903 const int num_cols = value_converters.size();
3904 const size_t start = thread_start_idx[
thread_id];
3905 const size_t end = thread_end_idx[
thread_id];
3907 bool stop_convert =
false;
3909 size_t local_idx = 0;
3910 for (idx = start; idx < end; ++idx, ++local_idx) {
3911 if (
UNLIKELY((local_idx & 0xFFFF) == 0 &&
3913 throw std::runtime_error(
3914 "Query execution has been interrupted while performing " +
3917 single_threaded_value_converter(idx, block_end, num_cols, stop_convert);
3923 for (idx = start; idx < end; ++idx) {
3924 single_threaded_value_converter(idx, end, num_cols, stop_convert);
3933 if (can_go_parallel) {
3934 const size_t entry_count = result_rows->entryCount();
3938 stride = (entry_count + num_worker_threads - 1) / num_worker_threads;
3939 i < num_worker_threads && start_entry < entry_count;
3940 ++i, start_entry += stride) {
3941 const auto end_entry = std::min(start_entry + stride, entry_count);
3942 thread_start_idx[i] = start_entry;
3943 thread_end_idx[i] = end_entry;
3946 thread_start_idx[0] = 0;
3947 thread_end_idx[0] = result_rows->entryCount();
3952 for (
size_t block_start = 0; block_start < num_rows;
3953 block_start += rows_per_block) {
3954 const auto num_rows_this_itr = block_start + rows_per_block < num_rows
3956 : num_rows - block_start;
3958 value_converters.clear();
3960 for (
const auto targetDescriptor : target_column_descriptors) {
3961 auto sourceDataMetaInfo =
res.targets_meta[colNum++];
3967 targetDescriptor->columnType,
3968 !targetDescriptor->columnType.get_notnull(),
3969 result_rows->getRowSetMemOwner()->getLiteralStringDictProxy(),
3971 sourceDataMetaInfo.get_type_info().is_dict_encoded_string()
3972 ? executor->getStringDictionaryProxy(
3973 sourceDataMetaInfo.get_type_info().get_comp_param(),
3974 result_rows->getRowSetMemOwner(),
3977 IS_GEO_POLY(targetDescriptor->columnType.get_type()) &&
3979 ? &render_group_analyzer_map
3981 auto converter = factory.
create(param);
3982 value_converters.push_back(std::move(converter));
3986 if (can_go_parallel) {
3987 std::vector<std::future<void>> worker_threads;
3988 for (
int i = 0; i < num_worker_threads; ++i) {
3989 worker_threads.push_back(
3993 for (
auto& child : worker_threads) {
3996 for (
auto& child : worker_threads) {
4001 single_threaded_convert_function(0, num_rows_this_itr);
4005 auto finalizer_func =
4006 [](std::unique_ptr<TargetValueConverter>::pointer targetValueConverter) {
4007 targetValueConverter->finalizeDataBlocksForInsertData();
4010 std::vector<std::future<void>> worker_threads;
4011 for (
auto& converterPtr : value_converters) {
4012 worker_threads.push_back(
4016 for (
auto& child : worker_threads) {
4019 for (
auto& child : worker_threads) {
4024 insert_data.
databaseId = catalog.getCurrentDB().dbId;
4027 insert_data.
numRows = num_rows_this_itr;
4029 for (
int col_idx = 0; col_idx < target_column_descriptors.size(); col_idx++) {
4032 throw std::runtime_error(
4033 "Query execution has been interrupted while performing " +
4036 value_converters[col_idx]->addDataBlocksToInsertData(insert_data);
4038 total_target_value_translate_time_ms +=
timer_stop(translate_clock_begin);
4041 auto data_memory_holder =
4043 insertDataLoader.
insertData(*session, insert_data);
4044 total_data_load_time_ms +=
timer_stop(data_load_clock_begin);
4051 }
catch (std::exception& e) {
4052 LOG(
ERROR) <<
"An error occurred during ITAS rollback attempt. Table id: "
4053 << td->
tableId <<
", Error: " << e.what();
4058 int64_t total_time_ms = total_source_query_time_ms +
4059 total_target_value_translate_time_ms + total_data_load_time_ms;
4061 VLOG(1) <<
"CTAS/ITAS " << total_row_count <<
" rows loaded in " << total_time_ms
4062 <<
"ms (outer_frag_count=" << outer_frag_count
4063 <<
", query_time=" << total_source_query_time_ms
4064 <<
"ms, translation_time=" << total_target_value_translate_time_ms
4065 <<
"ms, data_load_time=" << total_data_load_time_ms
4068 if (!is_temporary) {
4075 const std::string& table_name) {
4076 auto table_id = catalog.
getTableId(table_name);
4077 if (!table_id.has_value()) {
4078 throw std::runtime_error{
"Table \"" + table_name +
"\" does not exist."};
4080 return table_id.value();
4085 const std::string& query_str,
4087 const std::optional<std::string>& insert_table_name = {}) {
4089 const auto calciteQueryParsingOption =
4090 calcite_mgr->getCalciteQueryParsingOption(
true,
false,
true);
4091 const auto calciteOptimizationOption =
4093 const auto result = calcite_mgr->process(query_state_proxy,
4095 calciteQueryParsingOption,
4096 calciteOptimizationOption);
4099 auto comparator = [&catalog](
const std::string& table_1,
const std::string& table_2) {
4102 std::set<std::string, decltype(comparator)>
tables(comparator);
4103 for (
auto& tab :
result.resolved_accessed_objects.tables_selected_from) {
4106 if (insert_table_name.has_value()) {
4107 tables.emplace(insert_table_name.value());
4110 for (
const auto& table :
tables) {
4115 if (insert_table_name.has_value() && table == insert_table_name.value()) {
4132 auto session_copy = session;
4133 auto session_ptr = std::shared_ptr<Catalog_Namespace::SessionInfo>(
4134 &session_copy, boost::null_deleter());
4136 auto stdlog =
STDLOG(query_state);
4137 auto& catalog = session_ptr->getCatalog();
4139 const auto execute_read_lock = mapd_shared_lock<mapd_shared_mutex>(
4144 throw std::runtime_error(
"ITAS failed: table " +
table_name_ +
" does not exist.");
4154 populateData(query_state->createQueryStateProxy(), td,
true,
false);
4162 if (payload.HasMember(
"temporary")) {
4168 if (payload.HasMember(
"ifNotExists")) {
4178 auto session_copy = session;
4179 auto session_ptr = std::shared_ptr<Catalog_Namespace::SessionInfo>(
4180 &session_copy, boost::null_deleter());
4182 auto stdlog =
STDLOG(query_state);
4187 std::set<std::string> select_tables;
4189 const auto execute_write_lock = mapd_unique_lock<mapd_shared_mutex>(
4196 throw std::runtime_error(
"CTAS failed. Table " +
table_name_ +
4197 " will not be created. User has no create privileges.");
4205 " already exists and no data was loaded.");
4211 auto validate_result = local_connector.
query(
4212 query_state->createQueryStateProxy(),
select_query_, {},
true,
false);
4214 const auto column_descriptors_for_create =
4218 for (
auto& cd : column_descriptors_for_create) {
4219 if (cd.columnType.is_decimal() && cd.columnType.get_precision() > 18) {
4220 throw std::runtime_error(cd.columnName +
": Precision too high, max 18.");
4227 td.
nColumns = column_descriptors_for_create.size();
4242 bool use_shared_dictionaries =
true;
4246 if (boost::to_lower_copy<std::string>(*p->get_name()) ==
4247 "use_shared_dictionaries") {
4250 if (
nullptr == literal) {
4251 throw std::runtime_error(
4252 "USE_SHARED_DICTIONARIES must be a string parameter");
4254 std::string val = boost::to_lower_copy<std::string>(*literal->
get_stringval());
4255 use_shared_dictionaries = val ==
"true" || val ==
"1" || val ==
"t";
4262 std::vector<SharedDictionaryDef> sharedDictionaryRefs;
4264 if (use_shared_dictionaries) {
4265 const auto source_column_descriptors =
4269 for (
auto& source_cd : source_column_descriptors) {
4270 const auto& ti = source_cd.columnType;
4271 if (ti.is_string()) {
4273 int dict_id = ti.get_comp_param();
4274 auto it = mapping.find(dict_id);
4275 if (mapping.end() != it) {
4276 const auto targetColumn = it->second;
4280 LOG(
INFO) <<
"CTAS: sharing text dictionary on column "
4281 << source_cd.columnName <<
" with " << targetTable->tableName
4282 <<
"." << targetColumn->columnName;
4283 sharedDictionaryRefs.emplace_back(
4284 source_cd.columnName, targetTable->tableName, targetColumn->columnName);
4294 catalog.
createTable(td, column_descriptors_for_create, sharedDictionaryRefs,
true);
4297 SysCatalog::instance().createDBObject(
4303 const auto execute_read_lock = mapd_shared_lock<mapd_shared_mutex>(
4311 populateData(query_state->createQueryStateProxy(), td,
false,
true);
4324 CHECK(payload.HasMember(
"tableName"));
4325 table_ = std::make_unique<std::string>(
json_str(payload[
"tableName"]));
4328 if (payload.HasMember(
"ifExists")) {
4329 if_exists_ =
json_bool(payload[
"ifExists"]);
4334 const auto execute_read_lock = mapd_shared_lock<mapd_shared_mutex>(
4339 std::unique_ptr<lockmgr::TableSchemaLockContainer<lockmgr::WriteLock>> td_with_lock;
4342 std::make_unique<lockmgr::TableSchemaLockContainer<lockmgr::WriteLock>>(
4344 catalog, *
table_,
false));
4345 td = (*td_with_lock)();
4346 }
catch (
const std::runtime_error& e) {
4355 CHECK(td_with_lock);
4360 throw std::runtime_error(
"Table " + *
table_ +
4361 " will not be dropped. User has no proper privileges.");
4367 auto table_data_read_lock =
4372 auto table_data_write_lock =
4374 catalog.dropTable(td);
4380 CHECK(payload.HasMember(
"tableName"));
4381 auto tableName =
json_str(payload[
"tableName"]);
4383 CHECK(payload.HasMember(
"alterType"));
4386 if (type ==
"RENAME_TABLE") {
4387 CHECK(payload.HasMember(
"newTableName"));
4388 auto newTableName =
json_str(payload[
"newTableName"]);
4390 new std::string(tableName),
new std::string(newTableName)));
4392 }
else if (type ==
"RENAME_COLUMN") {
4393 CHECK(payload.HasMember(
"columnName"));
4394 auto columnName =
json_str(payload[
"columnName"]);
4395 CHECK(payload.HasMember(
"newColumnName"));
4396 auto newColumnName =
json_str(payload[
"newColumnName"]);
4397 return std::unique_ptr<DDLStmt>(
4399 new std::string(columnName),
4400 new std::string(newColumnName)));
4402 }
else if (type ==
"ADD_COLUMN") {
4403 CHECK(payload.HasMember(
"columnData"));
4404 CHECK(payload[
"columnData"].IsArray());
4407 std::list<ColumnDef*>* table_element_list_ =
new std::list<ColumnDef*>;
4409 const auto elements = payload[
"columnData"].GetArray();
4410 for (
const auto& element : elements) {
4411 CHECK(element.IsObject());
4412 CHECK(element.HasMember(
"type"));
4413 if (
json_str(element[
"type"]) ==
"SQL_COLUMN_DECLARATION") {
4415 table_element_list_->emplace_back(col_def.release());
4417 LOG(
FATAL) <<
"Unsupported element type for ALTER TABLE: "
4418 << element[
"type"].GetString();
4422 return std::unique_ptr<DDLStmt>(
4425 }
else if (type ==
"DROP_COLUMN") {
4426 CHECK(payload.HasMember(
"columnData"));
4427 auto columnData =
json_str(payload[
"columnData"]);
4430 std::list<std::string*>* cols =
new std::list<std::string*>;
4431 std::vector<std::string> cols1;
4432 boost::split(cols1, columnData, boost::is_any_of(
","));
4433 for (
auto s : cols1) {
4435 boost::algorithm::trim_if(s, boost::is_any_of(
" \"'`"));
4436 std::string* str =
new std::string(s);
4437 cols->emplace_back(str);
4440 return std::unique_ptr<DDLStmt>(
4443 }
else if (type ==
"ALTER_OPTIONS") {
4444 CHECK(payload.HasMember(
"options"));
4446 if (payload[
"options"].IsObject()) {
4447 for (
const auto& option : payload[
"options"].GetObject()) {
4448 std::string* option_name =
new std::string(
json_str(option.name));
4450 if (option.value.IsString()) {
4451 std::string literal_string =
json_str(option.value);
4456 int iVal = std::stoi(literal_string, &sz);
4457 if (sz == literal_string.size()) {
4462 }
else if (option.value.IsInt() || option.value.IsInt64()) {
4464 }
else if (option.value.IsNull()) {
4467 throw std::runtime_error(
"Unable to handle literal for " + *option_name);
4469 CHECK(literal_value);
4472 return std::unique_ptr<DDLStmt>(
4477 CHECK(payload[
"options"].IsNull());
4484 CHECK(payload.HasMember(
"tableName"));
4485 table_ = std::make_unique<std::string>(
json_str(payload[
"tableName"]));
4489 const auto execute_read_lock = mapd_shared_lock<mapd_shared_mutex>(
4493 const auto td_with_lock =
4496 const auto td = td_with_lock();
4498 throw std::runtime_error(
"Table " + *
table_ +
" does not exist.");
4502 std::vector<DBObject> privObjects;
4506 privObjects.push_back(dbObject);
4507 if (!SysCatalog::instance().checkPrivileges(session.
get_currentUser(), privObjects)) {
4508 throw std::runtime_error(
"Table " + *
table_ +
" will not be truncated. User " +
4510 " has no proper privileges.");
4514 throw std::runtime_error(*
table_ +
" is a view. Cannot Truncate.");
4520 auto table_data_read_lock =
4525 auto table_data_write_lock =
4527 catalog.truncateTable(td);
4531 CHECK(payload.HasMember(
"tableName"));
4532 table_ = std::make_unique<std::string>(
json_str(payload[
"tableName"]));
4542 std::vector<DBObject> privObjects;
4546 privObjects.push_back(dbObject);
4547 return SysCatalog::instance().checkPrivileges(session_info.
get_currentUser(),
4555 const auto execute_read_lock = mapd_shared_lock<mapd_shared_mutex>(
4559 const auto td_with_lock =
4562 const auto td = td_with_lock();
4565 throw std::runtime_error(
"Table " + *
table_ +
" does not exist.");
4569 throw std::runtime_error(
"OPTIMIZE TABLE command is not supported on views.");
4584 bool repair_type(std::list<std::unique_ptr<NameValueAssign>>& options) {
4585 for (
const auto& opt : options) {
4586 if (boost::iequals(*opt->get_name(),
"REPAIR_TYPE")) {
4588 static_cast<const StringLiteral*
>(opt->get_value())->get_stringval();
4593 throw std::runtime_error(
"REPAIR_TYPE must be REMOVE.");
4596 throw std::runtime_error(
"The only VALIDATE WITH options is REPAIR_TYPE.");
4605 throw std::runtime_error(
"Validation Type is required for VALIDATE command.");
4607 std::list<std::unique_ptr<NameValueAssign>> options;
4609 for (
const auto e : *with_opts) {
4610 options.emplace_back(e);
4619 CHECK(payload.HasMember(
"type"));
4620 type_ = std::make_unique<std::string>(
json_str(payload[
"type"]));
4622 std::list<std::unique_ptr<NameValueAssign>> options;
4634 std::vector<DBObject> privObjects;
4638 privObjects.push_back(dbObject);
4639 if (!SysCatalog::instance().checkPrivileges(session.
get_currentUser(), privObjects)) {
4640 throw std::runtime_error(
"Current user does not have the privilege to alter table: " +
4646 CHECK(payload.HasMember(
"name"));
4648 CHECK(payload.HasMember(
"newName"));
4654 throw std::runtime_error(
"Only a super user can rename users.");
4658 if (!SysCatalog::instance().getMetadataForUser(*
username_, user)) {
4659 throw std::runtime_error(
"User " + *
username_ +
" does not exist.");
4666 CHECK(payload.HasMember(
"name"));
4668 CHECK(payload.HasMember(
"newName"));
4676 const auto execute_write_lock = mapd_unique_lock<mapd_shared_mutex>(
4680 if (!SysCatalog::instance().getMetadataForDB(*
database_name_, db)) {
4681 throw std::runtime_error(
"Database " + *
database_name_ +
" does not exist.");
4686 throw std::runtime_error(
"Only a super user or the owner can rename the database.");
4693 CHECK(payload.HasMember(
"tableNames"));
4694 CHECK(payload[
"tableNames"].IsArray());
4695 const auto elements = payload[
"tableNames"].GetArray();
4696 for (
const auto& element : elements) {
4697 CHECK(element.HasMember(
"name"));
4698 CHECK(element.HasMember(
"newName"));
4700 new std::string(
json_str(element[
"newName"])));
4709 std::list<std::pair<std::string, std::string>> tableNames) {
4710 for (
auto item : tableNames) {
4712 new std::string(item.second));
4727 std::time_t
result = std::time(
nullptr);
4732 sMap[oldName] = newName;
4737 std::string tableName) {
4738 if (sMap.find(tableName) != sMap.end()) {
4742 return sMap[tableName];
4747 sMap[tableName] = tableName;
4764 for (
auto it : sMap) {
4765 if ((it.second) !=
EMPTY_NAME && (it.first) != (it.second)) {
4766 throw std::runtime_error(
4767 "Error: Attempted to overwrite and lose data in table: \'" + (it.first) +
"\'");
4776 throw std::runtime_error(td->
tableName +
" is a foreign table. " +
4777 "Use ALTER FOREIGN TABLE.");
4786 const auto execute_write_lock = mapd_unique_lock<mapd_shared_mutex>(
4791 std::vector<std::pair<std::string, std::string>> names;
4796 std::string curTableName = *(item.first);
4797 std::string newTableName = *(item.second);
4804 std::string altCurTableName =
loadTable(catalog, tableSubtituteMap, curTableName);
4805 std::string altNewTableName =
loadTable(catalog, tableSubtituteMap, newTableName);
4807 if (altCurTableName != curTableName && altCurTableName !=
EMPTY_NAME) {
4809 recordRename(tableSubtituteMap, curTableName, curTableName);
4820 if (
hasData(tableSubtituteMap, altCurTableName)) {
4829 if (
hasData(tableSubtituteMap, altNewTableName)) {
4834 recordRename(tableSubtituteMap, altNewTableName, tmpNewTableName);
4835 recordRename(tableSubtituteMap, tmpNewTableName, tmpNewTableName);
4836 names.emplace_back(altNewTableName, tmpNewTableName);
4837 names.emplace_back(altCurTableName, altNewTableName);
4841 recordRename(tableSubtituteMap, altNewTableName, altNewTableName);
4842 names.emplace_back(altCurTableName, altNewTableName);
4845 throw std::runtime_error(
"Source table \'" + curTableName +
"\' does not exist.");
4853 while (!tablesToRename_.empty()) {
4854 tablesToRename_.pop_front();
4861 if (cc ==
nullptr) {
4866 std::string default_value;
4867 const std::string* default_value_ptr =
nullptr;
4870 auto defaultsp =
dynamic_cast<const StringLiteral*
>(def_val_literal);
4872 defaultsp ? *defaultsp->
get_stringval() : def_val_literal->to_string();
4877 std::regex array_re(R
"(^ARRAY\s*\[(.*)\]$)", std::regex_constants::icase);
4878 default_value = std::regex_replace(default_value, array_re, "{$1}");
4879 boost::erase_all(default_value,
"\'");
4881 default_value_ptr = &default_value;
4896 throw std::runtime_error(
"Table " + *
table_ +
" does not exist.");
4899 throw std::runtime_error(
"Adding columns to a view is not supported.");
4903 throw std::runtime_error(
4904 "Adding columns to temporary tables is not yet supported.");
4914 for (
const auto& coldef :
coldefs_) {
4915 auto& new_column_name = *coldef->get_column_name();
4917 throw std::runtime_error(
"Column " + new_column_name +
" already exists.");
4924 const auto execute_write_lock = mapd_unique_lock<mapd_shared_mutex>(
4928 const auto td_with_lock =
4931 const auto td = td_with_lock();
4936 if (std::dynamic_pointer_cast<Fragmenter_Namespace::SortedOrderFragmenter>(
4938 throw std::runtime_error(
4939 "Adding columns to a table is not supported when using the \"sort_column\" "
4944 std::vector<int> table_key{catalog.getCurrentDB().dbId, td->
tableId};
4951 catalog.getSqliteConnector().query(
"BEGIN TRANSACTION");
4953 std::map<const std::string, const ColumnDescriptor> cds;
4954 std::map<const int, const ColumnDef*> cid_coldefs;
4955 for (
const auto& coldef :
coldefs_) {
4958 catalog.addColumn(*td, cd);
4959 cds.emplace(*coldef->get_column_name(), cd);
4960 cid_coldefs.emplace(cd.
columnId, coldef.get());
4964 std::list<ColumnDescriptor> phy_geo_columns;
4965 catalog.expandGeoColumn(cd, phy_geo_columns);
4966 for (
auto& cd : phy_geo_columns) {
4967 catalog.addColumn(*td, cd);
4969 cid_coldefs.emplace(cd.
columnId,
nullptr);
4975 std::vector<std::unique_ptr<import_export::TypedImportBuffer>> import_buffers;
4976 for (
const auto& cd : cds) {
4977 import_buffers.emplace_back(std::make_unique<import_export::TypedImportBuffer>(
4978 &cd.second, loader->getStringDict(&cd.second)));
4980 loader->setAddingColumns(
true);
4984 import_buffers.end(),
4985 [](decltype(import_buffers[0])&
a, decltype(import_buffers[0])& b) {
4986 return a->getColumnDesc()->columnId < b->getColumnDesc()->columnId;
4992 const auto physical_tds = catalog.getPhysicalTablesDescriptors(td);
4994 std::for_each(physical_tds.begin(), physical_tds.end(), [&nrows](
const auto& td) {
4999 int skip_physical_cols = 0;
5000 for (
const auto cit : cid_coldefs) {
5001 const auto cd = catalog.getMetadataForColumn(td->
tableId, cit.first);
5002 const auto coldef = cit.second;
5003 const bool is_null = !cd->default_value.has_value();
5005 if (cd->columnType.get_notnull() &&
is_null) {
5006 throw std::runtime_error(
"Default value required for column " + cd->columnName +
5007 " because of NOT NULL constraint");
5010 for (
auto it = import_buffers.begin(); it < import_buffers.end(); ++it) {
5011 auto& import_buffer = *it;
5012 if (cd->columnId == import_buffer->getColumnDesc()->columnId) {
5013 if (coldef !=
nullptr ||
5014 skip_physical_cols-- <= 0) {
5015 import_buffer->add_value(cd,
5016 cd->default_value.value_or(
"NULL"),
5019 if (cd->columnType.is_geometry()) {
5020 std::vector<double> coords, bounds;
5021 std::vector<int> ring_sizes, poly_rings;
5022 int render_group = 0;
5025 cd->default_value.value_or(
"NULL"),
5032 throw std::runtime_error(
"Bad geometry data: '" +
5033 cd->default_value.value_or(
"NULL") +
"'");
5035 size_t col_idx = 1 + std::distance(import_buffers.begin(), it);
5046 skip_physical_cols = cd->columnType.get_physical_cols();
5055 if (!loader->loadNoCheckpoint(import_buffers, nrows, &session)) {
5056 throw std::runtime_error(
"loadNoCheckpoint failed!");
5059 catalog.resetTableEpochFloor(td->
tableId);
5060 loader->checkpoint();
5061 catalog.getSqliteConnector().query(
"END TRANSACTION");
5063 catalog.roll(
false);
5064 catalog.getSqliteConnector().query(
"ROLLBACK TRANSACTION");
5071 const auto execute_write_lock = mapd_unique_lock<mapd_shared_mutex>(
5075 const auto td_with_lock =
5078 const auto td = td_with_lock();
5080 throw std::runtime_error(
"Table " + *
table_ +
" does not exist.");
5084 throw std::runtime_error(
"Dropping a column from a view is not supported.");
5087 throw std::runtime_error(
5088 "Dropping a column from a temporary table is not yet supported.");
5093 for (
const auto& column :
columns_) {
5094 if (
nullptr == catalog.getMetadataForColumn(td->
tableId, *column)) {
5095 throw std::runtime_error(
"Column " + *column +
" does not exist.");
5100 throw std::runtime_error(
"Table " + *
table_ +
" has only one column.");
5106 catalog.getSqliteConnector().query(
"BEGIN TRANSACTION");
5108 std::vector<int> columnIds;
5109 for (
const auto& column : columns_) {
5112 throw std::runtime_error(
"Dropping sharding column " + cd.
columnName +
5113 " is not supported.");
5115 catalog.dropColumn(*td, cd);
5118 const auto pcd = catalog.getMetadataForColumn(td->
tableId, cd.
columnId + i + 1);
5120 catalog.dropColumn(*td, *pcd);
5121 columnIds.push_back(cd.
columnId + i + 1);
5125 for (
auto shard : catalog.getPhysicalTablesDescriptors(td)) {
5126 shard->fragmenter->dropColumns(columnIds);
5130 throw std::runtime_error(
"lol!");
5134 catalog.resetTableEpochFloor(td->
tableId);
5135 catalog.checkpoint(td->
tableId);
5137 catalog.getSqliteConnector().query(
"END TRANSACTION");
5139 catalog.setForReload(td->
tableId);
5140 catalog.roll(
false);
5141 catalog.getSqliteConnector().query(
"ROLLBACK TRANSACTION");
5149 const auto execute_read_lock = mapd_shared_lock<mapd_shared_mutex>(
5153 const auto td_with_lock =
5155 catalog, *
table_,
false);
5156 const auto td = td_with_lock();
5162 if (cd ==
nullptr) {
5163 throw std::runtime_error(
"Column " + *
column_ +
" does not exist.");
5166 throw std::runtime_error(
"Column " + *
new_column_name_ +
" already exists.");
5172 enum TableParamType { MaxRollbackEpochs,
Epoch, MaxRows };
5173 static const std::unordered_map<std::string, TableParamType> param_map = {
5174 {
"max_rollback_epochs", TableParamType::MaxRollbackEpochs},
5175 {
"epoch", TableParamType::Epoch},
5176 {
"max_rows", TableParamType::MaxRows}};
5177 const auto execute_read_lock = mapd_shared_lock<mapd_shared_mutex>(
5181 const auto td_with_lock =
5183 catalog, *
table_,
false);
5184 const auto td = td_with_lock();
5186 throw std::runtime_error(
"Table " + *
table_ +
" does not exist.");
5189 throw std::runtime_error(
"Setting parameters for a view is not supported.");
5192 throw std::runtime_error(
5193 "Setting parameters for a temporary table is not yet supported.");
5198 std::vector<int> table_key{catalog.getCurrentDB().dbId, td->
tableId};
5201 std::string param_name(*
param_->get_name());
5205 if (val_int_literal ==
nullptr) {
5206 throw std::runtime_error(
"Table parameters should be integers.");
5208 const int64_t param_val = val_int_literal->
get_intval();
5210 const auto param_it = param_map.find(param_name);
5211 if (param_it == param_map.end()) {
5212 throw std::runtime_error(param_name +
" is not a settable table parameter.");
5214 switch (param_it->second) {
5215 case MaxRollbackEpochs: {
5216 catalog.setMaxRollbackEpochs(td->
tableId, param_val);
5220 catalog.setTableEpoch(catalog.getDatabaseId(), td->
tableId, param_val);
5224 catalog.setMaxRows(td->
tableId, param_val);
5228 UNREACHABLE() <<
"Unexpected TableParamType value: " << param_it->second
5229 <<
", key: " << param_it->first;
5236 std::list<NameValueAssign*>* o)
5237 : table_(t), copy_from_source_pattern_(f), success_(
true) {
5239 for (
const auto e : *o) {
5247 CHECK(payload.HasMember(
"table"));
5248 table_ = std::make_unique<std::string>(
json_str(payload[
"table"]));
5250 CHECK(payload.HasMember(
"filePath"));
5251 std::string fs =
json_str(payload[
"filePath"]);
5253 boost::algorithm::trim_if(fs, boost::is_any_of(
" \"'`"));
5262 const std::string& copy_from_source,
5264 -> std::unique_ptr<import_export::AbstractImporter> {
5267 return execute(session, importer_factory);
5272 const std::function<std::unique_ptr<import_export::AbstractImporter>(
5277 size_t total_time = 0;
5280 const auto execute_read_lock = mapd_shared_lock<mapd_shared_mutex>(
5285 std::unique_ptr<lockmgr::TableSchemaLockContainer<lockmgr::ReadLock>> td_with_lock;
5286 std::unique_ptr<lockmgr::WriteLock> insert_data_lock;
5291 td_with_lock = std::make_unique<lockmgr::TableSchemaLockContainer<lockmgr::ReadLock>>(
5294 td = (*td_with_lock)();
5295 insert_data_lock = std::make_unique<lockmgr::WriteLock>(
5297 }
catch (
const std::runtime_error& e) {
5305 std::vector<DBObject> privObjects;
5309 privObjects.push_back(dbObject);
5310 if (!SysCatalog::instance().checkPrivileges(session.
get_currentUser(), privObjects)) {
5311 throw std::runtime_error(
"Violation of access privileges: user " +
5313 " has no insert privileges for table " + *
table_ +
".");
5321 std::vector<std::string> warnings;
5324 boost::regex non_local_file_regex{R
"(^\s*(s3|http|https)://.+)",
5325 boost::regex::extended | boost::regex::icase};
5340 throw std::runtime_error(
5341 "Option \"SQL ORDER BY\" must be specified when copying from an ODBC source.");
5347 for (
auto const& warning : warnings) {
5348 tr += warning +
"\n";
5364 tr += std::string(
"Appending geo to table '") + *
table_ + std::string(
"'...");
5366 tr += std::string(
"Creating table '") + *
table_ +
5367 std::string(
"' and importing geo...");
5371 CHECK(td_with_lock);
5374 auto importer = importer_factory(catalog, td, copy_from_source, copy_params);
5378 auto query_str =
"COPYING " + td->
tableName;
5380 executor->enrollQuerySession(query_session,