OmniSciDB  a987f07e93
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Calcite.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
23 #include "Calcite.h"
24 #include "Catalog/Catalog.h"
25 #include "Logger/Logger.h"
27 #include "Shared/SysDefinitions.h"
29 #include "Shared/ThriftClient.h"
30 #include "Shared/fixautotools.h"
31 #include "Shared/measure.h"
33 
34 #include <thrift/protocol/TBinaryProtocol.h>
35 #include <thrift/transport/TSocket.h>
36 #include <thrift/transport/TTransportUtils.h>
37 #include <type_traits>
38 
39 #ifdef _MSC_VER
40 #include <process.h>
41 #endif
42 
43 #include "gen-cpp/CalciteServer.h"
44 
45 #include "rapidjson/document.h"
46 
47 #include <utility>
48 
49 using namespace rapidjson;
50 using namespace apache::thrift;
51 using namespace apache::thrift::protocol;
52 using namespace apache::thrift::transport;
53 
54 namespace {
55 template <typename XDEBUG_OPTION,
56  typename REMOTE_DEBUG_OPTION,
57  typename... REMAINING_ARGS>
58 int wrapped_execlp(char const* path,
59  XDEBUG_OPTION&& x_debug,
60  REMOTE_DEBUG_OPTION&& remote_debug,
61  REMAINING_ARGS&&... standard_args) {
62 #ifdef ENABLE_JAVA_REMOTE_DEBUG
63  return execlp(
64  path, x_debug, remote_debug, std::forward<REMAINING_ARGS>(standard_args)...);
65 #else
66  return execlp(path, std::forward<REMAINING_ARGS>(standard_args)...);
67 #endif
68 }
69 } // namespace
70 
71 static void start_calcite_server_as_daemon(const int db_port,
72  const int port,
73  const std::string& data_dir,
74  const size_t calcite_max_mem,
75  const std::string& ssl_trust_store,
76  const std::string& ssl_trust_password_X,
77  const std::string& ssl_keystore,
78  const std::string& ssl_keystore_password_X,
79  const std::string& ssl_key_file,
80  const std::string& db_config_file,
81  const std::string& udf_filename) {
82  auto root_abs_path = heavyai::get_root_abs_path();
83  std::string const xDebug = "-Xdebug";
84  std::string const remoteDebug =
85  "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005";
86  std::string xmxP = "-Xmx" + std::to_string(calcite_max_mem) + "m";
87  std::string jarP = "-jar";
88  std::string jarD =
89  root_abs_path + "/bin/calcite-1.0-SNAPSHOT-jar-with-dependencies.jar";
90  std::string extensionsP = "-e";
91  std::string extensionsD = root_abs_path + "/QueryEngine/";
92  std::string dataP = "-d";
93  std::string dataD = data_dir;
94  std::string localPortP = "-p";
95  std::string localPortD = std::to_string(port);
96  std::string dbPortP = "-m";
97  std::string dbPortD = std::to_string(db_port);
98  std::string TrustStoreP = "-T";
99  std::string TrustPasswdP = "-P";
100  std::string ConfigFileP = "-c";
101  std::string KeyStoreP = "-Y";
102  std::string KeyStorePasswdP = "-Z";
103  // FIXME: this path should be getting pulled from logger rather than hardcoded
104  std::string logDirectory =
105  "-DLOG_DIR=" + data_dir + "/" + shared::kDefaultLogDirName + "/";
106  std::string userDefinedFunctionsP = "";
107  std::string userDefinedFunctionsD = "";
108 
109  if (!udf_filename.empty()) {
110  userDefinedFunctionsP += "-u";
111  userDefinedFunctionsD += udf_filename;
112  }
113 
114  // If a config file hasn't been supplied then put the password in the params
115  // otherwise send an empty string and Calcite should get it from the config file.
116  std::string key_store_password = (db_config_file == "") ? ssl_keystore_password_X : "";
117  std::string trust_store_password = (db_config_file == "") ? ssl_trust_password_X : "";
118 #ifdef _MSC_VER
119  // TODO: enable UDF support
120  std::vector<std::string> args_vec;
121  args_vec.push_back("java");
122  args_vec.push_back(xDebug);
123  args_vec.push_back(remoteDebug);
124  args_vec.push_back(xmxP);
125  args_vec.push_back(logDirectory);
126  args_vec.push_back(jarP);
127  args_vec.push_back(jarD);
128  args_vec.push_back(extensionsP);
129  args_vec.push_back(extensionsD);
130  args_vec.push_back(dataP);
131  args_vec.push_back(dataD);
132  args_vec.push_back(localPortP);
133  args_vec.push_back(localPortD);
134  args_vec.push_back(dbPortP);
135  args_vec.push_back(dbPortD);
136  if (!ssl_trust_store.empty()) {
137  args_vec.push_back(TrustStoreP);
138  args_vec.push_back(ssl_trust_store);
139  }
140  if (!trust_store_password.empty()) {
141  args_vec.push_back(TrustPasswdP);
142  args_vec.push_back(trust_store_password);
143  }
144  if (!ssl_keystore.empty()) {
145  args_vec.push_back(KeyStoreP);
146  args_vec.push_back(ssl_keystore);
147  }
148  if (!key_store_password.empty()) {
149  args_vec.push_back(KeyStorePasswdP);
150  args_vec.push_back(key_store_password);
151  }
152  if (!db_config_file.empty()) {
153  args_vec.push_back(ConfigFileP);
154  args_vec.push_back(db_config_file);
155  }
156  std::string args{boost::algorithm::join(args_vec, " ")};
157  STARTUPINFO startup_info;
158  PROCESS_INFORMATION proc_info;
159  ZeroMemory(&startup_info, sizeof(startup_info));
160  startup_info.cb = sizeof(startup_info);
161  ZeroMemory(&proc_info, sizeof(proc_info));
162  LOG(INFO) << "Startup command: " << args;
163  std::wstring wargs = std::wstring(args.begin(), args.end());
164  const auto ret = CreateProcess(NULL,
165  (LPWSTR)wargs.c_str(),
166  NULL,
167  NULL,
168  false,
169  0,
170  NULL,
171  NULL,
172  &startup_info,
173  &proc_info);
174  if (ret == 0) {
175  LOG(FATAL) << "Failed to start Calcite server " << GetLastError();
176  }
177 #else
178  int pid = fork();
179  if (pid == 0) {
180  int i;
181 
182  if (udf_filename.empty()) {
183  i = wrapped_execlp("java",
184  xDebug.c_str(),
185  remoteDebug.c_str(),
186  xmxP.c_str(),
187  logDirectory.c_str(),
188  jarP.c_str(),
189  jarD.c_str(),
190  extensionsP.c_str(),
191  extensionsD.c_str(),
192  dataP.c_str(),
193  dataD.c_str(),
194  localPortP.c_str(),
195  localPortD.c_str(),
196  dbPortP.c_str(),
197  dbPortD.c_str(),
198  TrustStoreP.c_str(),
199  ssl_trust_store.c_str(),
200  TrustPasswdP.c_str(),
201  trust_store_password.c_str(),
202  KeyStoreP.c_str(),
203  ssl_keystore.c_str(),
204  KeyStorePasswdP.c_str(),
205  key_store_password.c_str(),
206  ConfigFileP.c_str(),
207  db_config_file.c_str(),
208  (char*)0);
209  } else {
210  i = wrapped_execlp("java",
211  xDebug.c_str(),
212  remoteDebug.c_str(),
213  xmxP.c_str(),
214  logDirectory.c_str(),
215  jarP.c_str(),
216  jarD.c_str(),
217  extensionsP.c_str(),
218  extensionsD.c_str(),
219  dataP.c_str(),
220  dataD.c_str(),
221  localPortP.c_str(),
222  localPortD.c_str(),
223  dbPortP.c_str(),
224  dbPortD.c_str(),
225  TrustStoreP.c_str(),
226  ssl_trust_store.c_str(),
227  TrustPasswdP.c_str(),
228  trust_store_password.c_str(),
229  KeyStoreP.c_str(),
230  ssl_keystore.c_str(),
231  KeyStorePasswdP.c_str(),
232  key_store_password.c_str(),
233  ConfigFileP.c_str(),
234  db_config_file.c_str(),
235  userDefinedFunctionsP.c_str(),
236  userDefinedFunctionsD.c_str(),
237  (char*)0);
238  }
239 
240  if (i) {
241  int errsv = errno;
242  LOG(FATAL) << "Failed to start Calcite server [errno=" << errsv
243  << "]: " << strerror(errsv);
244  } else {
245  LOG(INFO) << "Successfully started Calcite server";
246  }
247  }
248 #endif
249 }
250 
251 std::pair<std::shared_ptr<CalciteServerClient>, std::shared_ptr<TTransport>>
253  const auto transport = connMgr_->open_buffered_client_transport("localhost",
254  port,
255  ssl_ca_file_,
256  true,
257  service_keepalive_,
258  service_timeout_,
259  service_timeout_,
260  service_timeout_);
261  try {
262  transport->open();
263 
264  } catch (TException& tx) {
265  throw tx;
266  } catch (std::exception& ex) {
267  throw ex;
268  }
269  std::shared_ptr<TProtocol> protocol(new TBinaryProtocol(transport));
270  std::shared_ptr<CalciteServerClient> client;
271  client.reset(new CalciteServerClient(protocol));
272  std::pair<std::shared_ptr<CalciteServerClient>, std::shared_ptr<TTransport>> ret;
273  return std::make_pair(client, transport);
274 }
275 
276 void Calcite::runServer(const int db_port,
277  const int port,
278  const std::string& data_dir,
279  const size_t calcite_max_mem,
280  const std::string& udf_filename) {
281  LOG(INFO) << "Running Calcite server as a daemon";
282 
283  // ping server to see if for any reason there is an orphaned one
284  int ping_time = ping();
285  if (ping_time > -1) {
286  // we have an orphaned server shut it down
287  LOG(ERROR)
288  << "Appears to be orphaned Calcite server already running, shutting it down";
289  LOG(ERROR) << "Please check that you are not trying to run two servers on same port";
290  LOG(ERROR) << "Attempting to shutdown orphaned Calcite server";
291  try {
292  auto clientP = getClient(remote_calcite_port_);
293  clientP.first->shutdown();
294  clientP.second->close();
295  LOG(ERROR) << "orphaned Calcite server shutdown";
296 
297  } catch (TException& tx) {
298  LOG(ERROR) << "Failed to shutdown orphaned Calcite server, reason: " << tx.what();
299  }
300  }
301 
302  // start the calcite server as a seperate process
304  port,
305  data_dir,
306  calcite_max_mem,
307  ssl_trust_store_,
308  ssl_trust_password_,
309  ssl_keystore_,
310  ssl_keystore_password_,
311  ssl_key_file_,
312  db_config_file_,
313  udf_filename);
314 
315  // check for new server for 30 seconds max
316  std::this_thread::sleep_for(std::chrono::milliseconds(200));
317  int retry_max = 300;
318  for (int i = 2; i <= retry_max; i++) {
319  int ping_time = ping(i, retry_max);
320  if (ping_time > -1) {
321  LOG(INFO) << "Calcite server start took " << i * 100 << " ms ";
322  LOG(INFO) << "ping took " << ping_time << " ms ";
323  server_available_ = true;
324  return;
325  } else {
326  // wait 100 ms
327  std::this_thread::sleep_for(std::chrono::milliseconds(100));
328  }
329  }
330  server_available_ = false;
331  LOG(FATAL) << "Could not connect to Calcite remote server running on port [" << port
332  << "]";
333 }
334 
335 // ping existing server
336 // return -1 if no ping response
337 // params set to default values in header
338 int Calcite::ping(int retry_num, int max_retry) {
339  try {
340  auto ms = measure<>::execution([&]() {
341  auto clientP = getClient(remote_calcite_port_);
342  clientP.first->ping();
343  clientP.second->close();
344  });
345  return ms;
346 
347  } catch (TException& tx) {
348  if (retry_num >= max_retry) {
349  LOG(ERROR) << "Problems connecting to Calcite. Thrift error - " << tx.what();
350  }
351  return -1;
352  }
353 }
354 
355 Calcite::Calcite(const int db_port,
356  const int calcite_port,
357  const std::string& data_dir,
358  const size_t calcite_max_mem,
359  const size_t service_timeout,
360  const bool service_keepalive,
361  const std::string& udf_filename)
362  : server_available_(false)
363  , service_timeout_(service_timeout)
364  , service_keepalive_(service_keepalive) {
365  init(db_port, calcite_port, data_dir, calcite_max_mem, udf_filename);
366 }
367 
368 void Calcite::init(const int db_port,
369  const int calcite_port,
370  const std::string& data_dir,
371  const size_t calcite_max_mem,
372  const std::string& udf_filename) {
373  LOG(INFO) << "Creating Calcite Handler, Calcite Port is " << calcite_port
374  << " base data dir is " << data_dir;
375  connMgr_ = std::make_shared<ThriftClientConnection>();
376  if (calcite_port < 0) {
377  CHECK(false) << "JNI mode no longer supported.";
378  }
379  if (calcite_port == 0) {
380  // dummy process for initheavy
381  remote_calcite_port_ = calcite_port;
382  server_available_ = false;
383  } else {
384  remote_calcite_port_ = calcite_port;
385  runServer(db_port, calcite_port, data_dir, calcite_max_mem, udf_filename);
386  server_available_ = true;
387  }
388 }
389 
390 Calcite::Calcite(const SystemParameters& system_parameters,
391  const std::string& data_dir,
392  const std::string& udf_filename)
393  : service_timeout_(system_parameters.calcite_timeout)
394  , service_keepalive_(system_parameters.calcite_keepalive)
395  , ssl_trust_store_(system_parameters.ssl_trust_store)
396  , ssl_trust_password_(system_parameters.ssl_trust_password)
397  , ssl_key_file_(system_parameters.ssl_key_file)
398  , ssl_keystore_(system_parameters.ssl_keystore)
399  , ssl_keystore_password_(system_parameters.ssl_keystore_password)
400  , ssl_ca_file_(system_parameters.ssl_trust_ca_file)
401  , db_config_file_(system_parameters.config_file) {
402  init(system_parameters.omnisci_server_port,
403  system_parameters.calcite_port,
404  data_dir,
405  system_parameters.calcite_max_mem,
406  udf_filename);
407 }
408 
409 void Calcite::updateMetadata(std::string catalog, std::string table) {
410  if (server_available_) {
411  auto ms = measure<>::execution([&]() {
412  auto clientP = getClient(remote_calcite_port_);
413  clientP.first->updateMetadata(catalog, table);
414  clientP.second->close();
415  });
416  LOG(INFO) << "Time to updateMetadata " << ms << " (ms)";
417  } else {
418  LOG(INFO) << "Not routing to Calcite, server is not up";
419  }
420 }
421 
423  std::vector<std::vector<std::string>> tableOrViewNames,
424  AccessPrivileges tablePrivs,
425  AccessPrivileges viewPrivs) {
426  // TODO MAT this needs to be able to check privileges from other catalogs
427  Catalog_Namespace::Catalog& catalog = session_info.getCatalog();
428 
429  for (auto tableOrViewName : tableOrViewNames) {
430  const TableDescriptor* tableMeta =
431  catalog.getMetadataForTable(tableOrViewName[0], false);
432 
433  if (!tableMeta) {
434  throw std::runtime_error("unknown table of view: " + tableOrViewName[0]);
435  }
436 
437  DBObjectKey key;
438  key.dbId = catalog.getCurrentDB().dbId;
441  key.objectId = tableMeta->tableId;
442  AccessPrivileges privs = tableMeta->isView ? viewPrivs : tablePrivs;
443  DBObject dbobject(key, privs, tableMeta->userId);
444  std::vector<DBObject> privObjects{dbobject};
445 
446  if (!privs.hasAny()) {
447  throw std::runtime_error("Operation not supported for object " +
448  tableOrViewName[0]);
449  }
450 
451  if (!Catalog_Namespace::SysCatalog::instance().checkPrivileges(
452  session_info.get_currentUser(), privObjects)) {
453  throw std::runtime_error("Violation of access privileges: user " +
454  session_info.get_currentUser().userLoggable() +
455  " has no proper privileges for object " +
456  tableOrViewName[0]);
457  }
458  }
459 }
460 
461 TPlanResult Calcite::process(query_state::QueryStateProxy query_state_proxy,
462  std::string sql_string,
463  const TQueryParsingOption& query_parsing_option,
464  const TOptimizationOption& optimization_option,
465  const std::string& calcite_session_id) {
466  TPlanResult result = processImpl(query_state_proxy,
467  std::move(sql_string),
468  query_parsing_option,
469  optimization_option,
470  calcite_session_id);
471  if (query_parsing_option.check_privileges && !query_parsing_option.is_explain) {
472  checkAccessedObjectsPrivileges(query_state_proxy, result);
473  }
474  return result;
475 }
476 
478  query_state::QueryStateProxy query_state_proxy,
479  TPlanResult plan) const {
480  AccessPrivileges NOOP;
481  // check the individual tables
482  auto const session_ptr = query_state_proxy->getConstSessionInfo();
483  checkPermissionForTables(*session_ptr,
484  plan.primary_accessed_objects.tables_selected_from,
487  checkPermissionForTables(*session_ptr,
488  plan.primary_accessed_objects.tables_inserted_into,
490  NOOP);
491  checkPermissionForTables(*session_ptr,
492  plan.primary_accessed_objects.tables_updated_in,
494  NOOP);
495  checkPermissionForTables(*session_ptr,
496  plan.primary_accessed_objects.tables_deleted_from,
498  NOOP);
499 }
500 
501 std::vector<TCompletionHint> Calcite::getCompletionHints(
502  const Catalog_Namespace::SessionInfo& session_info,
503  const std::vector<std::string>& visible_tables,
504  const std::string sql_string,
505  const int cursor) {
506  std::vector<TCompletionHint> hints;
507  auto& cat = session_info.getCatalog();
508  const auto user = session_info.get_currentUser().userName;
509  const auto session = session_info.get_session_id();
510  const auto catalog = cat.getCurrentDB().dbName;
511  auto client = getClient(remote_calcite_port_);
512  client.first->getCompletionHints(
513  hints, user, session, catalog, visible_tables, sql_string, cursor);
514  return hints;
515 }
516 
517 std::vector<std::string> Calcite::get_db_objects(const std::string ra) {
518  std::vector<std::string> v_db_obj;
519  Document document;
520  document.Parse(ra.c_str());
521  const Value& rels = document["rels"];
522  CHECK(rels.IsArray());
523  for (auto& v : rels.GetArray()) {
524  std::string relOp(v["relOp"].GetString());
525  if (!relOp.compare("EnumerableTableScan")) {
526  std::string x;
527  auto t = v["table"].GetArray();
528  x = t[1].GetString();
529  v_db_obj.push_back(x);
530  }
531  }
532 
533  return v_db_obj;
534 }
535 
537  const std::string sql_string,
538  const TQueryParsingOption& query_parsing_option,
539  const TOptimizationOption& optimization_option,
540  const std::string& calcite_session_id) {
541  query_state::Timer timer = query_state_proxy.createTimer(__func__);
542  const auto& user_session_info = query_state_proxy->getConstSessionInfo();
543  const auto& cat = user_session_info->getCatalog();
544  const std::string user = getInternalSessionProxyUserName();
545  const std::string catalog = cat.getCurrentDB().dbName;
546  LOG(INFO) << "User " << user << " catalog " << catalog << " sql '"
547  << hide_sensitive_data_from_query(sql_string) << "'";
548  LOG(IR) << "SQL query\n"
549  << hide_sensitive_data_from_query(sql_string) << "\nEnd of SQL query";
550  LOG(PTX) << "SQL query\n"
551  << hide_sensitive_data_from_query(sql_string) << "\nEnd of SQL query";
552 
553  std::vector<TRestriction> trestrictions;
554 
555  TPlanResult ret;
556  if (server_available_) {
557  try {
558  // calcite_session_id would be an empty string when accessed by internal resources
559  // that would not access `process` through handler instance, like for eg: Unit
560  // Tests. In these cases we would use the session_id from query state.
561  auto ms = measure<>::execution([&]() {
562  auto clientP = getClient(remote_calcite_port_);
563  clientP.first->process(ret,
564  user,
565  calcite_session_id.empty()
566  ? user_session_info->get_session_id()
567  : calcite_session_id,
568  catalog,
569  sql_string,
570  query_parsing_option,
571  optimization_option,
572  trestrictions);
573  clientP.second->close();
574  });
575 
576  // LOG(INFO) << ret.plan_result;
577  LOG(INFO) << "Time in Thrift "
578  << (ms > ret.execution_time_ms ? ms - ret.execution_time_ms : 0)
579  << " (ms), Time in Java Calcite server " << ret.execution_time_ms
580  << " (ms)";
581  } catch (InvalidParseRequest& e) {
582  throw std::invalid_argument(e.whyUp);
583  } catch (const TTransportException& ex) {
584  if (ex.getType() == TTransportException::TIMED_OUT) {
585  LOG(WARNING) << "Calcite request timed out: " << ex.what();
586  } else {
587  LOG(FATAL) << "Error occurred trying to communicate with Calcite server, the "
588  "error was: '"
589  << ex.what() << "', heavydb restart will be required";
590  }
591  throw;
592  } catch (const std::exception& ex) {
593  LOG(FATAL)
594  << "Error occurred trying to communicate with Calcite server, the error was: '"
595  << ex.what() << "', heavydb restart will be required";
596  return ret; // satisfy return-type warning
597  }
598  } else {
599  LOG(FATAL) << "Not routing to Calcite, server is not up";
600  }
601  return ret;
602 }
603 
605  if (server_available_) {
606  TPlanResult ret;
607  std::string whitelist;
608 
609  auto clientP = getClient(remote_calcite_port_);
610  clientP.first->getExtensionFunctionWhitelist(whitelist);
611  clientP.second->close();
612  VLOG(1) << whitelist;
613  return whitelist;
614  } else {
615  LOG(FATAL) << "Not routing to Calcite, server is not up";
616  return "";
617  }
618  CHECK(false);
619  return "";
620 }
621 
623  if (server_available_) {
624  TPlanResult ret;
625  std::string whitelist;
626 
627  auto clientP = getClient(remote_calcite_port_);
628  clientP.first->getUserDefinedFunctionWhitelist(whitelist);
629  clientP.second->close();
630  VLOG(1) << "User defined functions whitelist loaded from Calcite: " << whitelist;
631  return whitelist;
632  } else {
633  LOG(FATAL) << "Not routing to Calcite, server is not up";
634  return "";
635  }
636  UNREACHABLE();
637  return "";
638 }
639 
641  std::call_once(shutdown_once_flag_,
642  [this, log]() { this->inner_close_calcite_server(log); });
643 }
644 
646  if (server_available_) {
647  LOG_IF(INFO, log) << "Shutting down Calcite server";
648  try {
649  auto clientP = getClient(remote_calcite_port_);
650  clientP.first->shutdown();
651  clientP.second->close();
652  } catch (const std::exception& e) {
653  if (std::string(e.what()) != "connect() failed: Connection refused" &&
654  std::string(e.what()) != "socket open() error: Connection refused" &&
655  std::string(e.what()) != "No more data to read.") {
656  std::cerr << "Error shutting down Calcite server: " << e.what() << std::endl;
657  } // else Calcite already shut down
658  }
659  LOG_IF(INFO, log) << "shut down Calcite";
660  server_available_ = false;
661  }
662 }
663 
665  close_calcite_server(false);
666 }
667 
669  if (server_available_) {
670  TPlanResult ret;
671  std::string whitelist;
672  auto clientP = getClient(remote_calcite_port_);
673  clientP.first->getRuntimeExtensionFunctionWhitelist(whitelist);
674  clientP.second->close();
675  VLOG(1) << "Runtime extension functions whitelist loaded from Calcite: " << whitelist;
676  return whitelist;
677  } else {
678  LOG(FATAL) << "Not routing to Calcite, server is not up";
679  return "";
680  }
681  UNREACHABLE();
682  return "";
683 }
684 
686  const std::vector<TUserDefinedFunction>& udfs,
687  const std::vector<TUserDefinedTableFunction>& udtfs,
688  bool isruntime) {
689  if (server_available_) {
690  auto clientP = getClient(remote_calcite_port_);
691  clientP.first->setRuntimeExtensionFunctions(udfs, udtfs, isruntime);
692  clientP.second->close();
693  } else {
694  LOG(FATAL) << "Not routing to Calcite, server is not up";
695  }
696 }
697 
698 TQueryParsingOption Calcite::getCalciteQueryParsingOption(bool legacy_syntax,
699  bool is_explain,
700  bool check_privileges) {
701  TQueryParsingOption query_parsing_info;
702  query_parsing_info.legacy_syntax = legacy_syntax;
703  query_parsing_info.is_explain = is_explain;
704  query_parsing_info.check_privileges = check_privileges;
705  return query_parsing_info;
706 }
707 
709  bool is_view_optimize,
710  bool enable_watchdog,
711  const std::vector<TFilterPushDownInfo>& filter_push_down_info,
712  bool distributed_mode) {
713  TOptimizationOption optimization_option;
714  optimization_option.filter_push_down_info = filter_push_down_info;
715  optimization_option.is_view_optimize = is_view_optimize;
716  optimization_option.enable_watchdog = enable_watchdog;
717  optimization_option.distributed_mode = distributed_mode;
718  return optimization_option;
719 }
std::vector< std::string > get_db_objects(const std::string ra)
Definition: Calcite.cpp:517
std::string hide_sensitive_data_from_query(std::string const &query_str)
int wrapped_execlp(char const *path, XDEBUG_OPTION &&x_debug, REMOTE_DEBUG_OPTION &&remote_debug, REMAINING_ARGS &&...standard_args)
Definition: Calcite.cpp:58
TPlanResult processImpl(query_state::QueryStateProxy, std::string sql_string, const TQueryParsingOption &query_parsing_option, const TOptimizationOption &optimization_option, const std::string &calcite_session_id)
Definition: Calcite.cpp:536
std::string cat(Ts &&...args)
std::once_flag shutdown_once_flag_
Definition: Calcite.h:144
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:132
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
std::string get_root_abs_path()
void init(const int db_port, const int port, const std::string &data_dir, const size_t calcite_max_mem, const std::string &udf_filename)
Definition: Calcite.cpp:368
#define LOG(tag)
Definition: Logger.h:283
static std::string const getInternalSessionProxyUserName()
Definition: Calcite.h:96
const std::string kDefaultLogDirName
static const AccessPrivileges INSERT_INTO_TABLE
Definition: DBObject.h:161
std::string join(T const &container, std::string const &delim)
#define UNREACHABLE()
Definition: Logger.h:333
bool hasAny() const
Definition: DBObject.h:140
void setRuntimeExtensionFunctions(const std::vector< TUserDefinedFunction > &udfs, const std::vector< TUserDefinedTableFunction > &udtfs, bool isruntime=true)
Definition: Calcite.cpp:685
int32_t objectId
Definition: DBObject.h:55
std::shared_ptr< ThriftClientConnection > connMgr_
Definition: Calcite.h:132
Timer createTimer(char const *event_name)
Definition: QueryState.cpp:129
TOptimizationOption getCalciteOptimizationOption(bool is_view_optimize, bool enable_watchdog, const std::vector< TFilterPushDownInfo > &filter_push_down_info, bool distributed_mode)
Definition: Calcite.cpp:708
static const AccessPrivileges SELECT_FROM_TABLE
Definition: DBObject.h:160
std::string to_string(char const *&&v)
#define LOG_IF(severity, condition)
Definition: Logger.h:379
void updateMetadata(std::string catalog, std::string table)
Definition: Calcite.cpp:409
Calcite(const int db_port, const int port, const std::string &data_dir, const size_t calcite_max_mem, const size_t service_timeout, const bool service_keepalive, const std::string &udf_filename="")
Definition: Calcite.cpp:355
This file contains the class specification and related data structures for Catalog.
void runServer(const int db_port, const int port, const std::string &data_dir, const size_t calcite_max_mem, const std::string &udf_filename)
Definition: Calcite.cpp:276
void checkPermissionForTables(const Catalog_Namespace::SessionInfo &session_info, std::vector< std::vector< std::string >> tableOrViewNames, AccessPrivileges tablePrivs, AccessPrivileges viewPrivs)
Definition: Calcite.cpp:422
static SysCatalog & instance()
Definition: SysCatalog.h:341
static void start_calcite_server_as_daemon(const int db_port, const int port, const std::string &data_dir, const size_t calcite_max_mem, const std::string &ssl_trust_store, const std::string &ssl_trust_password_X, const std::string &ssl_keystore, const std::string &ssl_keystore_password_X, const std::string &ssl_key_file, const std::string &db_config_file, const std::string &udf_filename)
Definition: Calcite.cpp:71
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:248
std::vector< TCompletionHint > getCompletionHints(const Catalog_Namespace::SessionInfo &session_info, const std::vector< std::string > &visible_tables, const std::string sql_string, const int cursor)
Definition: Calcite.cpp:501
TQueryParsingOption getCalciteQueryParsingOption(bool legacy_syntax, bool is_explain, bool check_privileges)
Definition: Calcite.cpp:698
static const AccessPrivileges DELETE_FROM_TABLE
Definition: DBObject.h:163
int ping(int retry_num=0, int max_retry=50)
Definition: Calcite.cpp:338
std::string getUserDefinedFunctionWhitelist()
Definition: Calcite.cpp:622
std::string get_session_id() const
Definition: SessionInfo.h:93
static const AccessPrivileges SELECT_FROM_VIEW
Definition: DBObject.h:180
Catalog & getCatalog() const
Definition: SessionInfo.h:75
int remote_calcite_port_
Definition: Calcite.h:136
int32_t dbId
Definition: DBObject.h:54
void inner_close_calcite_server(bool log)
Definition: Calcite.cpp:645
bool server_available_
Definition: Calcite.h:133
bool g_enable_watchdog false
Definition: Execute.cpp:79
#define CHECK(condition)
Definition: Logger.h:289
std::pair< std::shared_ptr< CalciteServerClient >, std::shared_ptr< TTransport > > getClient(int port)
Definition: Calcite.cpp:252
int32_t permissionType
Definition: DBObject.h:53
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
TPlanResult process(query_state::QueryStateProxy, std::string sql_string, const TQueryParsingOption &query_parsing_option, const TOptimizationOption &optimization_option, const std::string &calcite_session_id="")
Definition: Calcite.cpp:461
static const AccessPrivileges UPDATE_IN_TABLE
Definition: DBObject.h:162
void checkAccessedObjectsPrivileges(query_state::QueryStateProxy query_state_prox, TPlanResult plan) const
Definition: Calcite.cpp:477
std::string userLoggable() const
Definition: SysCatalog.cpp:158
std::shared_ptr< Catalog_Namespace::SessionInfo const > getConstSessionInfo() const
Definition: QueryState.cpp:84
~Calcite()
Definition: Calcite.cpp:664
std::string getExtensionFunctionWhitelist()
Definition: Calcite.cpp:604
const UserMetadata & get_currentUser() const
Definition: SessionInfo.h:88
std::string getRuntimeExtensionFunctionWhitelist()
Definition: Calcite.cpp:668
#define VLOG(n)
Definition: Logger.h:383
void close_calcite_server(bool log=true)
Definition: Calcite.cpp:640