OmniSciDB  04ee39c94c
omnisql.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2019 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
24 #include <rapidjson/document.h>
25 #include <termios.h>
26 #include <boost/algorithm/string.hpp>
27 #include <boost/algorithm/string/join.hpp>
28 #include <boost/algorithm/string/predicate.hpp>
29 #include <boost/algorithm/string/trim.hpp>
30 #include <boost/filesystem.hpp>
31 #include <boost/format.hpp>
32 #include <boost/program_options.hpp>
33 #include <cmath>
34 #include <csignal>
35 #include <cstdio>
36 #include <cstdlib>
37 #include <cstring>
38 #include <iomanip>
39 #include <iostream>
40 #include <limits>
41 #include <sstream>
42 #include <string>
43 
44 #include <thrift/protocol/TBinaryProtocol.h>
45 #include <thrift/protocol/TJSONProtocol.h>
46 #include <thrift/transport/TBufferTransports.h>
47 #include <thrift/transport/TSocket.h>
48 #include "../Fragmenter/InsertOrderFragmenter.h"
49 #include "MapDRelease.h"
50 #include "MapDServer.h"
51 #include "Shared/Logger.h"
52 #include "Shared/StringTransform.h"
53 #include "Shared/ThriftClient.h"
55 #include "Shared/base64.h"
56 #include "Shared/checked_alloc.h"
57 #include "Shared/mapd_shared_ptr.h"
58 #include "gen-cpp/MapD.h"
59 
60 #include "linenoise.h"
61 
62 #include "ClientContext.h"
63 #include "CommandFunctors.h"
64 #include "CommandHistoryFile.h"
65 
66 using namespace ::apache::thrift;
67 using namespace ::apache::thrift::protocol;
68 using namespace ::apache::thrift::transport;
69 
70 using namespace std::string_literals;
71 
72 const std::string OmniSQLRelease(MAPD_RELEASE);
73 
74 namespace {
75 
77 
78 void completion(const char* buf, linenoiseCompletions* lc) {
81  for (const auto& completion_hint : g_client_context_ptr->completion_hints) {
82  for (const auto& hint_str : completion_hint.hints) {
83  CHECK_LE(completion_hint.replaced.size(), strlen(buf));
84  std::string partial_query(buf, buf + strlen(buf) - completion_hint.replaced.size());
85  partial_query += hint_str;
86  linenoiseAddCompletion(lc, partial_query.c_str());
87  }
88  }
89 }
90 
91 #define LOAD_PATCH_SIZE 10000
92 
93 void copy_table(char const* filepath, char const* table, ClientContext& context) {
94  if (context.session == INVALID_SESSION_ID) {
95  std::cerr << "Not connected to any databases." << std::endl;
96  return;
97  }
98  if (!boost::filesystem::exists(filepath)) {
99  std::cerr << "File does not exist." << std::endl;
100  return;
101  }
102  if (!thrift_with_retry(kGET_TABLE_DETAILS, context, table)) {
103  return;
104  }
105  const TRowDescriptor& row_desc = context.table_details.row_desc;
106  std::ifstream infile(filepath);
107  std::string line;
108  const char* delim = ",";
109  int l = strlen(filepath);
110  if (l >= 4 && strcmp(filepath + l - 4, ".tsv") == 0) {
111  delim = "\t";
112  }
113  std::vector<TStringRow> input_rows;
114  TStringRow row;
115  boost::char_separator<char> sep{delim, "", boost::keep_empty_tokens};
116  try {
117  while (std::getline(infile, line)) {
118  row.cols.clear();
119  boost::tokenizer<boost::char_separator<char>> tok{line, sep};
120  for (const auto& s : tok) {
121  TStringValue ts;
122  ts.str_val = s;
123  ts.is_null = s.empty();
124  row.cols.push_back(ts);
125  }
126  /*
127  std::cout << "Row: ";
128  for (const auto &p : row.cols) {
129  std::cout << p.str_val << ", ";
130  }
131  std::cout << std::endl;
132  */
133  if (row.cols.size() != row_desc.size()) {
134  std::cerr << "Incorrect number of columns: (" << row.cols.size() << " vs "
135  << row_desc.size() << ") " << line << std::endl;
136  continue;
137  }
138  input_rows.push_back(row);
139  if (input_rows.size() >= LOAD_PATCH_SIZE) {
140  try {
141  context.client.load_table(context.session, table, input_rows);
142  } catch (TMapDException& e) {
143  std::cerr << e.error_msg << std::endl;
144  }
145  input_rows.clear();
146  }
147  }
148  if (input_rows.size() > 0) {
149  context.client.load_table(context.session, table, input_rows);
150  }
151  } catch (TMapDException& e) {
152  std::cerr << e.error_msg << std::endl;
153  } catch (TException& te) {
154  std::cerr << "Thrift error: " << te.what() << std::endl;
155  }
156 }
157 
158 void detect_table(char* file_name, TCopyParams& copy_params, ClientContext& context) {
159  if (context.session == INVALID_SESSION_ID) {
160  std::cerr << "Not connected to any databases." << std::endl;
161  return;
162  }
163 
164  TDetectResult _return;
165 
166  try {
167  context.client.detect_column_types(_return, context.session, file_name, copy_params);
168  // print result only for verifying detect_column_types api
169  // as this cmd seems never planned for public use
170  for (const auto tct : _return.row_set.row_desc) {
171  printf("%20.20s ", tct.col_name.c_str());
172  }
173  printf("\n");
174  for (const auto tct : _return.row_set.row_desc) {
175  printf("%20.20s ", type_info_from_thrift(tct.col_type).get_type_name().c_str());
176  }
177  printf("\n");
178  for (const auto row : _return.row_set.rows) {
179  for (const auto col : row.cols) {
180  printf("%20.20s ", col.val.str_val.c_str());
181  }
182  printf("\n");
183  }
184  // output CREATE TABLE command
185  std::stringstream oss;
186  oss << "CREATE TABLE your_table_name(";
187  for (size_t i = 0; i < _return.row_set.row_desc.size(); ++i) {
188  const auto tct = _return.row_set.row_desc[i];
189  oss << (i ? ", " : "") << tct.col_name << " "
190  << type_info_from_thrift(tct.col_type).get_type_name();
191  if (type_info_from_thrift(tct.col_type).is_string()) {
192  oss << " ENCODING DICT";
193  }
194  if (type_info_from_thrift(tct.col_type).is_array()) {
195  oss << "[";
196  if (type_info_from_thrift(tct.col_type).get_size() > 0) {
197  oss << type_info_from_thrift(tct.col_type).get_size();
198  }
199  oss << "]";
200  }
201  }
202  oss << ");";
203  printf("\n%s\n", oss.str().c_str());
204  } catch (TMapDException& e) {
205  std::cerr << e.error_msg << std::endl;
206  } catch (TException& te) {
207  std::cerr << "Thrift error in detect_table: " << te.what() << std::endl;
208  }
209 }
210 
212  CHECK(!query_result.row_set.row_desc.empty());
213  if (query_result.row_set.columns.empty()) {
214  return 0;
215  }
216  CHECK_EQ(query_result.row_set.columns.size(), query_result.row_set.row_desc.size());
217  return query_result.row_set.columns.front().nulls.size();
218 }
219 
220 void get_table_epoch(ClientContext& context, const std::string& table_specifier) {
221  if (table_specifier.size() == 0) {
222  std::cerr
223  << "get table epoch requires table to be specified by name or by db_id:table_id"
224  << std::endl;
225  return;
226  }
227 
228  if (isdigit(table_specifier.at(0))) {
229  std::vector<std::string> split_result;
230 
231  boost::split(
232  split_result,
233  table_specifier,
234  boost::is_any_of(":"),
235  boost::token_compress_on); // SplitVec == { "hello abc","ABC","aBc goodbye" }
236 
237  if (split_result.size() != 2) {
238  std::cerr << "get table epoch does not contain db_id:table_id " << table_specifier
239  << std::endl;
240  return;
241  }
242 
243  // validate db identifier is a number
244  try {
245  context.db_id = std::stoi(split_result[0]);
246  } catch (std::exception& e) {
247  std::cerr << "non numeric db number: " << table_specifier << std::endl;
248  return;
249  }
250  // validate table identifier is a number
251  try {
252  context.table_id = std::stoi(split_result[1]);
253  } catch (std::exception& e) {
254  std::cerr << "non-numeric table number: " << table_specifier << std::endl;
255  return;
256  }
257 
258  if (thrift_with_retry(kGET_TABLE_EPOCH, context, nullptr)) {
259  std::cout << "table epoch is " << context.epoch_value << std::endl;
260  }
261  } else {
262  // presume we have a table name
263  // get the db_id and table_id from the table metadata
264  if (thrift_with_retry(kGET_PHYSICAL_TABLES, context, nullptr)) {
265  if (std::find(context.names_return.begin(),
266  context.names_return.end(),
267  table_specifier) == context.names_return.end()) {
268  std::cerr << "table " << table_specifier << " not found" << std::endl;
269  return;
270  }
271  } else {
272  return;
273  }
274  context.table_name = table_specifier;
275 
276  if (thrift_with_retry(kGET_TABLE_EPOCH_BY_NAME, context, nullptr)) {
277  std::cout << "table epoch is " << context.epoch_value << std::endl;
278  }
279  }
280 }
281 
282 void set_table_epoch(ClientContext& context, const std::string& table_details) {
283  if (table_details.size() == 0) {
284  std::cerr << "set table epoch requires table and epoch to be specified by name epoch "
285  "or by db_id:table_id:epoch"
286  << std::endl;
287  return;
288  }
289  if (isdigit(table_details.at(0))) {
290  std::vector<std::string> split_result;
291 
292  boost::split(
293  split_result,
294  table_details,
295  boost::is_any_of(":"),
296  boost::token_compress_on); // SplitVec == { "hello abc","ABC","aBc goodbye" }
297 
298  if (split_result.size() != 3) {
299  std::cerr << "Set table epoch does not contain db_id:table_id:epoch "
300  << table_details << std::endl;
301  return;
302  }
303 
304  // validate db identifier is a number
305  try {
306  context.db_id = std::stoi(split_result[0]);
307  } catch (std::exception& e) {
308  std::cerr << "non numeric db number: " << table_details << std::endl;
309  return;
310  }
311 
312  // validate table identifier is a number
313  try {
314  context.table_id = std::stoi(split_result[1]);
315  } catch (std::exception& e) {
316  std::cerr << "non-numeric table number: " << table_details << std::endl;
317  return;
318  }
319 
320  // validate epoch value
321  try {
322  context.epoch_value = std::stoi(split_result[2]);
323  } catch (std::exception& e) {
324  std::cerr << "non-numeric epoch number: " << table_details << std::endl;
325  return;
326  }
327  if (context.epoch_value < 0) {
328  std::cerr << "Epoch value can not be negative: " << table_details << std::endl;
329  return;
330  }
331 
332  if (thrift_with_retry(kSET_TABLE_EPOCH, context, nullptr)) {
333  std::cout << "table epoch set" << std::endl;
334  }
335  } else {
336  std::vector<std::string> split_result;
337  boost::split(
338  split_result, table_details, boost::is_any_of(" "), boost::token_compress_on);
339 
340  if (split_result.size() < 2) {
341  std::cerr << "Set table epoch does not contain table_name epoch " << std::endl;
342  return;
343  }
344 
345  if (thrift_with_retry(kGET_PHYSICAL_TABLES, context, nullptr)) {
346  if (std::find(context.names_return.begin(),
347  context.names_return.end(),
348  split_result[0]) == context.names_return.end()) {
349  std::cerr << "table " << split_result[0] << " not found" << std::endl;
350  return;
351  }
352  } else {
353  return;
354  }
355  context.table_name = split_result[0];
356  // validate epoch value
357  try {
358  context.epoch_value = std::stoi(split_result[1]);
359  } catch (std::exception& e) {
360  std::cerr << "non-numeric epoch number: " << table_details << std::endl;
361  return;
362  }
363  if (context.epoch_value < 0) {
364  std::cerr << "Epoch value can not be negative: " << table_details << std::endl;
365  return;
366  }
367  if (thrift_with_retry(kSET_TABLE_EPOCH_BY_NAME, context, nullptr)) {
368  std::cout << "table epoch set" << std::endl;
369  }
370  }
371 }
372 
373 void process_backslash_commands(char* command, ClientContext& context) {
374  // clang-format off
375  auto resolution_status = CommandResolutionChain<>( command, "\\h", 1, 1, HelpCmd<>( context ) )
376  ( "\\d", 2, 2, ListColumnsCmd<>( context ), "Usage: \\d <table>" )
377  ( "\\o", 2, 2, GetOptimizedSchemaCmd<>( context ), "Usage: \\o <table>" )
378  ( "\\t", 1, 1, ListTablesCmd<>( context ) )
379  ( "\\v", 1, 1, ListViewsCmd<>( context ) )
380  ( "\\c", 4, 4, ConnectToDBCmd<>( context ), "Usage: \\c <database> <user> <password>" )
381  ( "\\u", 1, 1, ListUsersCmd<>( context ) )
382  ( "\\l", 1, 1, ListDatabasesCmd<>( context ) )
383  .is_resolved();
384 
385  if( !resolution_status ) {
386  std::cerr << "Invalid backslash command: " << command << std::endl;
387  }
388  // clang-format on
389 }
390 
391 std::string scalar_datum_to_string(const TDatum& datum, const TTypeInfo& type_info) {
392  if (datum.is_null) {
393  return "NULL";
394  }
395  switch (type_info.type) {
396  case TDatumType::TINYINT:
397  case TDatumType::SMALLINT:
398  case TDatumType::INT:
399  case TDatumType::BIGINT:
400  return std::to_string(datum.val.int_val);
401  case TDatumType::DECIMAL: {
402  std::ostringstream dout;
403  const auto format_str = "%." + std::to_string(type_info.scale) + "f";
404  dout << boost::format(format_str) % datum.val.real_val;
405  return dout.str();
406  }
407  case TDatumType::DOUBLE: {
408  std::ostringstream dout;
409  dout << std::setprecision(std::numeric_limits<double>::digits10 + 1)
410  << datum.val.real_val;
411  return dout.str();
412  }
413  case TDatumType::FLOAT: {
414  std::ostringstream out;
415  out << std::setprecision(std::numeric_limits<float>::digits10 + 1)
416  << datum.val.real_val;
417  return out.str();
418  }
419  case TDatumType::STR:
420  return datum.val.str_val;
421  case TDatumType::TIME: {
422  time_t t = static_cast<time_t>(datum.val.int_val);
423  std::tm tm_struct;
424  gmtime_r(&t, &tm_struct);
425  char buf[9];
426  strftime(buf, 9, "%T", &tm_struct);
427  return buf;
428  }
429  case TDatumType::TIMESTAMP: {
430  std::tm tm_struct;
431  if (type_info.precision > 0) {
432  auto scale = static_cast<int64_t>(std::pow(10, type_info.precision));
433  auto dv = std::div(datum.val.int_val, scale);
434  auto modulus = (dv.rem + scale) % scale;
435  time_t sec = static_cast<time_t>(dv.quot - (dv.quot < 0 && modulus > 0));
436  gmtime_r(&sec, &tm_struct);
437  char buf[21];
438  strftime(buf, 21, "%F %T.", &tm_struct);
439  auto subsecond = std::to_string(modulus);
440  return std::string(buf) +
441  std::string(type_info.precision - subsecond.length(), '0') + subsecond;
442  } else {
443  time_t sec = static_cast<time_t>(datum.val.int_val);
444  gmtime_r(&sec, &tm_struct);
445  char buf[20];
446  strftime(buf, 20, "%F %T", &tm_struct);
447  return std::string(buf);
448  }
449  }
450  case TDatumType::DATE: {
451  time_t t = static_cast<time_t>(datum.val.int_val);
452  std::tm tm_struct;
453  gmtime_r(&t, &tm_struct);
454  char buf[11];
455  strftime(buf, 11, "%F", &tm_struct);
456  return buf;
457  }
458  case TDatumType::BOOL:
459  return (datum.val.int_val ? "true" : "false");
460  case TDatumType::INTERVAL_DAY_TIME:
461  return std::to_string(datum.val.int_val) + " ms (day-time interval)";
462  case TDatumType::INTERVAL_YEAR_MONTH:
463  return std::to_string(datum.val.int_val) + " month(s) (year-month interval)";
464  default:
465  return "Unknown column type.\n";
466  }
467 }
468 
469 std::string datum_to_string(const TDatum& datum, const TTypeInfo& type_info) {
470  if (datum.is_null) {
471  return "NULL";
472  }
473  if (type_info.is_array) {
474  std::vector<std::string> elem_strs;
475  elem_strs.reserve(datum.val.arr_val.size());
476  for (const auto& elem_datum : datum.val.arr_val) {
477  TTypeInfo elem_type_info{type_info};
478  elem_type_info.is_array = false;
479  elem_strs.push_back(scalar_datum_to_string(elem_datum, elem_type_info));
480  }
481  return "{" + boost::algorithm::join(elem_strs, ", ") + "}";
482  }
483  if (type_info.type == TDatumType::POINT || type_info.type == TDatumType::LINESTRING ||
484  type_info.type == TDatumType::POLYGON ||
485  type_info.type == TDatumType::MULTIPOLYGON) {
486  return datum.val.str_val;
487  }
488  return scalar_datum_to_string(datum, type_info);
489 }
490 
491 TDatum columnar_val_to_datum(const TColumn& col,
492  const size_t row_idx,
493  const TTypeInfo& col_type) {
494  TDatum datum;
495  if (col_type.is_array) {
496  auto elem_type = col_type;
497  elem_type.is_array = false;
498  datum.is_null = col.nulls[row_idx];
499  CHECK_LT(row_idx, col.data.arr_col.size());
500  const auto& arr_col = col.data.arr_col[row_idx];
501  for (size_t elem_idx = 0; elem_idx < arr_col.nulls.size(); ++elem_idx) {
502  datum.val.arr_val.push_back(columnar_val_to_datum(arr_col, elem_idx, elem_type));
503  }
504  return datum;
505  }
506  datum.is_null = col.nulls[row_idx];
507  switch (col_type.type) {
508  case TDatumType::TINYINT:
509  case TDatumType::SMALLINT:
510  case TDatumType::INT:
511  case TDatumType::BIGINT:
512  case TDatumType::TIME:
513  case TDatumType::TIMESTAMP:
514  case TDatumType::DATE:
515  case TDatumType::BOOL:
516  case TDatumType::INTERVAL_DAY_TIME:
517  case TDatumType::INTERVAL_YEAR_MONTH: {
518  datum.val.int_val = col.data.int_col[row_idx];
519  break;
520  }
521  case TDatumType::DECIMAL:
522  case TDatumType::FLOAT:
523  case TDatumType::DOUBLE: {
524  datum.val.real_val = col.data.real_col[row_idx];
525  break;
526  }
527  case TDatumType::STR: {
528  datum.val.str_val = col.data.str_col[row_idx];
529  break;
530  }
531  case TDatumType::POINT:
532  case TDatumType::LINESTRING:
533  case TDatumType::POLYGON:
534  case TDatumType::MULTIPOLYGON: {
535  datum.val.str_val = col.data.str_col[row_idx];
536  break;
537  }
538  default:
539  CHECK(false);
540  }
541  return datum;
542 }
543 
544 // based on http://www.gnu.org/software/libc/manual/html_node/getpass.html
545 std::string mapd_getpass() {
546  struct termios origterm, tmpterm;
547 
548  tcgetattr(STDIN_FILENO, &origterm);
549  tmpterm = origterm;
550  tmpterm.c_lflag &= ~ECHO;
551  tcsetattr(STDIN_FILENO, TCSAFLUSH, &tmpterm);
552 
553  std::cout << "Password: ";
554  std::string password;
555  std::getline(std::cin, password);
556  std::cout << std::endl;
557 
558  tcsetattr(STDIN_FILENO, TCSAFLUSH, &origterm);
559 
560  return password;
561 }
562 
564 
565 bool backchannel(int action, ClientContext* cc, const std::string& ccn = "") {
566  enum State { UNINITIALIZED, INITIALIZED, INTERRUPTIBLE };
567  static int state = UNINITIALIZED;
568  static ClientContext* context{nullptr};
569  static std::string ca_cert_name{};
570 
571  if (action == INITIALIZE) {
572  CHECK(cc);
573  context = cc;
574  ca_cert_name = ccn;
575  state = INITIALIZED;
576  return false;
577  }
578  if (state == INITIALIZED && action == TURN_ON) {
579  state = INTERRUPTIBLE;
580  return false;
581  }
582  if (state == INTERRUPTIBLE && action == TURN_OFF) {
583  state = INITIALIZED;
584  return false;
585  }
586  if (state == INTERRUPTIBLE && action == INTERRUPT) {
587  CHECK(context);
588  mapd::shared_ptr<ThriftClientConnection> connMgr;
589  mapd::shared_ptr<TTransport> transport2;
590  mapd::shared_ptr<TProtocol> protocol2;
591  mapd::shared_ptr<TTransport> socket2;
592  connMgr = std::make_shared<ThriftClientConnection>();
593  if (context->http || context->https) {
594  transport2 = connMgr->open_http_client_transport(context->server_host,
595  context->port,
596  ca_cert_name,
597  context->https,
598  context->skip_host_verify);
599  protocol2 = mapd::shared_ptr<TProtocol>(new TJSONProtocol(transport2));
600  } else {
601  transport2 = connMgr->open_buffered_client_transport(
602  context->server_host, context->port, ca_cert_name);
603  protocol2 = mapd::shared_ptr<TProtocol>(new TBinaryProtocol(transport2));
604  }
605  MapDClient c2(protocol2);
606  ClientContext context2(*transport2, c2);
607 
608  context2.db_name = context->db_name;
609  context2.user_name = context->user_name;
610  context2.passwd = context->passwd;
611 
612  context2.session = INVALID_SESSION_ID;
613 
614  transport2->open();
615  if (!transport2->isOpen()) {
616  std::cout << "Unable to send interrupt to the server.\n" << std::flush;
617  return false;
618  }
619 
620  (void)thrift_with_retry(kCONNECT, context2, nullptr);
621 
622  std::cout << "Asking server to interrupt query.\n" << std::flush;
623  (void)thrift_with_retry(kINTERRUPT, context2, nullptr);
624 
625  if (context2.session != INVALID_SESSION_ID) {
626  (void)thrift_with_retry(kDISCONNECT, context2, nullptr);
627  }
628  transport2->close();
629  state = INITIALIZED;
630  return true;
631  }
632  return false;
633 }
634 
635 void omnisql_signal_handler(int signal_number) {
636  std::cout << "\nInterrupt signal (" << signal_number << ") received.\n" << std::flush;
637 
638  if (backchannel(INTERRUPT, nullptr)) {
639  return;
640  }
641 
642  // terminate program
643  exit(signal_number);
644 }
645 
647  signal(SIGTERM, omnisql_signal_handler);
648  signal(SIGKILL, omnisql_signal_handler);
649  signal(SIGINT, omnisql_signal_handler);
650 }
651 
652 void print_memory_summary(ClientContext& context, std::string memory_level) {
653  std::ostringstream tss;
654  std::vector<TNodeMemoryInfo> memory_info;
655  std::string sub_system;
656  std::string cur_host = "^";
657  bool hasGPU = false;
658  bool multiNode = context.cpu_memory.size() > 1;
659  if (!memory_level.compare("gpu")) {
660  memory_info = context.gpu_memory;
661  sub_system = "GPU";
662  hasGPU = true;
663  } else {
664  memory_info = context.cpu_memory;
665  sub_system = "CPU";
666  }
667 
668  tss << "OmniSci Server " << sub_system << " Memory Summary:" << std::endl;
669 
670  if (multiNode) {
671  if (hasGPU) {
672  tss << " NODE[GPU] MAX USE ALLOCATED "
673  "FREE"
674  << std::endl;
675  } else {
676  tss << " NODE MAX USE ALLOCATED FREE"
677  << std::endl;
678  }
679  } else {
680  if (hasGPU) {
681  tss << "[GPU] MAX USE ALLOCATED FREE"
682  << std::endl;
683  } else {
684  tss << " MAX USE ALLOCATED FREE" << std::endl;
685  }
686  }
687  u_int16_t gpu_num = 0;
688  for (auto& nodeIt : memory_info) {
689  int MB = 1024 * 1024;
690  u_int64_t page_count = 0;
691  u_int64_t free_page_count = 0;
692  u_int64_t used_page_count = 0;
693  for (auto& segIt : nodeIt.node_memory_data) {
694  page_count += segIt.num_pages;
695  if (segIt.is_free) {
696  free_page_count += segIt.num_pages;
697  }
698  }
699  if (context.cpu_memory.size() > 1) {
700  }
701  used_page_count = page_count - free_page_count;
702  if (cur_host.compare(nodeIt.host_name)) {
703  gpu_num = 0;
704  cur_host = nodeIt.host_name;
705  } else {
706  ++gpu_num;
707  }
708  if (multiNode) {
709  tss << std::setfill(' ') << std::setw(12) << nodeIt.host_name;
710  if (hasGPU) {
711  tss << std::setfill(' ') << std::setw(3);
712  tss << "[" << gpu_num << "]";
713  } else {
714  }
715  } else {
716  if (hasGPU) {
717  tss << std::setfill(' ') << std::setw(3);
718  tss << "[" << gpu_num << "]";
719  } else {
720  }
721  }
722  tss << std::fixed;
723  tss << std::setprecision(2);
724  tss << std::setfill(' ') << std::setw(12)
725  << ((float)nodeIt.page_size * nodeIt.max_num_pages) / MB << " MB";
726  tss << std::setfill(' ') << std::setw(12)
727  << ((float)nodeIt.page_size * used_page_count) / MB << " MB";
728  tss << std::setfill(' ') << std::setw(12)
729  << ((float)nodeIt.page_size * page_count) / MB << " MB";
730  tss << std::setfill(' ') << std::setw(12)
731  << ((float)nodeIt.page_size * free_page_count) / MB << " MB";
732  if (nodeIt.is_allocation_capped) {
733  tss << " The allocation is capped!";
734  }
735  tss << std::endl;
736  }
737  std::cout << tss.str() << std::endl;
738 }
739 
740 void print_memory_info(ClientContext& context, std::string memory_level) {
741  int MB = 1024 * 1024;
742  std::ostringstream tss;
743  std::vector<TNodeMemoryInfo> memory_info;
744  std::string sub_system;
745  std::string cur_host = "^";
746  bool multiNode;
747  int mgr_num = 0;
748  if (!memory_level.compare("gpu")) {
749  memory_info = context.gpu_memory;
750  sub_system = "GPU";
751  multiNode = context.gpu_memory.size() > 1;
752  } else {
753  memory_info = context.cpu_memory;
754  sub_system = "CPU";
755  multiNode = context.cpu_memory.size() > 1;
756  }
757 
758  tss << "OmniSci Server Detailed " << sub_system << " Memory Usage:" << std::endl;
759  for (auto& nodeIt : memory_info) {
760  if (cur_host.compare(nodeIt.host_name)) {
761  mgr_num = 0;
762  cur_host = nodeIt.host_name;
763  if (multiNode) {
764  tss << "Node: " << nodeIt.host_name << std::endl;
765  }
766  tss << "Maximum Bytes for one page: " << nodeIt.page_size << " Bytes" << std::endl;
767  tss << "Maximum Bytes for node: " << (nodeIt.max_num_pages * nodeIt.page_size) / MB
768  << " MB" << std::endl;
769  tss << "Memory allocated: " << (nodeIt.num_pages_allocated * nodeIt.page_size) / MB
770  << " MB" << std::endl;
771  } else {
772  ++mgr_num;
773  }
774  cur_host = nodeIt.host_name;
775 
776  tss << sub_system << "[" << mgr_num << "]"
777  << " Slab Information:" << std::endl;
778  if (nodeIt.is_allocation_capped) {
779  tss << "The allocation is capped!";
780  }
781  tss << "SLAB ST_PAGE NUM_PAGE TOUCH CHUNK_KEY" << std::endl;
782  for (auto segIt = nodeIt.node_memory_data.begin();
783  segIt != nodeIt.node_memory_data.end();
784  ++segIt) {
785  tss << std::setfill(' ') << std::setw(4) << segIt->slab;
786  tss << std::setfill(' ') << std::setw(12) << segIt->start_page;
787  tss << std::setfill(' ') << std::setw(9) << segIt->num_pages;
788  tss << std::setfill(' ') << std::setw(7) << segIt->touch;
789  tss << std::setfill(' ') << std::setw(5);
790 
791  if (segIt->is_free) {
792  tss << "FREE";
793  } else {
794  tss << "USED";
795  tss << std::setfill(' ') << std::setw(5);
796  for (auto& vecIt : segIt->chunk_key) {
797  tss << vecIt << ",";
798  }
799  }
800 
801  tss << std::endl;
802  }
803  tss << "---------------------------------------------------------------" << std::endl;
804  }
805  std::cout << tss.str() << std::endl;
806 }
807 
808 std::string print_gpu_specification(TGpuSpecification gpu_spec) {
809  int Giga = 1024 * 1024 * 1024;
810  int Kilo = 1024;
811  std::ostringstream tss;
812  tss << "Number of SM :" << gpu_spec.num_sm << std::endl;
813  tss << "Clock frequency :" << gpu_spec.clock_frequency_kHz / Kilo << " MHz"
814  << std::endl;
815  tss << "Physical GPU Memory :" << gpu_spec.memory / Giga << " GB" << std::endl;
816  tss << "Compute capability :" << gpu_spec.compute_capability_major << "."
817  << gpu_spec.compute_capability_minor << std::endl;
818  return tss.str();
819 }
820 
821 std::string print_hardware_specification(THardwareInfo hw_spec) {
822  std::ostringstream tss;
823  if (hw_spec.host_name != "") {
824  tss << "Host name :" << hw_spec.host_name << std::endl;
825  }
826  tss << "Number of Physical GPUs :" << hw_spec.num_gpu_hw << std::endl;
827  tss << "Number of CPU core :" << hw_spec.num_cpu_hw << std::endl;
828  tss << "Number of GPUs allocated :" << hw_spec.num_gpu_allocated << std::endl;
829  tss << "Start GPU :" << hw_spec.start_gpu << std::endl;
830  for (auto gpu_spec : hw_spec.gpu_info) {
831  tss << "-------------------------------------------" << std::endl;
832  tss << print_gpu_specification(gpu_spec);
833  }
834  tss << "-------------------------------------------" << std::endl;
835  return tss.str();
836 }
837 
839  std::ostringstream tss;
840  for (auto hw_info : context.cluster_hardware_info.hardware_info) {
841  tss << "===========================================" << std::endl;
842  tss << print_hardware_specification(hw_info) << std::endl;
843  }
844  tss << "===========================================" << std::endl;
845  std::cout << tss.str();
846 }
847 
848 static void print_privs(const std::vector<bool>& privs, TDBObjectType::type type) {
849  // Client priviliges print lookup
850  static const std::unordered_map<const TDBObjectType::type,
851  const std::vector<std::string>>
852  privilege_names_lookup{
854  {" create"s, " drop"s, " view-sql-editor"s, " login-access"s}},
856  {" create"s,
857  " drop"s,
858  " select"s,
859  " insert"s,
860  " update"s,
861  " delete"s,
862  " truncate"s,
863  " alter"s}},
865  {" create"s, " delete"s, " view"s, " edit"s}},
867  {" create"s, " drop"s, " select"s, " insert"s, " update"s, " delete"s}}};
868 
869  const auto privilege_names = privilege_names_lookup.find(type);
870  size_t i = 0;
871  if (privilege_names != privilege_names_lookup.end()) {
872  for (const auto& priv : privs) {
873  if (priv) {
874  std::cout << privilege_names->second[i];
875  }
876  ++i;
877  }
878  }
879 }
880 
882  context.db_objects.clear();
883  if (thrift_with_retry(kGET_OBJECTS_FOR_GRANTEE, context, nullptr)) {
884  for (const auto& db_object : context.db_objects) {
885  bool any_granted_privs = false;
886  for (size_t j = 0; j < db_object.privs.size(); j++) {
887  if (db_object.privs[j]) {
888  any_granted_privs = true;
889  break;
890  }
891  }
892  if (!any_granted_privs) {
893  continue;
894  }
895  std::cout << db_object.objectName.c_str();
896  switch (db_object.objectType) {
898  std::cout << " (database):";
899  break;
900  }
902  std::cout << " (table):";
903  break;
904  }
906  std::cout << " (dashboard):";
907  break;
908  }
910  std::cout << " (view):";
911  break;
912  }
913  default: {
914  CHECK(false);
915  }
916  }
917  print_privs(db_object.privs, db_object.objectType);
918  std::cout << std::endl;
919  }
920  }
921 }
922 
924  context.db_objects.clear();
925  if (thrift_with_retry(kGET_OBJECT_PRIVS, context, nullptr)) {
926  for (const auto& db_object : context.db_objects) {
927  if (boost::to_upper_copy<std::string>(context.privs_object_name)
928  .compare(boost::to_upper_copy<std::string>(db_object.objectName))) {
929  continue;
930  }
931  bool any_granted_privs = false;
932  for (size_t j = 0; j < db_object.privs.size(); j++) {
933  if (db_object.privs[j]) {
934  any_granted_privs = true;
935  break;
936  }
937  }
938  if (!any_granted_privs) {
939  continue;
940  }
941  std::cout << db_object.grantee << " privileges:";
942  print_privs(db_object.privs, db_object.objectType);
943  std::cout << std::endl;
944  }
945  }
946 }
947 
948 void set_license_key(ClientContext& context, const std::string& token) {
949  context.license_key = token;
950  if (thrift_with_retry(kSET_LICENSE_KEY, context, nullptr)) {
951  for (auto claims : context.license_info.claims) {
952  std::vector<std::string> jwt;
953  boost::split(jwt, claims, boost::is_any_of("."));
954  if (jwt.size() > 1) {
955  std::cout << mapd::decode_base64(jwt[1]) << std::endl;
956  }
957  }
958  }
959 }
960 
962  if (thrift_with_retry(kGET_LICENSE_CLAIMS, context, nullptr)) {
963  for (auto claims : context.license_info.claims) {
964  std::vector<std::string> jwt;
965  boost::split(jwt, claims, boost::is_any_of("."));
966  if (jwt.size() > 1) {
967  std::cout << mapd::decode_base64(jwt[1]) << std::endl;
968  }
969  }
970  }
971 }
972 
973 std::string hide_sensitive_data_from_connect(const std::string& connect_str) {
974  auto result = connect_str;
975  boost::regex passwd{R"(^\\c\s+?.+?\s+?.+?\s+?(?<pwd>.+))",
976  boost::regex_constants::perl | boost::regex::icase};
977  boost::smatch matches;
978  if (boost::regex_search(connect_str, matches, passwd)) {
979  result.replace(
980  matches["pwd"].first - connect_str.begin(), matches["pwd"].length(), "XXXXXXXX");
981  }
982  return result;
983 }
984 
985 std::string get_process_role(TRole::type role) {
986  switch (role) {
987  case TRole::type::SERVER:
988  return "Server";
989  case TRole::type::AGGREGATOR:
990  return "Aggregator";
991  case TRole::type::LEAF:
992  return "Leaf";
993  case TRole::type::STRING_DICTIONARY:
994  return "String Dictionary Server";
995  }
996  UNREACHABLE();
997  return "";
998 }
999 
1000 void print_status(ClientContext& context) {
1001  std::ostringstream tss;
1002  const size_t lhs_width = 36;
1003  time_t t = (time_t)context.cluster_status[0].start_time;
1004  std::tm* tm_ptr = gmtime(&t);
1005  char buf[12] = {0};
1006  strftime(buf, 11, "%F", tm_ptr);
1007  const std::string agg_version = context.cluster_status[0].version;
1008  const bool is_cluster = context.cluster_status.size() > 1;
1009 
1010  std::string edition = "";
1011  if (context.cluster_status[0].edition == "ee") {
1012  edition = "Enterprise Edition";
1013  }
1014 
1015  const std::string deployment_type = (is_cluster) ? "Cluster" : "Server";
1016 
1017  tss << std::left << std::setfill(' ') << std::setw(lhs_width);
1018  tss << deployment_type + " Version"
1019  << ": " << agg_version << " " << edition;
1020  tss << std::endl;
1021 
1022  if (is_cluster) {
1023  tss << std::left << std::setfill(' ') << std::setw(lhs_width);
1024  tss << "Number of processes"
1025  << ": " << context.cluster_status.size() << std::endl;
1026  }
1027 
1028  for (auto node = context.cluster_status.begin(); node != context.cluster_status.end();
1029  ++node) {
1030  const std::string process_role = get_process_role(node->role);
1031 
1032  t = (time_t)node->start_time;
1033  memset(buf, 0, 12);
1034  std::tm* tm_ptr = gmtime(&t);
1035  strftime(buf, 11, "%F", tm_ptr);
1036 
1037  tss << "--------------------------------------------------" << std::endl;
1038 
1039  tss << std::left << std::setfill(' ') << std::setw(lhs_width);
1040  tss << process_role + " Name"
1041  << ": " << node->host_name << std::endl;
1042 
1043  tss << std::left << std::setfill(' ') << std::setw(lhs_width);
1044  tss << process_role + " Start Time"
1045  << ": " << buf << " : " << tm_ptr->tm_hour << ":" << tm_ptr->tm_min << ":"
1046  << tm_ptr->tm_sec << std::endl;
1047 
1048  if (agg_version != node->version) {
1049  tss << std::left << std::setfill(' ') << std::setw(lhs_width);
1050  tss << process_role + " Version "
1051  << ": " << node->version << std::endl
1052  << "\033[31m*** Version mismatch! ***\033[0m Please make "
1053  "sure All leaves, Aggregator and String Dictionary are running "
1054  "the same version of OmniSci."
1055  << std::endl;
1056  }
1057  }
1058  std::cout << tss.str() << std::endl;
1059 }
1060 
1061 } // namespace
1062 
1063 int main(int argc, char** argv) {
1064  std::string server_host{"localhost"};
1065  int port = 6274;
1066  std::string delimiter("|");
1067  bool print_header = true;
1068  bool print_connection = true;
1069  bool print_timing = false;
1070  bool http = false;
1071  bool https = false;
1072  bool skip_host_verify = false;
1073  TQueryResult _return;
1074  std::string db_name;
1075  std::string user_name{"admin"};
1076  std::string ca_cert_name{""};
1077  std::string passwd;
1078  CommandHistoryFile cmd_file;
1079 
1080  namespace po = boost::program_options;
1081 
1082  po::options_description desc("Options");
1083  desc.add_options()("help,h", "Print help messages ");
1084  desc.add_options()("version,v", "Print omnisql version number");
1085  desc.add_options()("no-header,n", "Do not print query result header");
1086  desc.add_options()(
1087  "timing,t",
1088  po::bool_switch(&print_timing)->default_value(print_timing)->implicit_value(true),
1089  "Print timing information");
1090  desc.add_options()("delimiter,d",
1091  po::value<std::string>(&delimiter)->default_value(delimiter),
1092  "Field delimiter in row output");
1093  desc.add_options()("db",
1094  po::value<std::string>(&db_name),
1095  "Database name (server default is omnisci)");
1096  desc.add_options()("user,u",
1097  po::value<std::string>(&user_name)->default_value(user_name),
1098  "User name");
1099  desc.add_options()(
1100  "ca-cert",
1101  po::value<std::string>(&ca_cert_name)->default_value(ca_cert_name),
1102  "Path to trusted server certificate. Initiates an encrypted connection");
1103  desc.add_options()("passwd,p", po::value<std::string>(&passwd), "Password");
1104  desc.add_options()(
1105  "history", po::value<CommandHistoryFile>(&cmd_file), "History filename");
1106  desc.add_options()("server,s",
1107  po::value<std::string>(&server_host)->default_value(server_host),
1108  "Server hostname");
1109  desc.add_options()("port", po::value<int>(&port)->default_value(port), "Port number");
1110  desc.add_options()("http",
1111  po::bool_switch(&http)->default_value(http)->implicit_value(true),
1112  "Use HTTP transport");
1113  desc.add_options()("https",
1114  po::bool_switch(&https)->default_value(https)->implicit_value(true),
1115  "Use HTTPS transport");
1116  desc.add_options()("skip-verify",
1117  po::bool_switch(&skip_host_verify)
1118  ->default_value(skip_host_verify)
1119  ->implicit_value(true),
1120  "Don't verify SSL certificate validity");
1121  desc.add_options()("quiet,q", "Do not print result headers or connection strings ");
1122 
1123  po::variables_map vm;
1124  po::positional_options_description positionalOptions;
1125  positionalOptions.add("db", 1);
1126 
1127  try {
1128  po::store(po::command_line_parser(argc, argv)
1129  .options(desc)
1130  .positional(positionalOptions)
1131  .run(),
1132  vm);
1133  if (vm.count("help")) {
1134  std::cout << "Usage: omnisql [<database>] [{--user|-u} <user>] [{--passwd|-p} "
1135  "<password>] [--port <port number>] "
1136  "[{-s|--server} <server host>] [--http] [{--no-header|-n}] "
1137  "[{--quiet|-q}] [{--delimiter|-d}]\n\n";
1138  std::cout << desc << std::endl;
1139  return 0;
1140  }
1141  if (vm.count("version")) {
1142  std::cout << "OmniSQL Version: " << OmniSQLRelease << std::endl;
1143  return 0;
1144  }
1145  if (vm.count("quiet")) {
1146  print_header = false;
1147  print_connection = false;
1148  }
1149  if (vm.count("no-header")) {
1150  print_header = false;
1151  }
1152  if (vm.count("db") && !vm.count("user")) {
1153  std::cerr << "Must specify a user name to access database " << db_name << std::endl;
1154  return 1;
1155  }
1156 
1157  po::notify(vm);
1158  } catch (boost::program_options::error& e) {
1159  std::cerr << "Usage Error: " << e.what() << std::endl;
1160  return 1;
1161  }
1162 
1163  if (!vm.count("passwd")) {
1164  passwd = mapd_getpass();
1165  }
1166  mapd::shared_ptr<ThriftClientConnection> connMgr;
1167  connMgr = std::make_shared<ThriftClientConnection>();
1168  mapd::shared_ptr<TTransport> transport;
1169  mapd::shared_ptr<TProtocol> protocol;
1170  mapd::shared_ptr<TTransport> socket;
1171  if (https || http) {
1172  transport = connMgr->open_http_client_transport(
1173  server_host, port, ca_cert_name, https, skip_host_verify);
1174  protocol = mapd::shared_ptr<TProtocol>(new TJSONProtocol(transport));
1175  } else {
1176  transport = connMgr->open_buffered_client_transport(server_host, port, ca_cert_name);
1177  protocol = mapd::shared_ptr<TProtocol>(new TBinaryProtocol(transport));
1178  }
1179  MapDClient c(protocol);
1180  ClientContext context(*transport, c);
1181  g_client_context_ptr = &context;
1182 
1183  context.db_name = db_name;
1184  context.user_name = user_name;
1185  context.passwd = passwd;
1186  context.server_host = server_host;
1187  context.port = port;
1188  context.http = http;
1189  context.https = https;
1190  context.skip_host_verify = skip_host_verify;
1191  context.session = INVALID_SESSION_ID;
1192 
1193  try {
1194  transport->open();
1195  } catch (...) {
1196  std::cout << "Failed to open transport. Is omnisci_server running?" << std::endl;
1197  return 1;
1198  }
1199 
1200  if (thrift_with_retry(kCONNECT, context, nullptr)) {
1201  if (print_connection) {
1202  std::cout << "User " << context.user_name << " connected to database "
1203  << context.db_name << std::endl;
1204  }
1205  }
1206  if (context.db_name.empty()) {
1207  std::cout << "Not connected to any database. See \\h for help." << std::endl;
1208  }
1209 
1211  (void)backchannel(INITIALIZE, &context, ca_cert_name);
1212 
1213  /* Set the completion callback. This will be called every time the
1214  * user uses the <tab> key. */
1215  linenoiseSetCompletionCallback(completion);
1216 
1217  /* Load history from file. The history file is just a plain text file
1218  * where entries are separated by newlines. */
1219  linenoiseHistoryLoad(cmd_file); /* Load the history at startup */
1220  /* default to multi-line mode */
1221  linenoiseSetMultiLine(1);
1222 
1223  std::string current_line;
1224  std::string prompt("omnisql> ");
1225 
1226  /* Now this is the main loop of the typical linenoise-based application.
1227  * The call to linenoise() will block as long as the user types something
1228  * and presses enter.
1229  *
1230  * The typed string that is malloc() allocated by linenoise() is managed by a smart
1231  * pointer with a custom free() deleter; no need to free this memory explicitly. */
1232 
1233  while (true) {
1234  using LineType = std::remove_pointer<__decltype(linenoise(prompt.c_str()))>::type;
1235  using LineTypePtr = LineType*;
1236 
1237  std::unique_ptr<LineType, std::function<void(LineTypePtr)>> smart_line(
1238  linenoise(prompt.c_str()), [](LineTypePtr l) { free((l)); });
1239  if (smart_line == nullptr) {
1240  break;
1241  }
1242  LineType* line = smart_line.get(); // Alias to make the C stuff work
1243 
1244  {
1245  TQueryResult empty;
1246  swap(_return, empty);
1247  }
1248 
1249  /* Do something with the string. */
1250  if (line[0] != '\0' && line[0] != '\\') {
1251  // printf("echo: '%s'\n", line);
1252  if (context.session == INVALID_SESSION_ID) {
1253  std::cerr << "Not connected to any OmniSci databases." << std::endl;
1254  continue;
1255  }
1256  std::string trimmed_line = std::string(line);
1257  boost::algorithm::trim(trimmed_line);
1258  current_line.append(" ").append(trimmed_line);
1259  boost::algorithm::trim(current_line);
1260  if (current_line.back() == ';') {
1261  std::string query(current_line);
1262  linenoiseHistoryAdd(hide_sensitive_data_from_query(current_line).c_str());
1263  linenoiseHistorySave(cmd_file);
1264  current_line.clear();
1265  prompt.assign("omnisql> ");
1266  (void)backchannel(TURN_ON, nullptr);
1267  if (thrift_with_retry(kSQL, context, query.c_str())) {
1268  (void)backchannel(TURN_OFF, nullptr);
1269  if (context.query_return.row_set.row_desc.empty()) {
1270  continue;
1271  }
1272  const size_t row_count{get_row_count(context.query_return)};
1273  if (!row_count) {
1274  static const std::string insert{"INSERT"};
1275  std::string verb(query, 0, insert.size());
1276  if (!boost::iequals(verb, insert)) {
1277  std::cout << "No rows returned." << std::endl;
1278  }
1279  if (print_timing) {
1280  std::cout << "Execution time: " << context.query_return.execution_time_ms
1281  << " ms,"
1282  << " Total time: " << context.query_return.total_time_ms << " ms"
1283  << std::endl;
1284  }
1285  continue;
1286  }
1287  bool not_first = false;
1288  if (print_header) {
1289  for (auto p : context.query_return.row_set.row_desc) {
1290  if (p.is_physical) {
1291  // TODO(d): skip if we decide to suppress displaying physical columns
1292  }
1293  if (not_first) {
1294  std::cout << delimiter;
1295  } else {
1296  not_first = true;
1297  }
1298  std::cout << p.col_name;
1299  }
1300  std::cout << std::endl;
1301  }
1302  for (size_t row_idx = 0; row_idx < row_count; ++row_idx) {
1303  const auto& col_desc = context.query_return.row_set.row_desc;
1304  for (size_t col_idx = 0; col_idx < col_desc.size(); ++col_idx) {
1305  if (col_idx) {
1306  std::cout << delimiter;
1307  }
1308  const auto& col_type = col_desc[col_idx].col_type;
1309  std::cout << datum_to_string(
1311  context.query_return.row_set.columns[col_idx], row_idx, col_type),
1312  col_type);
1313  }
1314  std::cout << std::endl;
1315  }
1316  if (print_timing) {
1317  std::cout << row_count << " rows returned." << std::endl;
1318  std::cout << "Execution time: " << context.query_return.execution_time_ms
1319  << " ms,"
1320  << " Total time: " << context.query_return.total_time_ms << " ms"
1321  << std::endl;
1322  }
1323  } else {
1324  (void)backchannel(TURN_OFF, nullptr);
1325  }
1326  } else {
1327  // change the prommpt
1328  prompt.assign("..> ");
1329  }
1330  continue;
1331  } else if (!strncmp(line, "\\interrupt", 10)) {
1332  (void)thrift_with_retry(kINTERRUPT, context, nullptr);
1333  } else if (!strncmp(line, "\\cpu", 4)) {
1335  (void)thrift_with_retry(kSET_EXECUTION_MODE, context, nullptr);
1336  } else if (!strncmp(line, "\\gpu", 4)) {
1338  (void)thrift_with_retry(kSET_EXECUTION_MODE, context, nullptr);
1339  } else if (!strncmp(line, "\\hybrid", 5)) {
1340  std::cout << "Hybrid execution mode has been deprecated." << std::endl;
1341  } else if (!strncmp(line, "\\version", 8)) {
1342  if (thrift_with_retry(kGET_VERSION, context, nullptr)) {
1343  std::cout << "OmniSci Server Version: " << context.version << std::endl;
1344  } else {
1345  std::cout << "Cannot connect to OmniSci Server." << std::endl;
1346  }
1347  } else if (!strncmp(line, "\\memory_gpu", 11)) {
1348  if (thrift_with_retry(kGET_MEMORY_GPU, context, nullptr)) {
1349  print_memory_info(context, "gpu");
1350  } else {
1351  std::cout << "Cannot connect to OmniSci Server." << std::endl;
1352  }
1353  } else if (!strncmp(line, "\\memory_cpu", 11)) {
1354  if (thrift_with_retry(kGET_MEMORY_CPU, context, nullptr)) {
1355  print_memory_info(context, "cpu");
1356  }
1357  } else if (!strncmp(line, "\\clear_gpu", 11)) {
1358  if (thrift_with_retry(kCLEAR_MEMORY_GPU, context, nullptr)) {
1359  std::cout << "OmniSci Server GPU memory Cleared " << std::endl;
1360  }
1361  } else if (!strncmp(line, "\\clear_cpu", 11)) {
1362  if (thrift_with_retry(kCLEAR_MEMORY_CPU, context, nullptr)) {
1363  std::cout << "OmniSci Server CPU memory Cleared " << std::endl;
1364  }
1365  } else if (!strncmp(line, "\\memory_summary", 11)) {
1366  if (thrift_with_retry(kGET_MEMORY_SUMMARY, context, nullptr)) {
1367  print_memory_summary(context, "cpu");
1368  print_memory_summary(context, "gpu");
1369  }
1370  } else if (!strncmp(line, "\\hardware_info", 13)) {
1371  if (context.cluster_hardware_info.hardware_info.size() > 0 ||
1372  thrift_with_retry(kGET_HARDWARE_INFO, context, nullptr)) {
1373  // TODO(vraj): try not to abuse using short circuit here
1374  print_all_hardware_info(context);
1375  }
1376  } else if (!strncmp(line, "\\status", 8)) {
1377  if (thrift_with_retry(kGET_SERVER_STATUS, context, nullptr)) {
1378  print_status(context);
1379  }
1380  } else if (!strncmp(line, "\\detect", 7)) {
1381  char* filepath = strtok(line + 8, " ");
1382  TCopyParams copy_params;
1383 #ifdef ENABLE_IMPORT_PARQUET
1384  // users may give only a directory name which has no obvious connection to parquet.
1385  // need users to explicitly specify that they want to detect cols in parquet files
1386  if (boost::iequals(filepath, "parquet")) {
1387  copy_params.file_type = TFileType::PARQUET;
1388  filepath = strtok(0, " ");
1389  }
1390 #endif
1391  copy_params.delimiter = ",";
1392  char* env;
1393  if (nullptr != (env = getenv("AWS_REGION"))) {
1394  copy_params.s3_region = env;
1395  }
1396  if (nullptr != (env = getenv("AWS_ACCESS_KEY_ID"))) {
1397  copy_params.s3_access_key = env;
1398  }
1399  if (nullptr != (env = getenv("AWS_SECRET_ACCESS_KEY"))) {
1400  copy_params.s3_secret_key = env;
1401  }
1402  detect_table(filepath, copy_params, context);
1403  } else if (!strncmp(line, "\\historylen", 11)) {
1404  /* The "/historylen" command will change the history len. */
1405  int len = atoi(line + 11);
1406  linenoiseHistorySetMaxLen(len);
1407  } else if (!strncmp(line, "\\privileges", 11)) {
1408  std::string temp_line(line);
1409  boost::algorithm::trim(temp_line);
1410  if (temp_line.size() > 11) {
1411  context.privs_role_name.clear();
1412  context.privs_role_name = strtok(line + 12, " ");
1413  if (!context.privs_role_name.compare(OMNISCI_ROOT_USER)) {
1414  std::cerr << "Command privileges failed because " << OMNISCI_ROOT_USER
1415  << " root user has all privileges by default." << std::endl;
1416  } else {
1417  get_db_objects_for_grantee(context);
1418  }
1419  } else {
1420  std::cerr << "Command privileges failed because parameter role name or user name "
1421  "is missing."
1422  << std::endl;
1423  }
1424  } else if (!strncmp(line, "\\object_privileges", 18)) {
1425  std::string cmd(line);
1426  boost::trim(cmd);
1427  std::vector<std::string> args;
1428  boost::split(args, cmd, boost::is_any_of("\t "), boost::token_compress_on);
1429  if (args.size() == 3) {
1430  context.privs_object_name = args[2];
1431  if (args[1] == "database") {
1433  get_db_object_privs(context);
1434  } else if (args[1] == "table") {
1436  get_db_object_privs(context);
1437  } else {
1438  std::cerr << "Object type should be on in { database, table }" << std::endl;
1439  }
1440  } else {
1441  std::cerr << "Command object_privileges failed. It requires two parameters: "
1442  "object type and object name."
1443  << std::endl;
1444  }
1445  } else if (line[0] == '\\' && line[1] == 'q') {
1446  break;
1447  } else { // Experimental Cleanup
1449 
1450  // clang-format off
1451  auto resolution_status = CommandResolutionChain<>( line, "\\copygeo", 1, 4, CopyGeoCmd<>(context), "") // deprecated
1452  ( "\\copy", 3, 3, [&](Params const& p) { copy_table(p[1].c_str() /* filepath */, p[2].c_str() /* table */, context); } )
1453  ( "\\ste", 2, 2, [&](Params const& p) { set_table_epoch(context, p[1] /* table_details */); } )
1454  ( "\\gte", 2, 2, [&](Params const& p) { get_table_epoch(context, p[1] /* table_details */); } )
1455  ( "\\export_dashboard", 3, 4, ExportDashboardCmd<>( context ), "Usage \\export_dashboard <dash name> <file name> <optional:dash_owner>" )
1456  ( "\\import_dashboard", 3, 3, ImportDashboardCmd<>( context ), "Usage \\import_dashboard <dash name> <file name>" )
1457  ( "\\role_list", 2, 2, RoleListCmd<>(context), "Usage: \\role_list <userName>")
1458  ( "\\roles", 1, 1, RolesCmd<>(context))("\\set_license", 2, 2, [&](Params const& p ) { set_license_key(context, p[1]); })
1459  ( "\\get_license", 1, 1, [&](Params const&) { get_license_claims(context); })
1460  ( "\\status", 1, 1, StatusCmd<>( context ), "Usage \\status" )
1461  ( "\\dash", 1, 1, ListDashboardsCmd<>( context ) )
1462  ( "\\multiline", 1, 1, [&](Params const&) { linenoiseSetMultiLine(1); } )
1463  ( "\\singleline", 1, 1, [&](Params const&) { linenoiseSetMultiLine(0); } )
1464  ( "\\keycodes", 1, 1, [&](Params const&) { linenoisePrintKeyCodes(); } )
1465  ( "\\timing", 1, 1, [&](Params const&) { print_timing = true; } )
1466  ( "\\notiming", 1, 1, [&](Params const&) { print_timing = false; } )
1467  ( "\\db", 1, 2, SwitchDatabaseCmd<>( context ), "Usage: \\db [database|...]" )
1468  .is_resolved();
1469 
1470  if (resolution_status == false) {
1471  if (line[0] == '\\' && line[1] == 'q') {
1472  break;
1473  } else if (line[0] == '\\') {
1474  process_backslash_commands(line, context);
1475  }
1476  }
1477  // clang-format on
1478  }
1479 
1480  /* Add to the history. */
1481  if (line[0] == '\\' && line[1] == 'c') {
1482  linenoiseHistoryAdd(hide_sensitive_data_from_connect(line).c_str());
1483  } else {
1484  linenoiseHistoryAdd(hide_sensitive_data_from_query(line).c_str());
1485  }
1486  linenoiseHistorySave(cmd_file);
1487  }
1488 
1489  if (context.session != INVALID_SESSION_ID) {
1490  if (thrift_with_retry(kDISCONNECT, context, nullptr)) {
1491  if (print_connection) {
1492  std::cout << "User " << context.user_name << " disconnected from database "
1493  << context.db_name << std::endl;
1494  }
1495  }
1496  }
1497  transport->close();
1498 
1499  return 0;
1500 }
1501 
TExecuteMode::type execution_mode
std::string hide_sensitive_data_from_query(std::string const &query_str)
ClientContext * g_client_context_ptr
Definition: omnisql.cpp:76
#define CHECK_EQ(x, y)
Definition: Logger.h:195
std::string table_name
HOST DEVICE int get_size() const
Definition: sqltypes.h:333
void print_status(ClientContext &context)
Definition: omnisql.cpp:1000
void get_license_claims(ClientContext &context)
Definition: omnisql.cpp:961
std::string get_process_role(TRole::type role)
Definition: omnisql.cpp:985
std::vector< TNodeMemoryInfo > cpu_memory
void completion(const char *buf, linenoiseCompletions *lc)
Definition: omnisql.cpp:78
std::string server_host
std::string datum_to_string(const TargetValue &tv, const SQLTypeInfo &ti, const std::string &delim)
static void print_privs(const std::vector< bool > &privs, TDBObjectType::type type)
Definition: omnisql.cpp:848
std::string privs_role_name
void c(const std::string &query_string, const ExecutorDeviceType device_type)
void get_table_epoch(ClientContext &context, const std::string &table_specifier)
Definition: omnisql.cpp:220
std::string join(T const &container, std::string const &delim)
void get_db_objects_for_grantee(ClientContext &context)
Definition: omnisql.cpp:881
#define UNREACHABLE()
Definition: Logger.h:231
void print_memory_info(ClientContext &context, std::string memory_level)
Definition: omnisql.cpp:740
void get_db_object_privs(ClientContext &context)
Definition: omnisql.cpp:923
TLicenseInfo license_info
const std::string OmniSQLRelease(MAPD_RELEASE)
std::string scalar_datum_to_string(const TDatum &datum, const TTypeInfo &type_info)
Definition: omnisql.cpp:391
TQueryResult query_return
TTableDetails table_details
#define INVALID_SESSION_ID
size_t get_row_count(const TQueryResult &query_result)
Definition: omnisql.cpp:211
std::string to_string(char const *&&v)
TDBObjectType::type object_type
void print_memory_summary(ClientContext &context, std::string memory_level)
Definition: omnisql.cpp:652
std::string get_type_name() const
Definition: sqltypes.h:426
void set_license_key(ClientContext &context, const std::string &token)
Definition: omnisql.cpp:948
std::vector< std::string > names_return
bool is_array() const
Definition: sqltypes.h:458
std::vector< TCompletionHint > completion_hints
std::string print_gpu_specification(TGpuSpecification gpu_spec)
Definition: omnisql.cpp:808
std::vector< TNodeMemoryInfo > gpu_memory
int main(int argc, char **argv)
Definition: omnisql.cpp:1063
SQLTypeInfo type_info_from_thrift(const TTypeInfo &thrift_ti, const bool strip_geo_encoding=false)
void process_backslash_commands(char *command, ClientContext &context)
Definition: omnisql.cpp:373
#define LOAD_PATCH_SIZE
Definition: omnisql.cpp:91
std::string decode_base64(const std::string &val, bool trim_nulls)
Definition: base64.h:26
TDatum columnar_val_to_datum(const TColumn &col, const size_t row_idx, const TTypeInfo &col_type)
Definition: omnisql.cpp:491
bool backchannel(int action, ClientContext *cc, const std::string &ccn="")
Definition: omnisql.cpp:565
std::string print_hardware_specification(THardwareInfo hw_spec)
Definition: omnisql.cpp:821
#define CHECK_LT(x, y)
Definition: Logger.h:197
const std::string OMNISCI_ROOT_USER
Definition: SysCatalog.h:59
#define CHECK_LE(x, y)
Definition: Logger.h:198
void oom_trace_dump()
Definition: omnisql.cpp:1502
void detect_table(char *file_name, TCopyParams &copy_params, ClientContext &context)
Definition: omnisql.cpp:158
std::vector< TDBObject > db_objects
std::vector< std::string > split(const std::string &str, const std::string &delim)
void register_signal_handler(int signum, void(*handler)(int))
Definition: MapDServer.cpp:100
TClusterHardwareInfo cluster_hardware_info
void set_table_epoch(ClientContext &context, const std::string &table_details)
Definition: omnisql.cpp:282
#define CHECK(condition)
Definition: Logger.h:187
void print_all_hardware_info(ClientContext &context)
Definition: omnisql.cpp:838
void copy_table(char const *filepath, char const *table, ClientContext &context)
Definition: omnisql.cpp:93
void omnisql_signal_handler(int signal_number)
Definition: omnisql.cpp:635
bool thrift_with_retry(SERVICE_ENUM which_service, CLIENT_CONTEXT &context, char const *arg, const int try_count=1)
static const std::string MAPD_RELEASE
Definition: release.h:43
static bool run
std::vector< TServerStatus > cluster_status
const int8_t const int64_t const uint64_t const int32_t const int64_t int64_t ** out
bool is_string() const
Definition: sqltypes.h:450
std::string privs_object_name
std::string license_key
std::string hide_sensitive_data_from_connect(const std::string &connect_str)
Definition: omnisql.cpp:973
std::vector< std::string > CommandTokenList