OmniSciDB  1dac507f6e
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
/home/jenkins-slave/workspace/core-os-doxygen/MapDServer.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2018 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "MapDServer.h"
19 
20 #ifdef HAVE_THRIFT_THREADFACTORY
21 #include <thrift/concurrency/ThreadFactory.h>
22 #else
23 #include <thrift/concurrency/PlatformThreadFactory.h>
24 #endif
25 
26 #include <thrift/concurrency/ThreadManager.h>
27 #include <thrift/protocol/TBinaryProtocol.h>
28 #include <thrift/protocol/TJSONProtocol.h>
29 #include <thrift/server/TThreadedServer.h>
30 #include <thrift/transport/TBufferTransports.h>
31 #include <thrift/transport/THttpServer.h>
32 #include <thrift/transport/TSSLServerSocket.h>
33 #include <thrift/transport/TSSLSocket.h>
34 #include <thrift/transport/TServerSocket.h>
35 
36 #include "MapDRelease.h"
37 
38 #include "Archive/S3Archive.h"
39 #include "Shared/Logger.h"
40 #include "Shared/MapDParameters.h"
41 #include "Shared/file_delete.h"
43 #include "Shared/mapd_shared_ptr.h"
44 #include "Shared/scope.h"
45 
46 #include <boost/algorithm/string.hpp>
47 #include <boost/algorithm/string/trim.hpp>
48 #include <boost/filesystem.hpp>
49 #include <boost/locale/generator.hpp>
50 #include <boost/make_shared.hpp>
51 #include <boost/program_options.hpp>
52 
53 #include <csignal>
54 #include <sstream>
55 #include <thread>
56 #include <vector>
57 #include "MapDRelease.h"
58 #include "Shared/Compressor.h"
59 #include "Shared/MapDParameters.h"
60 #include "Shared/file_delete.h"
61 #include "Shared/mapd_shared_ptr.h"
62 #include "Shared/scope.h"
63 
64 using namespace ::apache::thrift;
65 using namespace ::apache::thrift::concurrency;
66 using namespace ::apache::thrift::protocol;
67 using namespace ::apache::thrift::server;
68 using namespace ::apache::thrift::transport;
69 
70 unsigned connect_timeout{20000};
71 unsigned recv_timeout{300000};
72 unsigned send_timeout{300000};
73 
74 extern bool g_cache_string_hash;
75 extern size_t g_leaf_count;
76 extern bool g_skip_intermediate_count;
77 extern bool g_enable_bump_allocator;
78 extern size_t g_max_memory_allocation_size;
79 extern size_t g_min_memory_allocation_size;
81 extern bool g_enable_table_functions;
82 
84 
85 std::atomic<bool> g_running{true};
86 std::atomic<int> g_saw_signal{-1};
87 
89 TThreadedServer* g_thrift_http_server{nullptr};
90 TThreadedServer* g_thrift_buf_server{nullptr};
91 
93  const std::vector<TTableGeneration>& thrift_table_generations) {
94  TableGenerations table_generations;
95  for (const auto& thrift_table_generation : thrift_table_generations) {
96  table_generations.setGeneration(
97  thrift_table_generation.table_id,
98  TableGeneration{static_cast<size_t>(thrift_table_generation.tuple_count),
99  static_cast<size_t>(thrift_table_generation.start_rowid)});
100  }
101  return table_generations;
102 }
103 
104 mapd::shared_ptr<MapDHandler> g_warmup_handler =
105  0; // global "g_warmup_handler" needed to avoid circular dependency
106  // between "MapDHandler" & function "run_warmup_queries"
107 mapd::shared_ptr<MapDHandler> g_mapd_handler = 0;
108 std::once_flag g_shutdown_once_flag;
109 
111  if (g_mapd_handler) {
112  std::call_once(g_shutdown_once_flag, []() { g_mapd_handler->shutdown(); });
113  }
114 }
115 
116 void register_signal_handler(int signum, void (*handler)(int)) {
117  struct sigaction act;
118  memset(&act, 0, sizeof(act));
119  if (handler != SIG_DFL && handler != SIG_IGN) {
120  // block all signal deliveries while inside the signal handler
121  sigfillset(&act.sa_mask);
122  }
123  act.sa_handler = handler;
124  sigaction(signum, &act, NULL);
125 }
126 
127 // Signal handler to set a global flag telling the server to exit.
128 // Do not call other functions inside this (or any) signal handler
129 // unless you really know what you are doing. See also:
130 // man 7 signal-safety
131 // man 7 signal
132 // https://en.wikipedia.org/wiki/Reentrancy_(computing)
133 void omnisci_signal_handler(int signum) {
134  // Record the signal number for logging during shutdown.
135  // Only records the first signal if called more than once.
136  int expected_signal{-1};
137  if (!g_saw_signal.compare_exchange_strong(expected_signal, signum)) {
138  return; // this wasn't the first signal
139  }
140 
141  // This point should never be reached more than once.
142 
143  // Tell heartbeat() to shutdown by unsetting the 'g_running' flag.
144  // If 'g_running' is already false, this has no effect and the
145  // shutdown is already in progress.
146  g_running = false;
147 
148  // Handle core dumps specially by pausing inside this signal handler
149  // because on some systems, some signals will execute their default
150  // action immediately when and if the signal handler returns.
151  // We would like to do some emergency cleanup before core dump.
152  if (signum == SIGQUIT || signum == SIGABRT || signum == SIGSEGV || signum == SIGFPE) {
153  // Wait briefly to give heartbeat() a chance to flush the logs and
154  // do any other emergency shutdown tasks.
155  sleep(2);
156 
157  // Explicitly trigger whatever default action this signal would
158  // have done, such as terminate the process or dump core.
159  // Signals are currently blocked so this new signal will be queued
160  // until this signal handler returns.
161  register_signal_handler(signum, SIG_DFL);
162  kill(getpid(), signum);
163  }
164 }
165 
173  // Thrift secure socket can cause problems with SIGPIPE
174  register_signal_handler(SIGPIPE, SIG_IGN);
175 }
176 
177 void start_server(TThreadedServer& server, const int port) {
178  try {
179  server.serve();
180  } catch (std::exception& e) {
181  LOG(ERROR) << "Exception: " << e.what() << ": port " << port << std::endl;
182  }
183 }
184 
185 void releaseWarmupSession(TSessionId& sessionId, std::ifstream& query_file) {
186  query_file.close();
187  if (sessionId != g_warmup_handler->getInvalidSessionId()) {
188  g_warmup_handler->disconnect(sessionId);
189  }
190 }
191 
192 void run_warmup_queries(mapd::shared_ptr<MapDHandler> handler,
193  std::string base_path,
194  std::string query_file_path) {
195  // run warmup queries to load cache if requested
196  if (query_file_path.empty()) {
197  return;
198  }
199  LOG(INFO) << "Running DB warmup with queries from " << query_file_path;
200  try {
201  g_warmup_handler = handler;
202  std::string db_info;
203  std::string user_keyword, user_name, db_name;
204  std::ifstream query_file;
207  TSessionId sessionId = g_warmup_handler->getInvalidSessionId();
208 
209  ScopeGuard session_guard = [&] { releaseWarmupSession(sessionId, query_file); };
210  query_file.open(query_file_path);
211  while (std::getline(query_file, db_info)) {
212  if (db_info.length() == 0) {
213  continue;
214  }
215  std::istringstream iss(db_info);
216  iss >> user_keyword >> user_name >> db_name;
217  if (user_keyword.compare(0, 4, "USER") == 0) {
218  // connect to DB for given user_name/db_name with super_user_rights (without
219  // password), & start session
220  g_warmup_handler->super_user_rights_ = true;
221  g_warmup_handler->connect(sessionId, user_name, "", db_name);
222  g_warmup_handler->super_user_rights_ = false;
223 
224  // read and run one query at a time for the DB with the setup connection
225  TQueryResult ret;
226  std::string single_query;
227  while (std::getline(query_file, single_query)) {
228  boost::algorithm::trim(single_query);
229  if (single_query.length() == 0) {
230  continue;
231  }
232  if (single_query[0] == '}') {
233  single_query.clear();
234  break;
235  }
236  if (single_query.find(';') == single_query.npos) {
237  std::string multiline_query;
238  std::getline(query_file, multiline_query, ';');
239  single_query += multiline_query;
240  }
241 
242  try {
243  g_warmup_handler->sql_execute(ret, sessionId, single_query, true, "", -1, -1);
244  } catch (...) {
245  LOG(WARNING) << "Exception while executing '" << single_query
246  << "', ignoring";
247  }
248  single_query.clear();
249  }
250 
251  // stop session and disconnect from the DB
252  g_warmup_handler->disconnect(sessionId);
253  sessionId = g_warmup_handler->getInvalidSessionId();
254  } else {
255  LOG(WARNING) << "\nSyntax error in the file: " << query_file_path.c_str()
256  << " Missing expected keyword USER. Following line will be ignored: "
257  << db_info.c_str() << std::endl;
258  }
259  db_info.clear();
260  }
261  } catch (...) {
262  LOG(WARNING) << "Exception while executing warmup queries. "
263  << "Warmup may not be fully completed. Will proceed nevertheless."
264  << std::endl;
265  }
266 }
267 
268 namespace po = boost::program_options;
269 
271  public:
272  MapDProgramOptions(char const* argv0, bool dist_v5_ = false)
273  : log_options_(argv0), dist_v5_(dist_v5_) {
274  fillOptions();
276  }
277  int http_port = 6278;
278  size_t reserved_gpu_mem = 1 << 27;
279  std::string base_path;
280  std::string config_file = {"mapd.conf"};
281  std::string cluster_file = {"cluster.conf"};
282  std::string cluster_topology_file = {"cluster_topology.conf"};
283  std::string license_path = {""};
284  bool cpu_only = false;
285  bool verbose_logging = false;
286  bool jit_debug = false;
287  bool intel_jit_profile = false;
288  bool allow_multifrag = true;
289  bool read_only = false;
290  bool allow_loop_joins = false;
291  bool enable_legacy_syntax = true;
293 
295  bool enable_rendering = false;
297  int render_oom_retry_threshold = 0; // in milliseconds
298  size_t render_mem_bytes = 500000000;
299  size_t render_poly_cache_bytes = 300000000;
300 
301  bool enable_runtime_udf = false;
302 
303  bool enable_watchdog = true;
305  unsigned dynamic_watchdog_time_limit = 10000;
306 
311  int num_gpus = -1;
312  int start_gpu = 0;
316  size_t num_reader_threads = 0;
320  std::string db_query_file = {""};
324  bool exit_after_warmup = false;
334  std::string udf_file_name = {""};
335 
336  void fillOptions();
337  void fillAdvancedOptions();
338 
339  po::options_description help_desc;
340  po::options_description developer_desc;
342  po::positional_options_description positional_options;
343 
344  public:
345  std::vector<LeafHostInfo> db_leaves;
346  std::vector<LeafHostInfo> string_leaves;
347  po::variables_map vm;
348  std::string clusterIds_arg;
349 
350  std::string getNodeIds();
351  std::vector<std::string> getNodeIdsArray();
352  static const std::string nodeIds_token;
353 
354  boost::optional<int> parse_command_line(int argc, char const* const* argv);
355  void validate();
356  void validate_base_path();
357  void init_logging();
358  const bool dist_v5_;
359 };
360 
364  }
367 }
368 
370  help_desc.add_options()("help,h", "Show available options.");
371  help_desc.add_options()(
372  "allow-cpu-retry",
373  po::value<bool>(&g_allow_cpu_retry)
374  ->default_value(g_allow_cpu_retry)
375  ->implicit_value(true),
376  R"(Allow the queries which failed on GPU to retry on CPU, even when watchdog is enabled.)");
377  help_desc.add_options()("allow-loop-joins",
378  po::value<bool>(&allow_loop_joins)
379  ->default_value(allow_loop_joins)
380  ->implicit_value(true),
381  "Enable loop joins.");
382  help_desc.add_options()("bigint-count",
383  po::value<bool>(&g_bigint_count)
384  ->default_value(g_bigint_count)
385  ->implicit_value(false),
386  "Use 64-bit count.");
387  help_desc.add_options()("calcite-max-mem",
388  po::value<size_t>(&mapd_parameters.calcite_max_mem)
389  ->default_value(mapd_parameters.calcite_max_mem),
390  "Max memory available to calcite JVM.");
391  if (!dist_v5_) {
392  help_desc.add_options()("calcite-port",
393  po::value<int>(&mapd_parameters.calcite_port)
394  ->default_value(mapd_parameters.calcite_port),
395  "Calcite port number.");
396  }
397  help_desc.add_options()("config",
398  po::value<std::string>(&config_file),
399  "Path to server configuration file.");
400  help_desc.add_options()("cpu-buffer-mem-bytes",
401  po::value<size_t>(&mapd_parameters.cpu_buffer_mem_bytes)
402  ->default_value(mapd_parameters.cpu_buffer_mem_bytes),
403  "Size of memory reserved for CPU buffers, in bytes.");
404  help_desc.add_options()(
405  "cpu-only",
406  po::value<bool>(&cpu_only)->default_value(cpu_only)->implicit_value(true),
407  "Run on CPU only, even if GPUs are available.");
408  help_desc.add_options()("cuda-block-size",
409  po::value<size_t>(&mapd_parameters.cuda_block_size)
410  ->default_value(mapd_parameters.cuda_block_size),
411  "Size of block to use on GPU.");
412  help_desc.add_options()("cuda-grid-size",
413  po::value<size_t>(&mapd_parameters.cuda_grid_size)
414  ->default_value(mapd_parameters.cuda_grid_size),
415  "Size of grid to use on GPU.");
416  if (!dist_v5_) {
417  help_desc.add_options()(
418  "data",
419  po::value<std::string>(&base_path)->required()->default_value("data"),
420  "Directory path to OmniSci data storage (catalogs, raw data, log files, etc).");
421  positional_options.add("data", 1);
422  }
423  help_desc.add_options()("db-query-list",
424  po::value<std::string>(&db_query_file),
425  "Path to file containing OmniSci warmup queries.");
426  help_desc.add_options()(
427  "exit-after-warmup",
428  po::value<bool>(&exit_after_warmup)->default_value(false)->implicit_value(true),
429  "Exit after OmniSci warmup queries.");
430  help_desc.add_options()("dynamic-watchdog-time-limit",
431  po::value<unsigned>(&dynamic_watchdog_time_limit)
432  ->default_value(dynamic_watchdog_time_limit)
433  ->implicit_value(10000),
434  "Dynamic watchdog time limit, in milliseconds.");
435  help_desc.add_options()("enable-debug-timer",
436  po::value<bool>(&g_enable_debug_timer)
437  ->default_value(g_enable_debug_timer)
438  ->implicit_value(true),
439  "Enable debug timer logging.");
440  help_desc.add_options()("enable-dynamic-watchdog",
441  po::value<bool>(&enable_dynamic_watchdog)
442  ->default_value(enable_dynamic_watchdog)
443  ->implicit_value(true),
444  "Enable dynamic watchdog.");
445  help_desc.add_options()("enable-filter-push-down",
446  po::value<bool>(&g_enable_filter_push_down)
447  ->default_value(g_enable_filter_push_down)
448  ->implicit_value(true),
449  "Enable filter push down through joins.");
450  help_desc.add_options()("enable-overlaps-hashjoin",
451  po::value<bool>(&g_enable_overlaps_hashjoin)
452  ->default_value(g_enable_overlaps_hashjoin)
453  ->implicit_value(true),
454  "Enable the overlaps hash join framework allowing for range "
455  "join (e.g. spatial overlaps) computation using a hash table.");
456  if (!dist_v5_) {
457  help_desc.add_options()(
458  "enable-string-dict-hash-cache",
459  po::value<bool>(&g_cache_string_hash)
460  ->default_value(g_cache_string_hash)
461  ->implicit_value(true),
462  "Cache string hash values in the string dictionary server during import.");
463  }
464  help_desc.add_options()(
465  "enable-thrift-logs",
466  po::value<bool>(&g_enable_thrift_logs)
467  ->default_value(g_enable_thrift_logs)
468  ->implicit_value(true),
469  "Enable writing messages directly from thrift to stdout/stderr.");
470  help_desc.add_options()("enable-watchdog",
471  po::value<bool>(&enable_watchdog)
472  ->default_value(enable_watchdog)
473  ->implicit_value(true),
474  "Enable watchdog.");
475  help_desc.add_options()(
476  "filter-push-down-low-frac",
477  po::value<float>(&g_filter_push_down_low_frac)
478  ->default_value(g_filter_push_down_low_frac)
479  ->implicit_value(g_filter_push_down_low_frac),
480  "Lower threshold for selectivity of filters that are pushed down.");
481  help_desc.add_options()(
482  "filter-push-down-high-frac",
483  po::value<float>(&g_filter_push_down_high_frac)
484  ->default_value(g_filter_push_down_high_frac)
485  ->implicit_value(g_filter_push_down_high_frac),
486  "Higher threshold for selectivity of filters that are pushed down.");
487  help_desc.add_options()("filter-push-down-passing-row-ubound",
488  po::value<size_t>(&g_filter_push_down_passing_row_ubound)
490  ->implicit_value(g_filter_push_down_passing_row_ubound),
491  "Upperbound on the number of rows that should pass the filter "
492  "if the selectivity is less than "
493  "the high fraction threshold.");
494  help_desc.add_options()("from-table-reordering",
495  po::value<bool>(&g_from_table_reordering)
496  ->default_value(g_from_table_reordering)
497  ->implicit_value(true),
498  "Enable automatic table reordering in FROM clause.");
499  help_desc.add_options()("gpu-buffer-mem-bytes",
500  po::value<size_t>(&mapd_parameters.gpu_buffer_mem_bytes)
501  ->default_value(mapd_parameters.gpu_buffer_mem_bytes),
502  "Size of memory reserved for GPU buffers, in bytes, per GPU.");
503  help_desc.add_options()("gpu-input-mem-limit",
504  po::value<double>(&mapd_parameters.gpu_input_mem_limit)
505  ->default_value(mapd_parameters.gpu_input_mem_limit),
506  "Force query to CPU when input data memory usage exceeds this "
507  "percentage of available GPU memory.");
508  help_desc.add_options()(
509  "hll-precision-bits",
510  po::value<int>(&g_hll_precision_bits)
511  ->default_value(g_hll_precision_bits)
512  ->implicit_value(g_hll_precision_bits),
513  "Number of bits used from the hash value used to specify the bucket number.");
514  if (!dist_v5_) {
515  help_desc.add_options()("http-port",
516  po::value<int>(&http_port)->default_value(http_port),
517  "HTTP port number.");
518  }
519  help_desc.add_options()(
520  "idle-session-duration",
521  po::value<int>(&idle_session_duration)->default_value(idle_session_duration),
522  "Maximum duration of idle session.");
523  help_desc.add_options()("inner-join-fragment-skipping",
524  po::value<bool>(&g_inner_join_fragment_skipping)
525  ->default_value(g_inner_join_fragment_skipping)
526  ->implicit_value(true),
527  "Enable/disable inner join fragment skipping. This feature is "
528  "considered stable and is enabled by default. This "
529  "parameter will be removed in a future release.");
530  help_desc.add_options()(
531  "max-session-duration",
532  po::value<int>(&max_session_duration)->default_value(max_session_duration),
533  "Maximum duration of active session.");
534  help_desc.add_options()(
535  "null-div-by-zero",
536  po::value<bool>(&g_null_div_by_zero)
537  ->default_value(g_null_div_by_zero)
538  ->implicit_value(true),
539  "Return null on division by zero instead of throwing an exception.");
540  help_desc.add_options()(
541  "num-reader-threads",
542  po::value<size_t>(&num_reader_threads)->default_value(num_reader_threads),
543  "Number of reader threads to use.");
544  help_desc.add_options()(
545  "overlaps-max-table-size-bytes",
546  po::value<size_t>(&g_overlaps_max_table_size_bytes)
547  ->default_value(g_overlaps_max_table_size_bytes),
548  "The maximum size in bytes of the hash table for an overlaps hash join.");
549  if (!dist_v5_) {
550  help_desc.add_options()("port,p",
551  po::value<int>(&mapd_parameters.omnisci_server_port)
552  ->default_value(mapd_parameters.omnisci_server_port),
553  "TCP Port number.");
554  }
555  help_desc.add_options()("num-gpus",
556  po::value<int>(&num_gpus)->default_value(num_gpus),
557  "Number of gpus to use.");
558  help_desc.add_options()(
559  "read-only",
560  po::value<bool>(&read_only)->default_value(read_only)->implicit_value(true),
561  "Enable read-only mode.");
562  help_desc.add_options()(
563  "res-gpu-mem",
564  po::value<size_t>(&reserved_gpu_mem)->default_value(reserved_gpu_mem),
565  "Reduces GPU memory available to the OmniSci allocator by this amount. Used for "
566  "compiled code cache and ancillary GPU functions and other processes that may also "
567  "be using the GPU concurrent with OmniSciDB.");
568  help_desc.add_options()("start-gpu",
569  po::value<int>(&start_gpu)->default_value(start_gpu),
570  "First gpu to use.");
571  help_desc.add_options()("trivial-loop-join-threshold",
572  po::value<unsigned>(&g_trivial_loop_join_threshold)
573  ->default_value(g_trivial_loop_join_threshold)
574  ->implicit_value(1000),
575  "The maximum number of rows in the inner table of a loop join "
576  "considered to be trivially small.");
577  help_desc.add_options()("verbose",
578  po::value<bool>(&verbose_logging)
579  ->default_value(verbose_logging)
580  ->implicit_value(true),
581  "Write additional debug log messages to server logs.");
582  help_desc.add_options()(
583  "enable-runtime-udf",
584  po::value<bool>(&enable_runtime_udf)
585  ->default_value(enable_runtime_udf)
586  ->implicit_value(true),
587  "Enable runtime UDF registration by passing signatures and corresponding LLVM IR "
588  "to the `register_runtime_udf` endpoint. For use with the Python Remote Backend "
589  "Compiler server, packaged separately.");
590  help_desc.add_options()("version,v", "Print Version Number.");
591  help_desc.add_options()("enable-experimental-string-functions",
594  ->implicit_value(true),
595  "Enable experimental string functions.");
596 
598 }
599 
601  developer_desc.add_options()("dev-options", "Print internal developer options.");
602  developer_desc.add_options()(
603  "enable-calcite-view-optimize",
606  ->implicit_value(true),
607  "Enable additional calcite (query plan) optimizations when a view is part of the "
608  "query.");
609  developer_desc.add_options()(
610  "enable-columnar-output",
611  po::value<bool>(&g_enable_columnar_output)
612  ->default_value(g_enable_columnar_output)
613  ->implicit_value(true),
614  "Enable columnar output for intermediate/final query steps.");
615  developer_desc.add_options()("enable-legacy-syntax",
616  po::value<bool>(&enable_legacy_syntax)
617  ->default_value(enable_legacy_syntax)
618  ->implicit_value(true),
619  "Enable legacy syntax.");
620  developer_desc.add_options()(
621  "enable-multifrag",
622  po::value<bool>(&allow_multifrag)
623  ->default_value(allow_multifrag)
624  ->implicit_value(true),
625  "Enable execution over multiple fragments in a single round-trip to GPU.");
626  developer_desc.add_options()(
627  "enable-shared-mem-group-by",
628  po::value<bool>(&g_enable_smem_group_by)
629  ->default_value(g_enable_smem_group_by)
630  ->implicit_value(true),
631  "Enable using GPU shared memory for some GROUP BY queries.");
632  developer_desc.add_options()("enable-direct-columnarization",
633  po::value<bool>(&g_enable_direct_columnarization)
634  ->default_value(g_enable_direct_columnarization)
635  ->implicit_value(true),
636  "Enables/disables a more optimized columnarization method "
637  "for intermediate steps in multi-step queries.");
638  developer_desc.add_options()("enable-window-functions",
639  po::value<bool>(&g_enable_window_functions)
640  ->default_value(g_enable_window_functions)
641  ->implicit_value(true),
642  "Enable experimental window function support.");
643  developer_desc.add_options()("enable-table-functions",
644  po::value<bool>(&g_enable_table_functions)
645  ->default_value(g_enable_table_functions)
646  ->implicit_value(true),
647  "Enable experimental table functions support.");
648  developer_desc.add_options()(
649  "jit-debug-ir",
650  po::value<bool>(&jit_debug)->default_value(jit_debug)->implicit_value(true),
651  "Enable runtime debugger support for the JIT. Note that this flag is "
652  "incompatible "
653  "with the `ENABLE_JIT_DEBUG` build flag. The generated code can be found at "
654  "`/tmp/mapdquery`.");
655  developer_desc.add_options()(
656  "intel-jit-profile",
657  po::value<bool>(&intel_jit_profile)
658  ->default_value(intel_jit_profile)
659  ->implicit_value(true),
660  "Enable runtime support for the JIT code profiling using Intel VTune.");
661  developer_desc.add_options()(
662  "skip-intermediate-count",
663  po::value<bool>(&g_skip_intermediate_count)
664  ->default_value(g_skip_intermediate_count)
665  ->implicit_value(true),
666  "Skip pre-flight counts for intermediate projections with no filters.");
667  developer_desc.add_options()(
668  "strip-join-covered-quals",
669  po::value<bool>(&g_strip_join_covered_quals)
670  ->default_value(g_strip_join_covered_quals)
671  ->implicit_value(true),
672  "Remove quals from the filtered count if they are covered by a "
673  "join condition (currently only ST_Contains).");
674  developer_desc.add_options()(
675  "max-output-projection-allocation-bytes",
676  po::value<size_t>(&g_max_memory_allocation_size)
677  ->default_value(g_max_memory_allocation_size),
678  "Maximum allocation size for a fixed output buffer allocation for projection "
679  "queries with no pre-flight count. Default is the maximum slab size (sizes "
680  "greater "
681  "than the maximum slab size have no affect). Requires bump allocator.");
682  developer_desc.add_options()(
683  "min-output-projection-allocation-bytes",
684  po::value<size_t>(&g_min_memory_allocation_size)
685  ->default_value(g_min_memory_allocation_size),
686  "Minimum allocation size for a fixed output buffer allocation for projection "
687  "queries with no pre-flight count. If an allocation of this size cannot be "
688  "obtained, the query will be retried with different execution parameters and/or "
689  "on "
690  "CPU (if allow-cpu-retry is enabled). Requires bump allocator.");
691  developer_desc.add_options()("enable-bump-allocator",
692  po::value<bool>(&g_enable_bump_allocator)
693  ->default_value(g_enable_bump_allocator)
694  ->implicit_value(true),
695  "Enable the bump allocator for projection queries on "
696  "GPU. The bump allocator will "
697  "allocate a fixed size buffer for each query, track the "
698  "number of rows passing the "
699  "kernel during query execution, and copy back only the "
700  "rows that passed the kernel "
701  "to CPU after execution. When disabled, pre-flight "
702  "count queries are used to size "
703  "the output buffer for projection queries.");
704  developer_desc.add_options()("ssl-cert",
705  po::value<std::string>(&mapd_parameters.ssl_cert_file)
706  ->default_value(std::string("")),
707  "SSL Validated public certficate.");
708  developer_desc.add_options()("ssl-private-key",
709  po::value<std::string>(&mapd_parameters.ssl_key_file)
710  ->default_value(std::string("")),
711  "SSL private key file.");
712  // Note ssl_trust_store is passed through to Calcite via mapd_parameters
713  // todo(jack): add ensure ssl-trust-store exists if cert and private key in use
714  developer_desc.add_options()("ssl-trust-store",
715  po::value<std::string>(&mapd_parameters.ssl_trust_store)
716  ->default_value(std::string("")),
717  "SSL Validated public cert as a java trust store.");
718  developer_desc.add_options()("ssl-trust-password",
719  po::value<std::string>(&mapd_parameters.ssl_trust_password)
720  ->default_value(std::string("")),
721  "SSL java trust store password.");
722 
723  developer_desc.add_options()("ssl-keystore",
724  po::value<std::string>(&mapd_parameters.ssl_keystore)
725  ->default_value(std::string("")),
726  "SSL server credentials as a java key store.");
727  developer_desc.add_options()(
728  "ssl-keystore-password",
729  po::value<std::string>(&mapd_parameters.ssl_keystore_password)
730  ->default_value(std::string("")),
731  "SSL server keystore password.");
732  developer_desc.add_options()(
733  "udf",
734  po::value<std::string>(&udf_file_name),
735  "Load user defined extension functions from this file at startup. The file is "
736  "expected to be a C/C++ file with extension .cpp.");
737 }
738 
739 namespace {
740 
741 std::stringstream sanitize_config_file(std::ifstream& in) {
742  // Strip the web section out of the config file so boost can validate program options
743  std::stringstream ss;
744  std::string line;
745  while (std::getline(in, line)) {
746  ss << line << "\n";
747  if (line == "[web]") {
748  break;
749  }
750  }
751  return ss;
752 }
753 
754 bool trim_and_check_file_exists(std::string& filename, const std::string desc) {
755  if (!filename.empty()) {
756  boost::algorithm::trim_if(filename, boost::is_any_of("\"'"));
757  if (!boost::filesystem::exists(filename)) {
758  std::cerr << desc << " " << filename << " does not exist." << std::endl;
759  return false;
760  }
761  }
762  return true;
763 }
764 
765 } // namespace
766 
768  boost::algorithm::trim_if(base_path, boost::is_any_of("\"'"));
769  if (!boost::filesystem::exists(base_path)) {
770  throw std::runtime_error("OmniSci base directory does not exist at " + base_path);
771  }
772 }
773 
775  boost::algorithm::trim_if(base_path, boost::is_any_of("\"'"));
776  const auto data_path = boost::filesystem::path(base_path) / "mapd_data";
777  if (!boost::filesystem::exists(data_path)) {
778  throw std::runtime_error("OmniSci data directory does not exist at '" + base_path +
779  "'");
780  }
781 
782  {
783  const auto lock_file = boost::filesystem::path(base_path) / "omnisci_server_pid.lck";
784  auto pid = std::to_string(getpid());
785 
786  int pid_fd = open(lock_file.c_str(), O_RDWR | O_CREAT, 0644);
787  if (pid_fd == -1) {
788  auto err = std::string("Failed to open PID file ") + lock_file.c_str() + ". " +
789  strerror(errno) + ".";
790  throw std::runtime_error(err);
791  }
792  if (lockf(pid_fd, F_TLOCK, 0) == -1) {
793  close(pid_fd);
794  auto err = std::string("Another OmniSci Server is using data directory ") +
795  base_path + ".";
796  throw std::runtime_error(err);
797  }
798  if (ftruncate(pid_fd, 0) == -1) {
799  close(pid_fd);
800  auto err = std::string("Failed to truncate PID file ") + lock_file.c_str() + ". " +
801  strerror(errno) + ".";
802  throw std::runtime_error(err);
803  }
804  if (write(pid_fd, pid.c_str(), pid.length()) == -1) {
805  close(pid_fd);
806  auto err = std::string("Failed to write PID file ") + lock_file.c_str() + ". " +
807  strerror(errno) + ".";
808  throw std::runtime_error(err);
809  }
810  }
811  boost::algorithm::trim_if(db_query_file, boost::is_any_of("\"'"));
812  if (db_query_file.length() > 0 && !boost::filesystem::exists(db_query_file)) {
813  throw std::runtime_error("File containing DB queries " + db_query_file +
814  " does not exist.");
815  }
816  const auto db_file =
817  boost::filesystem::path(base_path) / "mapd_catalogs" / OMNISCI_SYSTEM_CATALOG;
818  if (!boost::filesystem::exists(db_file)) {
819  { // check old system catalog existsense
820  const auto db_file = boost::filesystem::path(base_path) / "mapd_catalogs/mapd";
821  if (!boost::filesystem::exists(db_file)) {
822  throw std::runtime_error("OmniSci system catalog " + OMNISCI_SYSTEM_CATALOG +
823  " does not exist.");
824  }
825  }
826  }
827 
828  // add all parameters to be displayed on startup
829  LOG(INFO) << "OmniSci started with data directory at '" << base_path << "'";
830  LOG(INFO) << " Watchdog is set to " << enable_watchdog;
831  LOG(INFO) << " Dynamic Watchdog is set to " << enable_dynamic_watchdog;
833  LOG(INFO) << " Dynamic Watchdog timeout is set to " << dynamic_watchdog_time_limit;
834  }
835 
836  LOG(INFO) << " Debug Timer is set to " << g_enable_debug_timer;
837 
838  LOG(INFO) << " Maximum Idle session duration " << idle_session_duration;
839 
840  LOG(INFO) << " Maximum active session duration " << max_session_duration;
841 }
842 
843 boost::optional<int> MapDProgramOptions::parse_command_line(int argc,
844  char const* const* argv) {
845  po::options_description all_desc("All options");
846  all_desc.add(help_desc).add(developer_desc);
847 
848  try {
849  po::store(po::command_line_parser(argc, argv)
850  .options(all_desc)
851  .positional(positional_options)
852  .run(),
853  vm);
854  po::notify(vm);
855 
856  if (vm.count("config")) {
857  std::ifstream settings_file(config_file);
858 
859  auto sanitized_settings = sanitize_config_file(settings_file);
860 
861  po::store(po::parse_config_file(sanitized_settings, all_desc, false), vm);
862  po::notify(vm);
863  settings_file.close();
864  }
865 
867  return 1;
868  }
870  return 1;
871  }
873  return 1;
874  }
876  return 1;
877  }
878 
879  if (vm.count("help")) {
880  std::cout << "Usage: omnisci_server <data directory path> [-p <port number>] "
881  "[--http-port <http port number>] [--flush-log] [--version|-v]"
882  << std::endl
883  << std::endl;
884  std::cout << help_desc << std::endl;
885  return 0;
886  }
887  if (vm.count("dev-options")) {
888  std::cout << "Usage: omnisci_server <data directory path> [-p <port number>] "
889  "[--http-port <http port number>] [--flush-log] [--version|-v]"
890  << std::endl
891  << std::endl;
892  std::cout << developer_desc << std::endl;
893  return 0;
894  }
895  if (vm.count("version")) {
896  std::cout << "OmniSci Version: " << MAPD_RELEASE << std::endl;
897  return 0;
898  }
899 
903  } catch (po::error& e) {
904  std::cerr << "Usage Error: " << e.what() << std::endl;
905  return 1;
906  }
907 
908  if (g_hll_precision_bits < 1 || g_hll_precision_bits > 16) {
909  std::cerr << "hll-precision-bits must be between 1 and 16." << std::endl;
910  return 1;
911  }
912 
914  LOG(INFO) << " From clause table reordering is disabled";
915  }
916 
918  LOG(INFO) << " Filter push down for JOIN is enabled";
919  }
920 
921  if (vm.count("udf")) {
922  boost::algorithm::trim_if(udf_file_name, boost::is_any_of("\"'"));
923 
924  if (!boost::filesystem::exists(udf_file_name)) {
925  LOG(ERROR) << " User defined function file " << udf_file_name << " does not exist.";
926  return 1;
927  }
928 
929  LOG(INFO) << " User provided extension functions loaded from " << udf_file_name;
930  }
931 
932  if (enable_runtime_udf) {
933  LOG(INFO) << " Runtime user defined extension functions enabled globally.";
934  }
935 
936  boost::algorithm::trim_if(mapd_parameters.ha_brokers, boost::is_any_of("\"'"));
937  boost::algorithm::trim_if(mapd_parameters.ha_group_id, boost::is_any_of("\"'"));
938  boost::algorithm::trim_if(mapd_parameters.ha_shared_data, boost::is_any_of("\"'"));
939  boost::algorithm::trim_if(mapd_parameters.ha_unique_server_id, boost::is_any_of("\"'"));
940 
941  if (!mapd_parameters.ha_group_id.empty()) {
942  LOG(INFO) << " HA group id " << mapd_parameters.ha_group_id;
943  if (mapd_parameters.ha_unique_server_id.empty()) {
944  LOG(ERROR) << "Starting server in HA mode --ha-unique-server-id must be set ";
945  return 5;
946  } else {
947  LOG(INFO) << " HA unique server id " << mapd_parameters.ha_unique_server_id;
948  }
949  if (mapd_parameters.ha_brokers.empty()) {
950  LOG(ERROR) << "Starting server in HA mode --ha-brokers must be set ";
951  return 6;
952  } else {
953  LOG(INFO) << " HA brokers " << mapd_parameters.ha_brokers;
954  }
955  if (mapd_parameters.ha_shared_data.empty()) {
956  LOG(ERROR) << "Starting server in HA mode --ha-shared-data must be set ";
957  return 7;
958  } else {
959  LOG(INFO) << " HA shared data is " << mapd_parameters.ha_shared_data;
960  }
961  }
962  LOG(INFO) << " cuda block size " << mapd_parameters.cuda_block_size;
963  LOG(INFO) << " cuda grid size " << mapd_parameters.cuda_grid_size;
964  LOG(INFO) << " calcite JVM max memory " << mapd_parameters.calcite_max_mem;
965  LOG(INFO) << " OmniSci Server Port " << mapd_parameters.omnisci_server_port;
966  LOG(INFO) << " OmniSci Calcite Port " << mapd_parameters.calcite_port;
967  LOG(INFO) << " Enable Calcite view optimize "
969 
970  LOG(INFO) << " Allow Local Auth Fallback: "
971  << (authMetadata.allowLocalAuthFallback ? "enabled" : "disabled");
972 
973  boost::algorithm::trim_if(authMetadata.distinguishedName, boost::is_any_of("\"'"));
974  boost::algorithm::trim_if(authMetadata.uri, boost::is_any_of("\"'"));
975  boost::algorithm::trim_if(authMetadata.ldapQueryUrl, boost::is_any_of("\"'"));
976  boost::algorithm::trim_if(authMetadata.ldapRoleRegex, boost::is_any_of("\"'"));
977  boost::algorithm::trim_if(authMetadata.ldapSuperUserRole, boost::is_any_of("\"'"));
978  boost::algorithm::trim_if(authMetadata.restToken, boost::is_any_of("\"'"));
979  boost::algorithm::trim_if(authMetadata.restUrl, boost::is_any_of("\"'"));
980 
981  return boost::none;
982 }
983 
984 void heartbeat() {
985  // Block all signals for this heartbeat thread, only.
986  sigset_t set;
987  sigfillset(&set);
988  int result = pthread_sigmask(SIG_BLOCK, &set, NULL);
989  if (result != 0) {
990  throw std::runtime_error("heartbeat() thread startup failed");
991  }
992 
993  // Sleep until omnisci_signal_handler or anything clears the g_running flag.
994  VLOG(1) << "heartbeat thread starting";
995  while (::g_running) {
996  using namespace std::chrono;
997  std::this_thread::sleep_for(1s);
998  }
999  VLOG(1) << "heartbeat thread exiting";
1000 
1001  // Get the signal number if there was a signal.
1002  int signum = g_saw_signal;
1003  if (signum >= 1 && signum != SIGTERM) {
1004  LOG(INFO) << "Interrupt signal (" << signum << ") received.";
1005  }
1006 
1007  // if dumping core, try to do some quick stuff
1008  if (signum == SIGQUIT || signum == SIGABRT || signum == SIGSEGV || signum == SIGFPE) {
1009  if (g_mapd_handler) {
1010  std::call_once(g_shutdown_once_flag,
1011  []() { g_mapd_handler->emergency_shutdown(); });
1012  }
1013  logger::shutdown();
1014  return;
1015  // core dump should begin soon after this, see omnisci_signal_handler()
1016  }
1017 
1018  // trigger an orderly shutdown by telling Thrift to stop serving
1019  {
1020  mapd_shared_lock<mapd_shared_mutex> read_lock(g_thrift_mutex);
1021  auto httpserv = g_thrift_http_server;
1022  if (httpserv) {
1023  httpserv->stop();
1024  }
1025  auto bufserv = g_thrift_buf_server;
1026  if (bufserv) {
1027  bufserv->stop();
1028  }
1029  // main() should return soon after this
1030  }
1031 }
1032 
1033 int startMapdServer(MapDProgramOptions& prog_config_opts, bool start_http_server = true) {
1034  // try to enforce an orderly shutdown even after a signal
1036 
1037  // register shutdown procedures for when a normal shutdown happens
1038  // be aware that atexit() functions run in reverse order
1039  atexit(&logger::shutdown);
1040  atexit(&shutdown_handler);
1041 
1042 #ifdef HAVE_AWS_S3
1043  // hold a s3 archive here to survive from a segfault that happens on centos
1044  // when s3 transactions and others openssl-ed sessions are interleaved...
1045  auto s3_survivor = std::make_unique<S3Archive>("s3://omnisci/s3_survivor.txt", true);
1046 #endif
1047 
1048  // start background thread to clean up _DELETE_ME files
1049  const unsigned int wait_interval =
1050  3; // wait time in secs after looking for deleted file before looking again
1051  std::thread file_delete_thread(file_delete,
1052  std::ref(g_running),
1053  wait_interval,
1054  prog_config_opts.base_path + "/mapd_data");
1055  std::thread heartbeat_thread(heartbeat);
1056 
1057  if (!g_enable_thrift_logs) {
1058  apache::thrift::GlobalOutput.setOutputFunction([](const char* msg) {});
1059  }
1060 
1062  // Use the locale setting of the server by default. The generate parameter can be
1063  // updated appropriately if a locale override option is ever supported.
1064  boost::locale::generator generator;
1065  std::locale::global(generator.generate(""));
1066  }
1067 
1068  try {
1069  g_mapd_handler =
1070  mapd::make_shared<MapDHandler>(prog_config_opts.db_leaves,
1071  prog_config_opts.string_leaves,
1072  prog_config_opts.base_path,
1073  prog_config_opts.cpu_only,
1074  prog_config_opts.allow_multifrag,
1075  prog_config_opts.jit_debug,
1076  prog_config_opts.intel_jit_profile,
1077  prog_config_opts.read_only,
1078  prog_config_opts.allow_loop_joins,
1079  prog_config_opts.enable_rendering,
1080  prog_config_opts.enable_auto_clear_render_mem,
1081  prog_config_opts.render_oom_retry_threshold,
1082  prog_config_opts.render_mem_bytes,
1083  prog_config_opts.num_gpus,
1084  prog_config_opts.start_gpu,
1085  prog_config_opts.reserved_gpu_mem,
1086  prog_config_opts.num_reader_threads,
1087  prog_config_opts.authMetadata,
1088  prog_config_opts.mapd_parameters,
1089  prog_config_opts.enable_legacy_syntax,
1090  prog_config_opts.idle_session_duration,
1091  prog_config_opts.max_session_duration,
1092  prog_config_opts.enable_runtime_udf,
1093  prog_config_opts.udf_file_name);
1094  } catch (const std::exception& e) {
1095  LOG(FATAL) << "Failed to initialize service handler: " << e.what();
1096  }
1097 
1098  mapd::shared_ptr<TServerSocket> serverSocket;
1099  mapd::shared_ptr<TServerSocket> httpServerSocket;
1100  if (!prog_config_opts.mapd_parameters.ssl_cert_file.empty() &&
1101  !prog_config_opts.mapd_parameters.ssl_key_file.empty()) {
1102  mapd::shared_ptr<TSSLSocketFactory> sslSocketFactory;
1103  sslSocketFactory =
1104  mapd::shared_ptr<TSSLSocketFactory>(new TSSLSocketFactory(SSLProtocol::SSLTLS));
1105  sslSocketFactory->loadCertificate(
1106  prog_config_opts.mapd_parameters.ssl_cert_file.c_str());
1107  sslSocketFactory->loadPrivateKey(
1108  prog_config_opts.mapd_parameters.ssl_key_file.c_str());
1109  sslSocketFactory->authenticate(false);
1110  sslSocketFactory->ciphers("ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH");
1111  serverSocket = mapd::shared_ptr<TServerSocket>(new TSSLServerSocket(
1112  prog_config_opts.mapd_parameters.omnisci_server_port, sslSocketFactory));
1113  httpServerSocket = mapd::shared_ptr<TServerSocket>(
1114  new TSSLServerSocket(prog_config_opts.http_port, sslSocketFactory));
1115  LOG(INFO) << " OmniSci server using encrypted connection. Cert file ["
1116  << prog_config_opts.mapd_parameters.ssl_cert_file << "], key file ["
1117  << prog_config_opts.mapd_parameters.ssl_key_file << "]";
1118  } else {
1119  LOG(INFO) << " OmniSci server using unencrypted connection";
1120  serverSocket = mapd::shared_ptr<TServerSocket>(
1121  new TServerSocket(prog_config_opts.mapd_parameters.omnisci_server_port));
1122  httpServerSocket =
1123  mapd::shared_ptr<TServerSocket>(new TServerSocket(prog_config_opts.http_port));
1124  }
1125 
1126  ScopeGuard pointer_to_thrift_guard = [] {
1127  mapd_lock_guard<mapd_shared_mutex> write_lock(g_thrift_mutex);
1129  };
1130 
1131  if (prog_config_opts.mapd_parameters.ha_group_id.empty()) {
1132  mapd::shared_ptr<TProcessor> processor(new MapDProcessor(g_mapd_handler));
1133 
1134  mapd::shared_ptr<TTransportFactory> bufTransportFactory(
1135  new TBufferedTransportFactory());
1136  mapd::shared_ptr<TProtocolFactory> bufProtocolFactory(new TBinaryProtocolFactory());
1137 
1138  mapd::shared_ptr<TServerTransport> bufServerTransport(serverSocket);
1139  TThreadedServer bufServer(
1140  processor, bufServerTransport, bufTransportFactory, bufProtocolFactory);
1141  {
1142  mapd_lock_guard<mapd_shared_mutex> write_lock(g_thrift_mutex);
1143  g_thrift_buf_server = &bufServer;
1144  }
1145 
1146  std::thread bufThread(start_server,
1147  std::ref(bufServer),
1148  prog_config_opts.mapd_parameters.omnisci_server_port);
1149 
1150  // run warm up queries if any exists
1152  g_mapd_handler, prog_config_opts.base_path, prog_config_opts.db_query_file);
1153  if (prog_config_opts.exit_after_warmup) {
1154  g_running = false;
1155  }
1156 
1157  mapd::shared_ptr<TServerTransport> httpServerTransport(httpServerSocket);
1158  mapd::shared_ptr<TTransportFactory> httpTransportFactory(
1159  new THttpServerTransportFactory());
1160  mapd::shared_ptr<TProtocolFactory> httpProtocolFactory(new TJSONProtocolFactory());
1161  TThreadedServer httpServer(
1162  processor, httpServerTransport, httpTransportFactory, httpProtocolFactory);
1163  if (start_http_server) {
1164  {
1165  mapd_lock_guard<mapd_shared_mutex> write_lock(g_thrift_mutex);
1166  g_thrift_http_server = &httpServer;
1167  }
1168  std::thread httpThread(
1169  start_server, std::ref(httpServer), prog_config_opts.http_port);
1170  bufThread.join();
1171  httpThread.join();
1172  } else {
1173  bufThread.join();
1174  }
1175  } else { // running ha server
1176  LOG(FATAL) << "No High Availability module available, please contact OmniSci support";
1177  }
1178 
1179  g_running = false;
1180  file_delete_thread.join();
1181  heartbeat_thread.join();
1182 
1183  int signum = g_saw_signal;
1184  if (signum <= 0 || signum == SIGTERM) {
1185  return 0;
1186  } else {
1187  return signum;
1188  }
1189 }
1190 
1191 const std::string MapDProgramOptions::nodeIds_token = {"node_id"};
1192 
1193 int main(int argc, char** argv) {
1194  bool has_clust_topo = false;
1195 
1196  MapDProgramOptions prog_config_opts(argv[0], has_clust_topo);
1197 
1198  try {
1199  if (auto return_code = prog_config_opts.parse_command_line(argc, argv)) {
1200  return *return_code;
1201  }
1202 
1203  if (!has_clust_topo) {
1204  prog_config_opts.validate_base_path();
1205  prog_config_opts.init_logging();
1206  prog_config_opts.validate();
1207  return (startMapdServer(prog_config_opts));
1208  }
1209  } catch (std::runtime_error& e) {
1210  std::cerr << "Can't start: " << e.what() << std::endl;
1211  return 1;
1212  } catch (boost::program_options::error& e) {
1213  std::cerr << "Usage Error: " << e.what() << std::endl;
1214  return 1;
1215  }
1216 }
std::string distinguishedName
Definition: AuthMetadata.h:26
mapd_shared_mutex g_thrift_mutex
Definition: MapDServer.cpp:88
TThreadedServer * g_thrift_buf_server
Definition: MapDServer.cpp:90
boost::optional< int > parse_command_line(int argc, char const *const *argv)
Definition: MapDServer.cpp:843
std::string filename(char const *path)
Definition: Logger.cpp:58
bool g_enable_smem_group_by
float g_filter_push_down_low_frac
Definition: Execute.cpp:83
std::string ldapQueryUrl
Definition: AuthMetadata.h:27
mapd::shared_ptr< MapDHandler > g_mapd_handler
Definition: MapDServer.cpp:107
bool enable_calcite_view_optimize
bool g_enable_bump_allocator
Definition: Execute.cpp:99
bool g_strip_join_covered_quals
Definition: Execute.cpp:90
std::string ssl_key_file
bool g_enable_direct_columnarization
Definition: Execute.cpp:101
std::once_flag g_shutdown_once_flag
Definition: MapDServer.cpp:108
#define LOG(tag)
Definition: Logger.h:185
size_t gpu_buffer_mem_bytes
bool g_enable_debug_timer
Definition: Logger.cpp:17
static const std::string nodeIds_token
Definition: MapDServer.cpp:352
std::string ldapRoleRegex
Definition: AuthMetadata.h:28
shared utility for mapd_server and string dictionary server to remove old files
std::string cluster_topology_file
Definition: MapDServer.cpp:282
void run_warmup_queries(mapd::shared_ptr< MapDHandler > handler, std::string base_path, std::string query_file_path)
Definition: MapDServer.cpp:192
static constexpr int64_t kMinsPerHour
std::string ssl_trust_store
const std::string OMNISCI_SYSTEM_CATALOG
Definition: SysCatalog.h:57
std::string udf_file_name
Definition: MapDServer.cpp:334
size_t g_filter_push_down_passing_row_ubound
Definition: Execute.cpp:85
void heartbeat()
Definition: MapDServer.cpp:984
bool g_enable_dynamic_watchdog
Definition: Execute.cpp:72
unsigned g_trivial_loop_join_threshold
Definition: Execute.cpp:76
size_t render_poly_cache_bytes
Definition: MapDServer.cpp:299
int g_hll_precision_bits
size_t cuda_block_size
unsigned connect_timeout
Definition: MapDServer.cpp:70
po::options_description help_desc
Definition: MapDServer.cpp:339
std::string to_string(char const *&&v)
MapDProgramOptions(char const *argv0, bool dist_v5_=false)
Definition: MapDServer.cpp:272
bool g_enable_overlaps_hashjoin
Definition: Execute.cpp:87
size_t g_max_memory_allocation_size
Definition: Execute.cpp:95
bool g_enable_watchdog
bool g_inner_join_fragment_skipping
Definition: Execute.cpp:78
boost::program_options::options_description const & get_options() const
Definition: Logger.cpp:119
static constexpr int64_t kMinsPerMonth
std::string ha_shared_data
const bool dist_v5_
Definition: MapDServer.cpp:358
bool g_null_div_by_zero
Definition: Execute.cpp:75
std::string restToken
Definition: AuthMetadata.h:32
TThreadedServer * g_thrift_http_server
Definition: MapDServer.cpp:89
bool g_enable_columnar_output
Definition: Execute.cpp:86
std::string getNodeIds()
void releaseWarmupSession(TSessionId &sessionId, std::ifstream &query_file)
Definition: MapDServer.cpp:185
std::string license_path
Definition: MapDServer.cpp:283
MapDParameters mapd_parameters
Definition: MapDServer.cpp:294
po::options_description developer_desc
Definition: MapDServer.cpp:340
bool g_from_table_reordering
Definition: Execute.cpp:77
singleton class to handle concurrancy and state for blosc library. A C++ wraper over a pure C library...
size_t g_min_memory_allocation_size
Definition: Execute.cpp:96
size_t g_leaf_count
Definition: ParserNode.cpp:66
std::string ssl_keystore
void init(LogOptions const &log_opts)
Definition: Logger.cpp:265
std::string restUrl
Definition: AuthMetadata.h:31
std::vector< LeafHostInfo > db_leaves
Definition: MapDServer.cpp:345
TableGenerations table_generations_from_thrift(const std::vector< TTableGeneration > &thrift_table_generations)
Definition: MapDServer.cpp:92
std::string ssl_keystore_password
std::shared_timed_mutex mapd_shared_mutex
unsigned dynamic_watchdog_time_limit
Definition: MapDServer.cpp:305
size_t cuda_grid_size
float g_filter_push_down_high_frac
Definition: Execute.cpp:84
bool g_enable_thrift_logs
Definition: initdb.cpp:39
std::string ssl_trust_password
std::string uri
Definition: AuthMetadata.h:25
bool g_bigint_count
std::string base_path
Definition: MapDServer.cpp:279
void start_server(TThreadedServer &server, const int port)
Definition: MapDServer.cpp:177
size_t g_overlaps_max_table_size_bytes
Definition: Execute.cpp:89
bool enable_auto_clear_render_mem
Definition: MapDServer.cpp:296
std::string ssl_cert_file
std::string config_file
Definition: MapDServer.cpp:280
po::variables_map vm
Definition: MapDServer.cpp:347
size_t calcite_max_mem
bool g_enable_window_functions
Definition: Execute.cpp:93
void shutdown_handler()
Definition: MapDServer.cpp:110
std::atomic< int > g_saw_signal
Definition: MapDServer.cpp:86
std::string ldapSuperUserRole
Definition: AuthMetadata.h:29
void fillAdvancedOptions()
Definition: MapDServer.cpp:600
bool trim_and_check_file_exists(std::string &filename, const std::string desc)
Definition: MapDServer.cpp:754
std::string ha_brokers
AuthMetadata authMetadata
Definition: MapDServer.cpp:292
std::string db_query_file
Definition: MapDServer.cpp:320
void register_signal_handlers()
Definition: MapDServer.cpp:166
double gpu_input_mem_limit
void setGeneration(const uint32_t id, const TableGeneration &generation)
void shutdown()
Definition: Logger.cpp:299
bool g_enable_experimental_string_functions
unsigned send_timeout
Definition: MapDServer.cpp:72
std::atomic< bool > g_running
Definition: MapDServer.cpp:85
std::stringstream sanitize_config_file(std::ifstream &in)
Definition: MapDServer.cpp:741
size_t cpu_buffer_mem_bytes
std::string cluster_file
Definition: MapDServer.cpp:281
mapd::shared_ptr< MapDHandler > g_warmup_handler
Definition: MapDServer.cpp:104
void register_signal_handler(int signum, void(*handler)(int))
Definition: MapDServer.cpp:116
std::vector< std::string > getNodeIdsArray()
bool g_cache_string_hash
Definition: Execute.cpp:88
bool g_enable_table_functions
Definition: Execute.cpp:94
Severity severity_
Definition: Logger.h:116
bool g_enable_filter_push_down
Definition: Execute.cpp:82
FILE * open(int fileId)
Opens/creates the file with the given id; returns NULL on error.
Definition: File.cpp:83
bool g_skip_intermediate_count
bool allowLocalAuthFallback
Definition: AuthMetadata.h:33
std::vector< LeafHostInfo > string_leaves
Definition: MapDServer.cpp:346
size_t write(FILE *f, const size_t offset, const size_t size, int8_t *buf)
Writes the specified number of bytes to the offset position in file f from buf.
Definition: File.cpp:121
void set_base_path(std::string const &base_path)
Definition: Logger.cpp:133
po::positional_options_description positional_options
Definition: MapDServer.cpp:342
void close(FILE *f)
Closes the file pointed to by the FILE pointer.
Definition: File.cpp:102
bool g_allow_cpu_retry
Definition: Execute.cpp:74
int startMapdServer(MapDProgramOptions &prog_config_opts, bool start_http_server=true)
std::string ha_group_id
static const std::string MAPD_RELEASE
Definition: release.h:43
void file_delete(std::atomic< bool > &program_is_running, const unsigned int wait_interval_seconds, const std::string base_path)
Definition: File.cpp:228
static bool run
unsigned g_dynamic_watchdog_time_limit
Definition: Execute.cpp:73
void omnisci_signal_handler(int signum)
Definition: MapDServer.cpp:133
unsigned recv_timeout
Definition: MapDServer.cpp:71
#define VLOG(n)
Definition: Logger.h:280
logger::LogOptions log_options_
Definition: MapDServer.cpp:341
std::string ha_unique_server_id
std::string clusterIds_arg
Definition: MapDServer.cpp:348