OmniSciDB  2e3a973ef4
CommandLineOptions Class Reference

#include <CommandLineOptions.h>

+ Collaboration diagram for CommandLineOptions:

Public Member Functions

 CommandLineOptions (char const *argv0, bool dist_v5_=false)
 
void fillOptions ()
 
void fillAdvancedOptions ()
 
std::string getNodeIds ()
 
std::vector< std::string > getNodeIdsArray ()
 
boost::optional< int > parse_command_line (int argc, char const *const *argv, const bool should_init_logging=false)
 
void validate ()
 
void validate_base_path ()
 
void init_logging ()
 

Public Attributes

int http_port = 6278
 
size_t reserved_gpu_mem = 384 * 1024 * 1024
 
std::string base_path
 
DiskCacheConfig disk_cache_config
 
std::string cluster_file = {"cluster.conf"}
 
std::string cluster_topology_file = {"cluster_topology.conf"}
 
std::string license_path = {""}
 
std::string encryption_key_store_path = {}
 
bool cpu_only = false
 
bool verbose_logging = false
 
bool jit_debug = false
 
bool intel_jit_profile = false
 
bool allow_multifrag = true
 
bool read_only = false
 
bool allow_loop_joins = false
 
bool enable_legacy_syntax = true
 
bool log_user_origin = true
 
AuthMetadata authMetadata
 
SystemParameters system_parameters
 
bool enable_rendering = false
 
bool enable_auto_clear_render_mem = false
 
int render_oom_retry_threshold = 0
 
size_t render_mem_bytes = 1000000000
 
size_t max_concurrent_render_sessions = 500
 
bool render_compositor_use_last_gpu = true
 
bool renderer_use_vulkan_driver = false
 
bool enable_runtime_udf = false
 
bool enable_watchdog = true
 
bool enable_dynamic_watchdog = false
 
bool enable_runtime_query_interrupt = false
 
bool use_estimator_result_cache = true
 
unsigned runtime_query_interrupt_frequency = 1000
 
unsigned dynamic_watchdog_time_limit = 10000
 
std::string disk_cache_level = ""
 
int num_gpus = -1
 
int start_gpu = 0
 
size_t num_reader_threads = 0
 
std::string db_query_file = {""}
 
bool exit_after_warmup = false
 
int idle_session_duration = kMinsPerHour
 
int max_session_duration = kMinsPerMonth
 
std::string udf_file_name = {""}
 
std::string udf_compiler_path = {""}
 
std::vector< std::string > udf_compiler_options
 
std::string compressor = std::string(BLOSC_LZ4HC_COMPNAME)
 
po::options_description help_desc
 
po::options_description developer_desc
 
logger::LogOptions log_options_
 
po::positional_options_description positional_options
 
std::vector< LeafHostInfodb_leaves
 
std::vector< LeafHostInfostring_leaves
 
po::variables_map vm
 
std::string clusterIds_arg
 
const bool dist_v5_
 

Static Public Attributes

static const std::string nodeIds_token = {"node_id"}
 

Detailed Description

Definition at line 39 of file CommandLineOptions.h.

Constructor & Destructor Documentation

◆ CommandLineOptions()

CommandLineOptions::CommandLineOptions ( char const *  argv0,
bool  dist_v5_ = false 
)
inline

Definition at line 41 of file CommandLineOptions.h.

References fillAdvancedOptions(), and fillOptions().

42  : log_options_(argv0), dist_v5_(dist_v5_) {
43  fillOptions();
45  }
logger::LogOptions log_options_
+ Here is the call graph for this function:

Member Function Documentation

◆ fillAdvancedOptions()

void CommandLineOptions::fillAdvancedOptions ( )

Definition at line 370 of file CommandLineOptions.cpp.

References allow_multifrag, authMetadata, AuthMetadata::ca_file_name, developer_desc, SystemParameters::enable_calcite_view_optimize, enable_legacy_syntax, g_bitmap_memory_limit, g_enable_bump_allocator, g_enable_columnar_output, g_enable_direct_columnarization, g_enable_filter_function, g_enable_smem_group_by, g_enable_smem_grouped_non_count_agg, g_enable_smem_non_grouped_agg, g_enable_table_functions, g_enable_window_functions, g_fraction_code_cache_to_evict, g_gpu_smem_threshold, g_large_ndv_multiplier, g_large_ndv_threshold, g_max_memory_allocation_size, g_min_memory_allocation_size, g_skip_intermediate_count, g_strip_join_covered_quals, g_use_table_device_offset, g_use_tbb_pool, intel_jit_profile, jit_debug, SystemParameters::max_cpu_slab_size, SystemParameters::max_gpu_slab_size, SystemParameters::min_cpu_slab_size, SystemParameters::min_gpu_slab_size, SystemParameters::num_executors, SystemParameters::ssl_cert_file, SystemParameters::ssl_key_file, SystemParameters::ssl_keystore, SystemParameters::ssl_keystore_password, SystemParameters::ssl_trust_ca_file, SystemParameters::ssl_trust_password, SystemParameters::ssl_trust_store, system_parameters, udf_compiler_options, udf_compiler_path, and udf_file_name.

Referenced by CommandLineOptions().

370  {
371  developer_desc.add_options()("dev-options", "Print internal developer options.");
372  developer_desc.add_options()(
373  "enable-calcite-view-optimize",
376  ->implicit_value(true),
377  "Enable additional calcite (query plan) optimizations when a view is part of the "
378  "query.");
379  developer_desc.add_options()(
380  "enable-columnar-output",
381  po::value<bool>(&g_enable_columnar_output)
382  ->default_value(g_enable_columnar_output)
383  ->implicit_value(true),
384  "Enable columnar output for intermediate/final query steps.");
385  developer_desc.add_options()("enable-legacy-syntax",
386  po::value<bool>(&enable_legacy_syntax)
387  ->default_value(enable_legacy_syntax)
388  ->implicit_value(true),
389  "Enable legacy syntax.");
390  developer_desc.add_options()(
391  "enable-multifrag",
392  po::value<bool>(&allow_multifrag)
393  ->default_value(allow_multifrag)
394  ->implicit_value(true),
395  "Enable execution over multiple fragments in a single round-trip to GPU.");
396  developer_desc.add_options()(
397  "enable-shared-mem-group-by",
398  po::value<bool>(&g_enable_smem_group_by)
399  ->default_value(g_enable_smem_group_by)
400  ->implicit_value(true),
401  "Enable using GPU shared memory for some GROUP BY queries.");
402  developer_desc.add_options()("num-executors",
403  po::value<int>(&system_parameters.num_executors)
404  ->default_value(system_parameters.num_executors),
405  "Number of executors to run in parallel.");
406  developer_desc.add_options()(
407  "gpu-shared-mem-threshold",
408  po::value<size_t>(&g_gpu_smem_threshold)->default_value(g_gpu_smem_threshold),
409  "GPU shared memory threshold (in bytes). If query requires larger buffers than "
410  "this threshold, we disable those optimizations. 0 (default) means no static cap.");
411  developer_desc.add_options()(
412  "enable-shared-mem-grouped-non-count-agg",
413  po::value<bool>(&g_enable_smem_grouped_non_count_agg)
414  ->default_value(g_enable_smem_grouped_non_count_agg)
415  ->implicit_value(true),
416  "Enable using GPU shared memory for grouped non-count aggregate queries.");
417  developer_desc.add_options()(
418  "enable-shared-mem-non-grouped-agg",
419  po::value<bool>(&g_enable_smem_non_grouped_agg)
420  ->default_value(g_enable_smem_non_grouped_agg)
421  ->implicit_value(true),
422  "Enable using GPU shared memory for non-grouped aggregate queries.");
423  developer_desc.add_options()("enable-direct-columnarization",
424  po::value<bool>(&g_enable_direct_columnarization)
425  ->default_value(g_enable_direct_columnarization)
426  ->implicit_value(true),
427  "Enables/disables a more optimized columnarization method "
428  "for intermediate steps in multi-step queries.");
429  developer_desc.add_options()(
430  "offset-device-by-table-id",
431  po::value<bool>(&g_use_table_device_offset)
432  ->default_value(g_use_table_device_offset)
433  ->implicit_value(true),
434  "Enables/disables offseting the chosen device ID by the table ID for a given "
435  "fragment. This improves balance of fragments across GPUs.");
436  developer_desc.add_options()("enable-window-functions",
437  po::value<bool>(&g_enable_window_functions)
438  ->default_value(g_enable_window_functions)
439  ->implicit_value(true),
440  "Enable experimental window function support.");
441  developer_desc.add_options()("enable-table-functions",
442  po::value<bool>(&g_enable_table_functions)
443  ->default_value(g_enable_table_functions)
444  ->implicit_value(true),
445  "Enable experimental table functions support.");
446  developer_desc.add_options()(
447  "jit-debug-ir",
448  po::value<bool>(&jit_debug)->default_value(jit_debug)->implicit_value(true),
449  "Enable runtime debugger support for the JIT. Note that this flag is "
450  "incompatible "
451  "with the `ENABLE_JIT_DEBUG` build flag. The generated code can be found at "
452  "`/tmp/mapdquery`.");
453  developer_desc.add_options()(
454  "intel-jit-profile",
455  po::value<bool>(&intel_jit_profile)
456  ->default_value(intel_jit_profile)
457  ->implicit_value(true),
458  "Enable runtime support for the JIT code profiling using Intel VTune.");
459  developer_desc.add_options()(
460  "enable-modern-thread-pool",
461  po::value<bool>(&g_use_tbb_pool)
462  ->default_value(g_use_tbb_pool)
463  ->implicit_value(true),
464  "Enable a new thread pool implementation for queuing kernels for execution.");
465  developer_desc.add_options()(
466  "skip-intermediate-count",
467  po::value<bool>(&g_skip_intermediate_count)
468  ->default_value(g_skip_intermediate_count)
469  ->implicit_value(true),
470  "Skip pre-flight counts for intermediate projections with no filters.");
471  developer_desc.add_options()(
472  "strip-join-covered-quals",
473  po::value<bool>(&g_strip_join_covered_quals)
474  ->default_value(g_strip_join_covered_quals)
475  ->implicit_value(true),
476  "Remove quals from the filtered count if they are covered by a "
477  "join condition (currently only ST_Contains).");
478 
479  developer_desc.add_options()(
480  "min-cpu-slab-size",
481  po::value<size_t>(&system_parameters.min_cpu_slab_size)
482  ->default_value(system_parameters.min_cpu_slab_size),
483  "Min slab size (size of memory allocations) for CPU buffer pool.");
484  developer_desc.add_options()(
485  "max-cpu-slab-size",
486  po::value<size_t>(&system_parameters.max_cpu_slab_size)
487  ->default_value(system_parameters.max_cpu_slab_size),
488  "Max CPU buffer pool slab size (size of memory allocations). Note if "
489  "there is not enough free memory to accomodate the target slab size, smaller "
490  "slabs will be allocated, down to the minimum size specified by "
491  "min-cpu-slab-size.");
492  developer_desc.add_options()(
493  "min-gpu-slab-size",
494  po::value<size_t>(&system_parameters.min_gpu_slab_size)
495  ->default_value(system_parameters.min_gpu_slab_size),
496  "Min slab size (size of memory allocations) for GPU buffer pools.");
497  developer_desc.add_options()(
498  "max-gpu-slab-size",
499  po::value<size_t>(&system_parameters.max_gpu_slab_size)
500  ->default_value(system_parameters.max_gpu_slab_size),
501  "Max GPU buffer pool slab size (size of memory allocations). Note if "
502  "there is not enough free memory to accomodate the target slab size, smaller "
503  "slabs will be allocated, down to the minimum size speified by "
504  "min-gpu-slab-size.");
505 
506  developer_desc.add_options()(
507  "max-output-projection-allocation-bytes",
508  po::value<size_t>(&g_max_memory_allocation_size)
509  ->default_value(g_max_memory_allocation_size),
510  "Maximum allocation size for a fixed output buffer allocation for projection "
511  "queries with no pre-flight count. Default is the maximum slab size (sizes "
512  "greater "
513  "than the maximum slab size have no affect). Requires bump allocator.");
514  developer_desc.add_options()(
515  "min-output-projection-allocation-bytes",
516  po::value<size_t>(&g_min_memory_allocation_size)
517  ->default_value(g_min_memory_allocation_size),
518  "Minimum allocation size for a fixed output buffer allocation for projection "
519  "queries with no pre-flight count. If an allocation of this size cannot be "
520  "obtained, the query will be retried with different execution parameters and/or "
521  "on "
522  "CPU (if allow-cpu-retry is enabled). Requires bump allocator.");
523  developer_desc.add_options()("enable-bump-allocator",
524  po::value<bool>(&g_enable_bump_allocator)
525  ->default_value(g_enable_bump_allocator)
526  ->implicit_value(true),
527  "Enable the bump allocator for projection queries on "
528  "GPU. The bump allocator will "
529  "allocate a fixed size buffer for each query, track the "
530  "number of rows passing the "
531  "kernel during query execution, and copy back only the "
532  "rows that passed the kernel "
533  "to CPU after execution. When disabled, pre-flight "
534  "count queries are used to size "
535  "the output buffer for projection queries.");
536  developer_desc.add_options()(
537  "code-cache-eviction-percent",
538  po::value<float>(&g_fraction_code_cache_to_evict)
539  ->default_value(g_fraction_code_cache_to_evict),
540  "Percentage of the GPU code cache to evict if an out of memory error is "
541  "encountered while attempting to place generated code on the GPU.");
542 
543  developer_desc.add_options()("ssl-cert",
544  po::value<std::string>(&system_parameters.ssl_cert_file)
545  ->default_value(std::string("")),
546  "SSL Validated public certficate.");
547 
548  developer_desc.add_options()("ssl-private-key",
549  po::value<std::string>(&system_parameters.ssl_key_file)
550  ->default_value(std::string("")),
551  "SSL private key file.");
552  // Note ssl_trust_store is passed through to Calcite via system_parameters
553  // todo(jack): add ensure ssl-trust-store exists if cert and private key in use
554  developer_desc.add_options()("ssl-trust-store",
555  po::value<std::string>(&system_parameters.ssl_trust_store)
556  ->default_value(std::string("")),
557  "SSL public CA certifcates (java trust store) to validate "
558  "TLS connections (passed through to the Calcite server).");
559 
560  developer_desc.add_options()(
561  "ssl-trust-password",
562  po::value<std::string>(&system_parameters.ssl_trust_password)
563  ->default_value(std::string("")),
564  "SSL password for java trust store provided via --ssl-trust-store parameter.");
565 
566  developer_desc.add_options()(
567  "ssl-trust-ca",
568  po::value<std::string>(&system_parameters.ssl_trust_ca_file)
569  ->default_value(std::string("")),
570  "SSL public CA certificates to validate TLS connection(as a client).");
571 
572  developer_desc.add_options()(
573  "ssl-trust-ca-server",
574  po::value<std::string>(&authMetadata.ca_file_name)->default_value(std::string("")),
575  "SSL public CA certificates to validate TLS connection(as a server).");
576 
577  developer_desc.add_options()("ssl-keystore",
578  po::value<std::string>(&system_parameters.ssl_keystore)
579  ->default_value(std::string("")),
580  "SSL server credentials as a java key store (passed "
581  "through to the Calcite server).");
582 
583  developer_desc.add_options()(
584  "ssl-keystore-password",
585  po::value<std::string>(&system_parameters.ssl_keystore_password)
586  ->default_value(std::string("")),
587  "SSL password for java keystore, provide by via --ssl-keystore.");
588 
589  developer_desc.add_options()(
590  "udf",
591  po::value<std::string>(&udf_file_name),
592  "Load user defined extension functions from this file at startup. The file is "
593  "expected to be a C/C++ file with extension .cpp.");
594 
595  developer_desc.add_options()(
596  "udf-compiler-path",
597  po::value<std::string>(&udf_compiler_path),
598  "Provide absolute path to clang++ used in udf compilation.");
599 
600  developer_desc.add_options()("udf-compiler-options",
601  po::value<std::vector<std::string>>(&udf_compiler_options),
602  "Specify compiler options to tailor udf compilation.");
603 
604 #ifdef ENABLE_GEOS
605  developer_desc.add_options()("libgeos-so-filename",
606  po::value<std::string>(&libgeos_so_filename),
607  "Specify libgeos shared object filename to be used for "
608  "geos-backed geo opertations.");
609 #endif
610  developer_desc.add_options()(
611  "large-ndv-threshold",
612  po::value<int64_t>(&g_large_ndv_threshold)->default_value(g_large_ndv_threshold));
613  developer_desc.add_options()(
614  "large-ndv-multiplier",
615  po::value<size_t>(&g_large_ndv_multiplier)->default_value(g_large_ndv_multiplier));
616  developer_desc.add_options()(
617  "bitmap-memory-limit",
618  po::value<int64_t>(&g_bitmap_memory_limit)->default_value(g_bitmap_memory_limit),
619  "Limit for count distinct bitmap memory use. The limit is computed by taking the "
620  "size of the group by buffer (entry count in Query Memory Descriptor) and "
621  "multiplying it by the number of count distinct expression and the size of bitmap "
622  "required for each. For approx_count_distinct this is typically 8192 bytes.");
623  developer_desc.add_options()(
624  "enable-filter-function",
625  po::value<bool>(&g_enable_filter_function)
626  ->default_value(g_enable_filter_function)
627  ->implicit_value(true),
628  "Enable the filter function protection feature for the SQL JIT compiler. "
629  "Normally should be on but techs might want to disable for troubleshooting.");
630 }
bool g_enable_smem_group_by
bool g_strip_join_covered_quals
Definition: Execute.cpp:95
int64_t g_large_ndv_threshold
bool g_enable_direct_columnarization
Definition: Execute.cpp:106
std::string udf_compiler_path
bool g_skip_intermediate_count
bool enable_calcite_view_optimize
float g_fraction_code_cache_to_evict
bool g_enable_smem_non_grouped_agg
Definition: Execute.cpp:119
std::string ssl_trust_ca_file
size_t g_large_ndv_multiplier
bool g_enable_columnar_output
Definition: Execute.cpp:91
std::string ssl_trust_store
size_t g_max_memory_allocation_size
Definition: Execute.cpp:100
std::string ca_file_name
Definition: AuthMetadata.h:31
std::string ssl_key_file
AuthMetadata authMetadata
bool g_enable_window_functions
Definition: Execute.cpp:98
size_t g_min_memory_allocation_size
Definition: Execute.cpp:101
bool g_enable_smem_grouped_non_count_agg
Definition: Execute.cpp:116
std::vector< std::string > udf_compiler_options
bool g_enable_filter_function
Definition: Execute.cpp:77
std::string ssl_keystore_password
std::string ssl_trust_password
bool g_enable_bump_allocator
Definition: Execute.cpp:104
bool g_use_table_device_offset
std::string ssl_keystore
po::options_description developer_desc
bool g_use_tbb_pool
Definition: Execute.cpp:76
int64_t g_bitmap_memory_limit
bool g_enable_table_functions
Definition: Execute.cpp:99
SystemParameters system_parameters
size_t g_gpu_smem_threshold
Definition: Execute.cpp:111
std::string ssl_cert_file
+ Here is the caller graph for this function:

◆ fillOptions()

void CommandLineOptions::fillOptions ( )

Definition at line 59 of file CommandLineOptions.cpp.

References allow_loop_joins, base_path, SystemParameters::calcite_keepalive, SystemParameters::calcite_max_mem, SystemParameters::calcite_port, SystemParameters::calcite_timeout, SystemParameters::config_file, SystemParameters::cpu_buffer_mem_bytes, cpu_only, SystemParameters::cuda_block_size, SystemParameters::cuda_grid_size, db_query_file, disk_cache_config, disk_cache_level, dist_v5_, dynamic_watchdog_time_limit, enable_dynamic_watchdog, enable_runtime_query_interrupt, enable_runtime_udf, enable_watchdog, encryption_key_store_path, exit_after_warmup, g_allow_cpu_retry, g_bigint_count, g_cache_string_hash, g_enable_debug_timer, g_enable_experimental_string_functions, g_enable_filter_push_down, g_enable_fsi, g_enable_hashjoin_many_to_many, g_enable_interop, g_enable_overlaps_hashjoin, g_enable_stringdict_parallel, g_enable_thrift_logs, g_enable_union, g_filter_push_down_high_frac, g_filter_push_down_low_frac, g_filter_push_down_passing_row_ubound, g_from_table_reordering, g_hll_precision_bits, g_inner_join_fragment_skipping, g_null_div_by_zero, g_overlaps_max_table_size_bytes, g_trivial_loop_join_threshold, logger::LogOptions::get_options(), SystemParameters::gpu_buffer_mem_bytes, SystemParameters::gpu_input_mem_limit, help_desc, http_port, idle_session_duration, log_options_, log_user_origin, max_session_duration, num_gpus, num_reader_threads, SystemParameters::omnisci_server_port, DiskCacheConfig::path, positional_options, read_only, run_benchmark_import::required, reserved_gpu_mem, runtime_query_interrupt_frequency, DiskCacheConfig::size_limit, start_gpu, system_parameters, use_estimator_result_cache, and verbose_logging.

Referenced by CommandLineOptions().

59  {
60  help_desc.add_options()("help,h", "Show available options.");
61  help_desc.add_options()(
62  "allow-cpu-retry",
63  po::value<bool>(&g_allow_cpu_retry)
64  ->default_value(g_allow_cpu_retry)
65  ->implicit_value(true),
66  R"(Allow the queries which failed on GPU to retry on CPU, even when watchdog is enabled.)");
67  help_desc.add_options()("allow-loop-joins",
68  po::value<bool>(&allow_loop_joins)
69  ->default_value(allow_loop_joins)
70  ->implicit_value(true),
71  "Enable loop joins.");
72  help_desc.add_options()("bigint-count",
73  po::value<bool>(&g_bigint_count)
74  ->default_value(g_bigint_count)
75  ->implicit_value(true),
76  "Use 64-bit count.");
77  help_desc.add_options()("calcite-max-mem",
78  po::value<size_t>(&system_parameters.calcite_max_mem)
79  ->default_value(system_parameters.calcite_max_mem),
80  "Max memory available to calcite JVM.");
81  if (!dist_v5_) {
82  help_desc.add_options()("calcite-port",
83  po::value<int>(&system_parameters.calcite_port)
84  ->default_value(system_parameters.calcite_port),
85  "Calcite port number.");
86  }
87  help_desc.add_options()("config",
88  po::value<std::string>(&system_parameters.config_file),
89  "Path to server configuration file.");
90  help_desc.add_options()("cpu-buffer-mem-bytes",
91  po::value<size_t>(&system_parameters.cpu_buffer_mem_bytes)
92  ->default_value(system_parameters.cpu_buffer_mem_bytes),
93  "Size of memory reserved for CPU buffers, in bytes.");
94 
95  help_desc.add_options()(
96  "cpu-only",
97  po::value<bool>(&cpu_only)->default_value(cpu_only)->implicit_value(true),
98  "Run on CPU only, even if GPUs are available.");
99  help_desc.add_options()("cuda-block-size",
100  po::value<size_t>(&system_parameters.cuda_block_size)
101  ->default_value(system_parameters.cuda_block_size),
102  "Size of block to use on GPU.");
103  help_desc.add_options()("cuda-grid-size",
104  po::value<size_t>(&system_parameters.cuda_grid_size)
105  ->default_value(system_parameters.cuda_grid_size),
106  "Size of grid to use on GPU.");
107  if (!dist_v5_) {
108  help_desc.add_options()(
109  "data",
110  po::value<std::string>(&base_path)->required()->default_value("data"),
111  "Directory path to OmniSci data storage (catalogs, raw data, log files, etc).");
112  positional_options.add("data", 1);
113  }
114  help_desc.add_options()("db-query-list",
115  po::value<std::string>(&db_query_file),
116  "Path to file containing OmniSci warmup queries.");
117  help_desc.add_options()(
118  "exit-after-warmup",
119  po::value<bool>(&exit_after_warmup)->default_value(false)->implicit_value(true),
120  "Exit after OmniSci warmup queries.");
121  help_desc.add_options()("dynamic-watchdog-time-limit",
122  po::value<unsigned>(&dynamic_watchdog_time_limit)
123  ->default_value(dynamic_watchdog_time_limit)
124  ->implicit_value(10000),
125  "Dynamic watchdog time limit, in milliseconds.");
126  help_desc.add_options()("enable-debug-timer",
127  po::value<bool>(&g_enable_debug_timer)
128  ->default_value(g_enable_debug_timer)
129  ->implicit_value(true),
130  "Enable debug timer logging.");
131  help_desc.add_options()("enable-dynamic-watchdog",
132  po::value<bool>(&enable_dynamic_watchdog)
133  ->default_value(enable_dynamic_watchdog)
134  ->implicit_value(true),
135  "Enable dynamic watchdog.");
136  help_desc.add_options()("enable-filter-push-down",
137  po::value<bool>(&g_enable_filter_push_down)
138  ->default_value(g_enable_filter_push_down)
139  ->implicit_value(true),
140  "Enable filter push down through joins.");
141  help_desc.add_options()("enable-overlaps-hashjoin",
142  po::value<bool>(&g_enable_overlaps_hashjoin)
143  ->default_value(g_enable_overlaps_hashjoin)
144  ->implicit_value(true),
145  "Enable the overlaps hash join framework allowing for range "
146  "join (e.g. spatial overlaps) computation using a hash table.");
147  help_desc.add_options()("enable-hashjoin-many-to-many",
148  po::value<bool>(&g_enable_hashjoin_many_to_many)
149  ->default_value(g_enable_hashjoin_many_to_many)
150  ->implicit_value(true),
151  "Enable the overlaps hash join framework allowing for range "
152  "join (e.g. spatial overlaps) computation using a hash table.");
153  help_desc.add_options()("enable-runtime-query-interrupt",
154  po::value<bool>(&enable_runtime_query_interrupt)
155  ->default_value(enable_runtime_query_interrupt)
156  ->implicit_value(true),
157  "Enable runtime query interrupt.");
158  help_desc.add_options()("runtime-query-interrupt-frequency",
159  po::value<unsigned>(&runtime_query_interrupt_frequency)
160  ->default_value(runtime_query_interrupt_frequency)
161  ->implicit_value(1000),
162  "A frequency of checking the request of runtime query "
163  "interrupt from user (in millisecond).");
164  help_desc.add_options()("use-estimator-result-cache",
165  po::value<bool>(&use_estimator_result_cache)
166  ->default_value(use_estimator_result_cache)
167  ->implicit_value(true),
168  "Use estimator result cache.");
169  if (!dist_v5_) {
170  help_desc.add_options()(
171  "enable-string-dict-hash-cache",
172  po::value<bool>(&g_cache_string_hash)
173  ->default_value(g_cache_string_hash)
174  ->implicit_value(true),
175  "Cache string hash values in the string dictionary server during import.");
176  }
177  help_desc.add_options()(
178  "enable-thrift-logs",
179  po::value<bool>(&g_enable_thrift_logs)
180  ->default_value(g_enable_thrift_logs)
181  ->implicit_value(true),
182  "Enable writing messages directly from thrift to stdout/stderr.");
183  help_desc.add_options()("enable-watchdog",
184  po::value<bool>(&enable_watchdog)
185  ->default_value(enable_watchdog)
186  ->implicit_value(true),
187  "Enable watchdog.");
188  help_desc.add_options()(
189  "filter-push-down-low-frac",
190  po::value<float>(&g_filter_push_down_low_frac)
191  ->default_value(g_filter_push_down_low_frac)
192  ->implicit_value(g_filter_push_down_low_frac),
193  "Lower threshold for selectivity of filters that are pushed down.");
194  help_desc.add_options()(
195  "filter-push-down-high-frac",
196  po::value<float>(&g_filter_push_down_high_frac)
197  ->default_value(g_filter_push_down_high_frac)
198  ->implicit_value(g_filter_push_down_high_frac),
199  "Higher threshold for selectivity of filters that are pushed down.");
200  help_desc.add_options()("filter-push-down-passing-row-ubound",
201  po::value<size_t>(&g_filter_push_down_passing_row_ubound)
203  ->implicit_value(g_filter_push_down_passing_row_ubound),
204  "Upperbound on the number of rows that should pass the filter "
205  "if the selectivity is less than "
206  "the high fraction threshold.");
207  help_desc.add_options()("from-table-reordering",
208  po::value<bool>(&g_from_table_reordering)
209  ->default_value(g_from_table_reordering)
210  ->implicit_value(true),
211  "Enable automatic table reordering in FROM clause.");
212  help_desc.add_options()("gpu-buffer-mem-bytes",
213  po::value<size_t>(&system_parameters.gpu_buffer_mem_bytes)
214  ->default_value(system_parameters.gpu_buffer_mem_bytes),
215  "Size of memory reserved for GPU buffers, in bytes, per GPU.");
216  help_desc.add_options()("gpu-input-mem-limit",
217  po::value<double>(&system_parameters.gpu_input_mem_limit)
218  ->default_value(system_parameters.gpu_input_mem_limit),
219  "Force query to CPU when input data memory usage exceeds this "
220  "percentage of available GPU memory.");
221  help_desc.add_options()(
222  "hll-precision-bits",
223  po::value<int>(&g_hll_precision_bits)
224  ->default_value(g_hll_precision_bits)
225  ->implicit_value(g_hll_precision_bits),
226  "Number of bits used from the hash value used to specify the bucket number.");
227  if (!dist_v5_) {
228  help_desc.add_options()("http-port",
229  po::value<int>(&http_port)->default_value(http_port),
230  "HTTP port number.");
231  }
232  help_desc.add_options()(
233  "idle-session-duration",
234  po::value<int>(&idle_session_duration)->default_value(idle_session_duration),
235  "Maximum duration of idle session.");
236  help_desc.add_options()("inner-join-fragment-skipping",
237  po::value<bool>(&g_inner_join_fragment_skipping)
238  ->default_value(g_inner_join_fragment_skipping)
239  ->implicit_value(true),
240  "Enable/disable inner join fragment skipping. This feature is "
241  "considered stable and is enabled by default. This "
242  "parameter will be removed in a future release.");
243  help_desc.add_options()(
244  "max-session-duration",
245  po::value<int>(&max_session_duration)->default_value(max_session_duration),
246  "Maximum duration of active session.");
247  help_desc.add_options()(
248  "null-div-by-zero",
249  po::value<bool>(&g_null_div_by_zero)
250  ->default_value(g_null_div_by_zero)
251  ->implicit_value(true),
252  "Return null on division by zero instead of throwing an exception.");
253  help_desc.add_options()(
254  "num-reader-threads",
255  po::value<size_t>(&num_reader_threads)->default_value(num_reader_threads),
256  "Number of reader threads to use.");
257  help_desc.add_options()(
258  "overlaps-max-table-size-bytes",
259  po::value<size_t>(&g_overlaps_max_table_size_bytes)
260  ->default_value(g_overlaps_max_table_size_bytes),
261  "The maximum size in bytes of the hash table for an overlaps hash join.");
262  if (!dist_v5_) {
263  help_desc.add_options()("port,p",
264  po::value<int>(&system_parameters.omnisci_server_port)
265  ->default_value(system_parameters.omnisci_server_port),
266  "TCP Port number.");
267  }
268  help_desc.add_options()("num-gpus",
269  po::value<int>(&num_gpus)->default_value(num_gpus),
270  "Number of gpus to use.");
271  help_desc.add_options()(
272  "read-only",
273  po::value<bool>(&read_only)->default_value(read_only)->implicit_value(true),
274  "Enable read-only mode.");
275 
276  help_desc.add_options()(
277  "res-gpu-mem",
278  po::value<size_t>(&reserved_gpu_mem)->default_value(reserved_gpu_mem),
279  "Reduces GPU memory available to the OmniSci allocator by this amount. Used for "
280  "compiled code cache and ancillary GPU functions and other processes that may also "
281  "be using the GPU concurrent with OmniSciDB.");
282 
283  help_desc.add_options()("start-gpu",
284  po::value<int>(&start_gpu)->default_value(start_gpu),
285  "First gpu to use.");
286  help_desc.add_options()("trivial-loop-join-threshold",
287  po::value<unsigned>(&g_trivial_loop_join_threshold)
288  ->default_value(g_trivial_loop_join_threshold)
289  ->implicit_value(1000),
290  "The maximum number of rows in the inner table of a loop join "
291  "considered to be trivially small.");
292  help_desc.add_options()("verbose",
293  po::value<bool>(&verbose_logging)
294  ->default_value(verbose_logging)
295  ->implicit_value(true),
296  "Write additional debug log messages to server logs.");
297  help_desc.add_options()(
298  "enable-runtime-udf",
299  po::value<bool>(&enable_runtime_udf)
300  ->default_value(enable_runtime_udf)
301  ->implicit_value(true),
302  "Enable runtime UDF registration by passing signatures and corresponding LLVM IR "
303  "to the `register_runtime_udf` endpoint. For use with the Python Remote Backend "
304  "Compiler server, packaged separately.");
305  help_desc.add_options()("version,v", "Print Version Number.");
306  help_desc.add_options()("enable-experimental-string-functions",
309  ->implicit_value(true),
310  "Enable experimental string functions.");
311 #ifdef ENABLE_FSI
312  help_desc.add_options()(
313  "enable-fsi",
314  po::value<bool>(&g_enable_fsi)->default_value(g_enable_fsi)->implicit_value(true),
315  "Enable foreign storage interface.");
316  help_desc.add_options()("encryption-key-store",
317  po::value<std::string>(&encryption_key_store_path),
318  "Path to directory where encryption related keys will reside.");
319  help_desc.add_options()("disk-cache-level",
320  po::value<std::string>(&(disk_cache_level))
321  ->default_value("fsi")
322  ->implicit_value("fsi"),
323  "Specify level of disk cache. Valid options are 'fsi', "
324  "'non_fsi, 'none', and 'all'.");
325  help_desc.add_options()("disk-cache-path",
326  po::value<std::string>(&disk_cache_config.path),
327  "Specify the path for the disk cache.");
328  help_desc.add_options()(
329  "disk-cache-size-limit",
330  po::value<std::size_t>(&(disk_cache_config.size_limit)),
331  "Specify the maximum size of the the disk cache per table in bytes.");
332 #endif // ENABLE_FSI
333  help_desc.add_options()(
334  "enable-interoperability",
335  po::value<bool>(&g_enable_interop)
336  ->default_value(g_enable_interop)
337  ->implicit_value(true),
338  "Enable offloading of query portions to an external execution engine.");
339  help_desc.add_options()("enable-union",
340  po::value<bool>(&g_enable_union)
341  ->default_value(g_enable_union)
342  ->implicit_value(true),
343  "Enable UNION ALL SQL clause.");
344  help_desc.add_options()(
345  "calcite-service-timeout",
346  po::value<size_t>(&system_parameters.calcite_timeout)
347  ->default_value(system_parameters.calcite_timeout),
348  "Calcite server timeout (milliseconds). Increase this on systems with frequent "
349  "schema changes or when running large numbers of parallel queries.");
350  help_desc.add_options()("calcite-service-keepalive",
351  po::value<size_t>(&system_parameters.calcite_keepalive)
352  ->default_value(system_parameters.calcite_keepalive)
353  ->implicit_value(true),
354  "Enable keepalive on Calcite connections.");
355  help_desc.add_options()(
356  "stringdict-parallelizm",
357  po::value<bool>(&g_enable_stringdict_parallel)
358  ->default_value(g_enable_stringdict_parallel)
359  ->implicit_value(true),
360  "Allow StringDictionary to parallelize loads using multiple threads");
361  help_desc.add_options()("log-user-origin",
362  po::value<bool>(&log_user_origin)
363  ->default_value(log_user_origin)
364  ->implicit_value(true),
365  "Lookup the origin of inbound connections by IP address/DNS "
366  "name, and print this information as part of stdlog.");
368 }
float g_filter_push_down_low_frac
Definition: Execute.cpp:88
logger::LogOptions log_options_
bool g_cache_string_hash
unsigned runtime_query_interrupt_frequency
po::options_description help_desc
bool g_enable_union
DiskCacheConfig disk_cache_config
bool g_enable_debug_timer
Definition: Logger.cpp:17
boost::program_options::options_description const & get_options() const
Definition: Logger.cpp:79
size_t g_filter_push_down_passing_row_ubound
Definition: Execute.cpp:90
unsigned g_trivial_loop_join_threshold
Definition: Execute.cpp:81
int g_hll_precision_bits
std::string config_file
std::string encryption_key_store_path
bool g_enable_overlaps_hashjoin
Definition: Execute.cpp:92
bool g_inner_join_fragment_skipping
Definition: Execute.cpp:83
bool g_null_div_by_zero
Definition: Execute.cpp:80
bool g_enable_interop
bool g_from_table_reordering
Definition: Execute.cpp:82
bool g_enable_hashjoin_many_to_many
Definition: Execute.cpp:93
float g_filter_push_down_high_frac
Definition: Execute.cpp:89
bool g_bigint_count
size_t g_overlaps_max_table_size_bytes
Definition: Execute.cpp:94
bool g_enable_thrift_logs
bool g_enable_experimental_string_functions
bool g_enable_filter_push_down
Definition: Execute.cpp:87
po::positional_options_description positional_options
bool g_allow_cpu_retry
Definition: Execute.cpp:79
bool g_enable_stringdict_parallel
std::string disk_cache_level
bool g_enable_fsi
Definition: Catalog.cpp:91
unsigned dynamic_watchdog_time_limit
SystemParameters system_parameters
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ getNodeIds()

std::string CommandLineOptions::getNodeIds ( )

◆ getNodeIdsArray()

std::vector<std::string> CommandLineOptions::getNodeIdsArray ( )

◆ init_logging()

void CommandLineOptions::init_logging ( )

Definition at line 50 of file CommandLineOptions.cpp.

References base_path, logger::DEBUG1, logger::init(), log_options_, logger::LogOptions::set_base_path(), logger::LogOptions::severity_, validate_base_path(), and verbose_logging.

Referenced by parse_command_line().

50  {
53  }
57 }
logger::LogOptions log_options_
void init(LogOptions const &log_opts)
Definition: Logger.cpp:280
Severity severity_
Definition: Logger.h:118
void set_base_path(std::string const &base_path)
Definition: Logger.cpp:93
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ parse_command_line()

boost::optional< int > CommandLineOptions::parse_command_line ( int  argc,
char const *const *  argv,
const bool  should_init_logging = false 
)

Definition at line 804 of file CommandLineOptions.cpp.

References AuthMetadata::allowLocalAuthFallback, authMetadata, AuthMetadata::ca_file_name, SystemParameters::calcite_max_mem, SystemParameters::calcite_port, SystemParameters::config_file, SystemParameters::cuda_block_size, SystemParameters::cuda_grid_size, developer_desc, AuthMetadata::distinguishedName, dynamic_watchdog_time_limit, SystemParameters::enable_calcite_view_optimize, enable_dynamic_watchdog, enable_runtime_query_interrupt, enable_runtime_udf, enable_watchdog, logger::ERROR, g_dynamic_watchdog_time_limit, g_enable_dynamic_watchdog, g_enable_filter_push_down, g_enable_runtime_query_interrupt, g_enable_watchdog, g_from_table_reordering, g_runtime_query_interrupt_frequency, g_use_estimator_result_cache, SystemParameters::ha_brokers, SystemParameters::ha_group_id, SystemParameters::ha_shared_data, SystemParameters::ha_unique_server_id, help_desc, logger::INFO, init_logging(), AuthMetadata::ldapQueryUrl, AuthMetadata::ldapRoleRegex, AuthMetadata::ldapSuperUserRole, LOG, MAPD_RELEASE, SystemParameters::max_cpu_slab_size, SystemParameters::max_gpu_slab_size, SystemParameters::min_cpu_slab_size, SystemParameters::min_gpu_slab_size, SystemParameters::omnisci_server_port, positional_options, run, runtime_query_interrupt_frequency, anonymous_namespace{CommandLineOptions.cpp}::sanitize_config_file(), SystemParameters::ssl_cert_file, SystemParameters::ssl_key_file, SystemParameters::ssl_keystore, SystemParameters::ssl_trust_ca_file, SystemParameters::ssl_trust_store, system_parameters, anonymous_namespace{CommandLineOptions.cpp}::trim_and_check_file_exists(), udf_compiler_options, udf_compiler_path, udf_file_name, AuthMetadata::uri, use_estimator_result_cache, and vm.

Referenced by main().

807  {
808  po::options_description all_desc("All options");
809  all_desc.add(help_desc).add(developer_desc);
810 
811  try {
812  po::store(po::command_line_parser(argc, argv)
813  .options(all_desc)
814  .positional(positional_options)
815  .run(),
816  vm);
817  po::notify(vm);
818 
819  if (vm.count("help")) {
820  std::cerr << "Usage: omnisci_server <data directory path> [-p <port number>] "
821  "[--http-port <http port number>] [--flush-log] [--version|-v]"
822  << std::endl
823  << std::endl;
824  std::cout << help_desc << std::endl;
825  return 0;
826  }
827  if (vm.count("dev-options")) {
828  std::cout << "Usage: omnisci_server <data directory path> [-p <port number>] "
829  "[--http-port <http port number>] [--flush-log] [--version|-v]"
830  << std::endl
831  << std::endl;
832  std::cout << developer_desc << std::endl;
833  return 0;
834  }
835  if (vm.count("version")) {
836  std::cout << "OmniSci Version: " << MAPD_RELEASE << std::endl;
837  return 0;
838  }
839 
840  if (vm.count("config")) {
841  std::ifstream settings_file(system_parameters.config_file);
842 
843  auto sanitized_settings = sanitize_config_file(settings_file);
844 
845  po::store(po::parse_config_file(sanitized_settings, all_desc, false), vm);
846  po::notify(vm);
847  settings_file.close();
848  }
849 
850  if (should_init_logging) {
851  init_logging();
852  }
853 
855  return 1;
856  }
857  if (!trim_and_check_file_exists(authMetadata.ca_file_name, "ca file name")) {
858  return 1;
859  }
861  return 1;
862  }
864  return 1;
865  }
867  return 1;
868  }
870  return 1;
871  }
872 
879  } catch (po::error& e) {
880  std::cerr << "Usage Error: " << e.what() << std::endl;
881  return 1;
882  }
883 
884  if (g_hll_precision_bits < 1 || g_hll_precision_bits > 16) {
885  std::cerr << "hll-precision-bits must be between 1 and 16." << std::endl;
886  return 1;
887  }
888 
890  LOG(INFO) << " From clause table reordering is disabled";
891  }
892 
894  LOG(INFO) << " Filter push down for JOIN is enabled";
895  }
896 
897  if (vm.count("udf")) {
898  boost::algorithm::trim_if(udf_file_name, boost::is_any_of("\"'"));
899 
900  if (!boost::filesystem::exists(udf_file_name)) {
901  LOG(ERROR) << " User defined function file " << udf_file_name << " does not exist.";
902  return 1;
903  }
904 
905  LOG(INFO) << " User provided extension functions loaded from " << udf_file_name;
906  }
907 
908  if (vm.count("udf-compiler-path")) {
909  boost::algorithm::trim_if(udf_compiler_path, boost::is_any_of("\"'"));
910  }
911 
912  auto trim_string = [](std::string& s) {
913  boost::algorithm::trim_if(s, boost::is_any_of("\"'"));
914  };
915 
916  if (vm.count("udf-compiler-options")) {
917  std::for_each(udf_compiler_options.begin(), udf_compiler_options.end(), trim_string);
918  }
919 
920  if (enable_runtime_udf) {
921  LOG(INFO) << " Runtime user defined extension functions enabled globally.";
922  }
923 
924  boost::algorithm::trim_if(system_parameters.ha_brokers, boost::is_any_of("\"'"));
925  boost::algorithm::trim_if(system_parameters.ha_group_id, boost::is_any_of("\"'"));
926  boost::algorithm::trim_if(system_parameters.ha_shared_data, boost::is_any_of("\"'"));
927  boost::algorithm::trim_if(system_parameters.ha_unique_server_id,
928  boost::is_any_of("\"'"));
929 
930  if (!system_parameters.ha_group_id.empty()) {
931  LOG(INFO) << " HA group id " << system_parameters.ha_group_id;
933  LOG(ERROR) << "Starting server in HA mode --ha-unique-server-id must be set ";
934  return 5;
935  } else {
936  LOG(INFO) << " HA unique server id " << system_parameters.ha_unique_server_id;
937  }
938  if (system_parameters.ha_brokers.empty()) {
939  LOG(ERROR) << "Starting server in HA mode --ha-brokers must be set ";
940  return 6;
941  } else {
942  LOG(INFO) << " HA brokers " << system_parameters.ha_brokers;
943  }
944  if (system_parameters.ha_shared_data.empty()) {
945  LOG(ERROR) << "Starting server in HA mode --ha-shared-data must be set ";
946  return 7;
947  } else {
948  LOG(INFO) << " HA shared data is " << system_parameters.ha_shared_data;
949  }
950  }
951  LOG(INFO) << " cuda block size " << system_parameters.cuda_block_size;
952  LOG(INFO) << " cuda grid size " << system_parameters.cuda_grid_size;
953  LOG(INFO) << " Min CPU buffer pool slab size " << system_parameters.min_cpu_slab_size;
954  LOG(INFO) << " Max CPU buffer pool slab size " << system_parameters.max_cpu_slab_size;
955  LOG(INFO) << " Min GPU buffer pool slab size " << system_parameters.min_gpu_slab_size;
956  LOG(INFO) << " Max GPU buffer pool slab size " << system_parameters.max_gpu_slab_size;
957  LOG(INFO) << " calcite JVM max memory " << system_parameters.calcite_max_mem;
958  LOG(INFO) << " OmniSci Server Port " << system_parameters.omnisci_server_port;
959  LOG(INFO) << " OmniSci Calcite Port " << system_parameters.calcite_port;
960  LOG(INFO) << " Enable Calcite view optimize "
962 
963  LOG(INFO) << " Allow Local Auth Fallback: "
964  << (authMetadata.allowLocalAuthFallback ? "enabled" : "disabled");
965 
966  boost::algorithm::trim_if(authMetadata.distinguishedName, boost::is_any_of("\"'"));
967  boost::algorithm::trim_if(authMetadata.uri, boost::is_any_of("\"'"));
968  boost::algorithm::trim_if(authMetadata.ldapQueryUrl, boost::is_any_of("\"'"));
969  boost::algorithm::trim_if(authMetadata.ldapRoleRegex, boost::is_any_of("\"'"));
970  boost::algorithm::trim_if(authMetadata.ldapSuperUserRole, boost::is_any_of("\"'"));
971 
972  return boost::none;
973 }
std::string distinguishedName
Definition: AuthMetadata.h:25
std::string ldapQueryUrl
Definition: AuthMetadata.h:26
bool trim_and_check_file_exists(std::string &filename, const std::string desc)
unsigned runtime_query_interrupt_frequency
std::string ha_shared_data
std::string udf_compiler_path
po::options_description help_desc
#define LOG(tag)
Definition: Logger.h:188
bool enable_calcite_view_optimize
std::string ldapRoleRegex
Definition: AuthMetadata.h:27
bool g_enable_dynamic_watchdog
Definition: Execute.cpp:75
std::string config_file
std::string ha_brokers
std::string ssl_trust_ca_file
std::string ssl_trust_store
bool g_from_table_reordering
Definition: Execute.cpp:82
std::string uri
Definition: AuthMetadata.h:24
std::string ha_unique_server_id
std::string ca_file_name
Definition: AuthMetadata.h:31
std::string ssl_key_file
AuthMetadata authMetadata
unsigned g_runtime_query_interrupt_frequency
Definition: Execute.cpp:110
std::string ldapSuperUserRole
Definition: AuthMetadata.h:28
std::stringstream sanitize_config_file(std::ifstream &in)
std::vector< std::string > udf_compiler_options
bool g_enable_filter_push_down
Definition: Execute.cpp:87
bool g_use_estimator_result_cache
Definition: Execute.cpp:109
bool allowLocalAuthFallback
Definition: AuthMetadata.h:32
po::positional_options_description positional_options
std::string ssl_keystore
bool g_enable_watchdog
Definition: Execute.cpp:74
po::options_description developer_desc
static const std::string MAPD_RELEASE
Definition: release.h:43
static bool run
po::variables_map vm
unsigned g_dynamic_watchdog_time_limit
Definition: Execute.cpp:78
std::string ha_group_id
bool g_enable_runtime_query_interrupt
Definition: Execute.cpp:108
unsigned dynamic_watchdog_time_limit
SystemParameters system_parameters
std::string ssl_cert_file
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ validate()

void CommandLineOptions::validate ( )

Definition at line 677 of file CommandLineOptions.cpp.

References anonymous_namespace{CommandLineOptions.cpp}::addOptionalFileToBlacklist(), ddl_utils::FilePathBlacklist::addToBlacklist(), all, authMetadata, base_path, AuthMetadata::ca_file_name, omnisci::close(), cluster_file, SystemParameters::config_file, db_query_file, disk_cache_config, disk_cache_level, dynamic_watchdog_time_limit, enable_dynamic_watchdog, enable_runtime_query_interrupt, enable_watchdog, DiskCacheConfig::enabled_level, fsi, g_enable_debug_timer, g_enable_fsi, idle_session_duration, logger::INFO, ddl_utils::FilePathWhitelist::initializeFromConfigFile(), license_path, LOG, max_session_duration, non_fsi, none, OMNISCI_SYSTEM_CATALOG, omnisci::open(), DiskCacheConfig::path, runtime_query_interrupt_frequency, SystemParameters::ssl_cert_file, SystemParameters::ssl_key_file, SystemParameters::ssl_keystore, SystemParameters::ssl_trust_ca_file, SystemParameters::ssl_trust_store, system_parameters, to_string(), vm, and File_Namespace::write().

Referenced by main().

677  {
678  boost::algorithm::trim_if(base_path, boost::is_any_of("\"'"));
679  const auto data_path = boost::filesystem::path(base_path) / "mapd_data";
680  if (!boost::filesystem::exists(data_path)) {
681  throw std::runtime_error("OmniSci data directory does not exist at '" + base_path +
682  "'");
683  }
684 
685  {
686  const auto lock_file = boost::filesystem::path(base_path) / "omnisci_server_pid.lck";
687  auto pid = std::to_string(getpid());
688 
689  int pid_fd = open(lock_file.c_str(), O_RDWR | O_CREAT, 0644);
690  if (pid_fd == -1) {
691  auto err = std::string("Failed to open PID file ") + lock_file.c_str() + ". " +
692  strerror(errno) + ".";
693  throw std::runtime_error(err);
694  }
695  if (lockf(pid_fd, F_TLOCK, 0) == -1) {
696  close(pid_fd);
697  auto err = std::string("Another OmniSci Server is using data directory ") +
698  base_path + ".";
699  throw std::runtime_error(err);
700  }
701  if (ftruncate(pid_fd, 0) == -1) {
702  close(pid_fd);
703  auto err = std::string("Failed to truncate PID file ") + lock_file.c_str() + ". " +
704  strerror(errno) + ".";
705  throw std::runtime_error(err);
706  }
707  if (write(pid_fd, pid.c_str(), pid.length()) == -1) {
708  close(pid_fd);
709  auto err = std::string("Failed to write PID file ") + lock_file.c_str() + ". " +
710  strerror(errno) + ".";
711  throw std::runtime_error(err);
712  }
713  }
714  boost::algorithm::trim_if(db_query_file, boost::is_any_of("\"'"));
715  if (db_query_file.length() > 0 && !boost::filesystem::exists(db_query_file)) {
716  throw std::runtime_error("File containing DB queries " + db_query_file +
717  " does not exist.");
718  }
719  const auto db_file =
720  boost::filesystem::path(base_path) / "mapd_catalogs" / OMNISCI_SYSTEM_CATALOG;
721  if (!boost::filesystem::exists(db_file)) {
722  { // check old system catalog existsense
723  const auto db_file = boost::filesystem::path(base_path) / "mapd_catalogs/mapd";
724  if (!boost::filesystem::exists(db_file)) {
725  throw std::runtime_error("OmniSci system catalog " + OMNISCI_SYSTEM_CATALOG +
726  " does not exist.");
727  }
728  }
729  }
730  if (license_path.length() == 0) {
731  license_path = base_path + "/omnisci.license";
732  }
733 
734  // add all parameters to be displayed on startup
735  LOG(INFO) << "OmniSci started with data directory at '" << base_path << "'";
736  if (vm.count("license-path")) {
737  LOG(INFO) << "License key path set to '" << license_path << "'";
738  }
739  LOG(INFO) << " Watchdog is set to " << enable_watchdog;
740  LOG(INFO) << " Dynamic Watchdog is set to " << enable_dynamic_watchdog;
742  LOG(INFO) << " Dynamic Watchdog timeout is set to " << dynamic_watchdog_time_limit;
743  }
744  LOG(INFO) << " Runtime query interrupt is set to " << enable_runtime_query_interrupt;
746  LOG(INFO) << " A frequency of checking runtime query interrupt request is set to "
747  << runtime_query_interrupt_frequency << " (in ms.)";
748  }
749 
750  LOG(INFO) << " Debug Timer is set to " << g_enable_debug_timer;
751 
752  LOG(INFO) << " Maximum Idle session duration " << idle_session_duration;
753 
754  LOG(INFO) << " Maximum active session duration " << max_session_duration;
755 
757 
761 
762  if (disk_cache_level == "fsi") {
763  if (g_enable_fsi) {
765  LOG(INFO) << "Disk cache enabled for foreign tables only";
766  } else {
767  LOG(INFO) << "Cannot enable disk cache for fsi when fsi is disabled. Defaulted to "
768  "disk cache disabled";
769  }
770  } else if (disk_cache_level == "all") {
772  LOG(INFO) << "Disk cache enabled for all tables";
773  } else if (disk_cache_level == "non_fsi") {
775  LOG(INFO) << "Disk cache enabled for non-FSI tables";
776  } else if (disk_cache_level == "none") {
778  LOG(INFO) << "Disk cache disabled";
779  } else {
781  LOG(INFO) << "Non-recognized value for disk-cache-level {" << disk_cache_level
782  << "}. Defaulted to disk cache disabled";
783  }
784 
785  if (disk_cache_config.path.empty()) {
786  disk_cache_config.path = base_path + "/omnisci_disk_cache";
787  }
789 
792 
793  // If passed in, blacklist all security config files
802 }
DiskCacheLevel enabled_level
int open(const char *path, int flags, int mode)
Definition: omnisci_fs.cpp:64
unsigned runtime_query_interrupt_frequency
#define LOG(tag)
Definition: Logger.h:188
DiskCacheConfig disk_cache_config
bool g_enable_debug_timer
Definition: Logger.cpp:17
static void initializeFromConfigFile(const std::string &server_config_path)
Definition: DdlUtils.cpp:661
const std::string OMNISCI_SYSTEM_CATALOG
Definition: SysCatalog.h:55
void addOptionalFileToBlacklist(std::string &filename)
std::string config_file
std::string to_string(char const *&&v)
std::string ssl_trust_ca_file
void close(const int fd)
Definition: omnisci_fs.cpp:68
std::string ssl_trust_store
std::string ca_file_name
Definition: AuthMetadata.h:31
std::string ssl_key_file
AuthMetadata authMetadata
size_t write(FILE *f, const size_t offset, const size_t size, int8_t *buf)
Writes the specified number of bytes to the offset position in file f from buf.
Definition: File.cpp:125
std::string ssl_keystore
static void addToBlacklist(const std::string &path)
Definition: DdlUtils.cpp:724
std::string disk_cache_level
po::variables_map vm
bool g_enable_fsi
Definition: Catalog.cpp:91
unsigned dynamic_watchdog_time_limit
SystemParameters system_parameters
std::string ssl_cert_file
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ validate_base_path()

void CommandLineOptions::validate_base_path ( )

Definition at line 670 of file CommandLineOptions.cpp.

References base_path.

Referenced by init_logging(), and main().

670  {
671  boost::algorithm::trim_if(base_path, boost::is_any_of("\"'"));
672  if (!boost::filesystem::exists(base_path)) {
673  throw std::runtime_error("OmniSci base directory does not exist at " + base_path);
674  }
675 }
+ Here is the caller graph for this function:

Member Data Documentation

◆ allow_loop_joins

bool CommandLineOptions::allow_loop_joins = false

Definition at line 60 of file CommandLineOptions.h.

Referenced by fillOptions(), and startMapdServer().

◆ allow_multifrag

bool CommandLineOptions::allow_multifrag = true

Definition at line 58 of file CommandLineOptions.h.

Referenced by fillAdvancedOptions(), and startMapdServer().

◆ authMetadata

AuthMetadata CommandLineOptions::authMetadata

◆ base_path

std::string CommandLineOptions::base_path

◆ cluster_file

std::string CommandLineOptions::cluster_file = {"cluster.conf"}

Definition at line 50 of file CommandLineOptions.h.

Referenced by validate().

◆ cluster_topology_file

std::string CommandLineOptions::cluster_topology_file = {"cluster_topology.conf"}

Definition at line 51 of file CommandLineOptions.h.

◆ clusterIds_arg

std::string CommandLineOptions::clusterIds_arg

Definition at line 133 of file CommandLineOptions.h.

◆ compressor

std::string CommandLineOptions::compressor = std::string(BLOSC_LZ4HC_COMPNAME)

Definition at line 122 of file CommandLineOptions.h.

◆ cpu_only

bool CommandLineOptions::cpu_only = false

Definition at line 54 of file CommandLineOptions.h.

Referenced by fillOptions(), and startMapdServer().

◆ db_leaves

std::vector<LeafHostInfo> CommandLineOptions::db_leaves

Definition at line 130 of file CommandLineOptions.h.

Referenced by startMapdServer().

◆ db_query_file

std::string CommandLineOptions::db_query_file = {""}

path to file containing warmup queries list

Definition at line 97 of file CommandLineOptions.h.

Referenced by fillOptions(), startMapdServer(), and validate().

◆ developer_desc

po::options_description CommandLineOptions::developer_desc

Definition at line 125 of file CommandLineOptions.h.

Referenced by fillAdvancedOptions(), and parse_command_line().

◆ disk_cache_config

DiskCacheConfig CommandLineOptions::disk_cache_config

Definition at line 49 of file CommandLineOptions.h.

Referenced by fillOptions(), startMapdServer(), and validate().

◆ disk_cache_level

std::string CommandLineOptions::disk_cache_level = ""

Definition at line 82 of file CommandLineOptions.h.

Referenced by fillOptions(), and validate().

◆ dist_v5_

const bool CommandLineOptions::dist_v5_

Definition at line 145 of file CommandLineOptions.h.

Referenced by fillOptions().

◆ dynamic_watchdog_time_limit

unsigned CommandLineOptions::dynamic_watchdog_time_limit = 10000

Definition at line 81 of file CommandLineOptions.h.

Referenced by fillOptions(), parse_command_line(), and validate().

◆ enable_auto_clear_render_mem

bool CommandLineOptions::enable_auto_clear_render_mem = false

Definition at line 67 of file CommandLineOptions.h.

Referenced by startMapdServer().

◆ enable_dynamic_watchdog

bool CommandLineOptions::enable_dynamic_watchdog = false

Definition at line 77 of file CommandLineOptions.h.

Referenced by fillOptions(), parse_command_line(), and validate().

◆ enable_legacy_syntax

bool CommandLineOptions::enable_legacy_syntax = true

Definition at line 61 of file CommandLineOptions.h.

Referenced by fillAdvancedOptions(), and startMapdServer().

◆ enable_rendering

bool CommandLineOptions::enable_rendering = false

Definition at line 66 of file CommandLineOptions.h.

Referenced by startMapdServer().

◆ enable_runtime_query_interrupt

bool CommandLineOptions::enable_runtime_query_interrupt = false

Definition at line 78 of file CommandLineOptions.h.

Referenced by fillOptions(), parse_command_line(), and validate().

◆ enable_runtime_udf

bool CommandLineOptions::enable_runtime_udf = false

Definition at line 74 of file CommandLineOptions.h.

Referenced by fillOptions(), parse_command_line(), and startMapdServer().

◆ enable_watchdog

bool CommandLineOptions::enable_watchdog = true

Definition at line 76 of file CommandLineOptions.h.

Referenced by fillOptions(), parse_command_line(), and validate().

◆ encryption_key_store_path

std::string CommandLineOptions::encryption_key_store_path = {}

Definition at line 53 of file CommandLineOptions.h.

Referenced by fillOptions().

◆ exit_after_warmup

bool CommandLineOptions::exit_after_warmup = false

exit after warmup

Definition at line 101 of file CommandLineOptions.h.

Referenced by fillOptions(), and startMapdServer().

◆ help_desc

po::options_description CommandLineOptions::help_desc

Definition at line 124 of file CommandLineOptions.h.

Referenced by fillOptions(), and parse_command_line().

◆ http_port

int CommandLineOptions::http_port = 6278

Definition at line 46 of file CommandLineOptions.h.

Referenced by fillOptions(), and startMapdServer().

◆ idle_session_duration

int CommandLineOptions::idle_session_duration = kMinsPerHour

Inactive session tolerance in mins (60 mins)

Definition at line 105 of file CommandLineOptions.h.

Referenced by fillOptions(), startMapdServer(), and validate().

◆ intel_jit_profile

bool CommandLineOptions::intel_jit_profile = false

Definition at line 57 of file CommandLineOptions.h.

Referenced by fillAdvancedOptions(), and startMapdServer().

◆ jit_debug

bool CommandLineOptions::jit_debug = false

Definition at line 56 of file CommandLineOptions.h.

Referenced by fillAdvancedOptions(), and startMapdServer().

◆ license_path

std::string CommandLineOptions::license_path = {""}

Definition at line 52 of file CommandLineOptions.h.

Referenced by validate().

◆ log_options_

logger::LogOptions CommandLineOptions::log_options_

Definition at line 126 of file CommandLineOptions.h.

Referenced by fillOptions(), and init_logging().

◆ log_user_origin

bool CommandLineOptions::log_user_origin = true

Definition at line 62 of file CommandLineOptions.h.

Referenced by fillOptions(), and startMapdServer().

◆ max_concurrent_render_sessions

size_t CommandLineOptions::max_concurrent_render_sessions = 500

Definition at line 70 of file CommandLineOptions.h.

Referenced by startMapdServer().

◆ max_session_duration

int CommandLineOptions::max_session_duration = kMinsPerMonth

Maximum session life in mins (43,200 mins == 30 Days) (https://pages.nist.gov/800-63-3/sp800-63b.html#aal3reauth)

Definition at line 110 of file CommandLineOptions.h.

Referenced by fillOptions(), startMapdServer(), and validate().

◆ nodeIds_token

const std::string CommandLineOptions::nodeIds_token = {"node_id"}
static

Definition at line 137 of file CommandLineOptions.h.

◆ num_gpus

int CommandLineOptions::num_gpus = -1

Can be used to override the number of gpus detected on the system -1 means do not override

Definition at line 88 of file CommandLineOptions.h.

Referenced by fillOptions(), and startMapdServer().

◆ num_reader_threads

size_t CommandLineOptions::num_reader_threads = 0

Number of threads used when loading data

Definition at line 93 of file CommandLineOptions.h.

Referenced by fillOptions(), and startMapdServer().

◆ positional_options

po::positional_options_description CommandLineOptions::positional_options

Definition at line 127 of file CommandLineOptions.h.

Referenced by fillOptions(), and parse_command_line().

◆ read_only

bool CommandLineOptions::read_only = false

Definition at line 59 of file CommandLineOptions.h.

Referenced by fillOptions(), and startMapdServer().

◆ render_compositor_use_last_gpu

bool CommandLineOptions::render_compositor_use_last_gpu = true

Definition at line 71 of file CommandLineOptions.h.

Referenced by startMapdServer().

◆ render_mem_bytes

size_t CommandLineOptions::render_mem_bytes = 1000000000

Definition at line 69 of file CommandLineOptions.h.

Referenced by startMapdServer().

◆ render_oom_retry_threshold

int CommandLineOptions::render_oom_retry_threshold = 0

Definition at line 68 of file CommandLineOptions.h.

Referenced by startMapdServer().

◆ renderer_use_vulkan_driver

bool CommandLineOptions::renderer_use_vulkan_driver = false

Definition at line 72 of file CommandLineOptions.h.

Referenced by startMapdServer().

◆ reserved_gpu_mem

size_t CommandLineOptions::reserved_gpu_mem = 384 * 1024 * 1024

Definition at line 47 of file CommandLineOptions.h.

Referenced by fillOptions(), and startMapdServer().

◆ runtime_query_interrupt_frequency

unsigned CommandLineOptions::runtime_query_interrupt_frequency = 1000

Definition at line 80 of file CommandLineOptions.h.

Referenced by fillOptions(), parse_command_line(), and validate().

◆ start_gpu

int CommandLineOptions::start_gpu = 0

Definition at line 89 of file CommandLineOptions.h.

Referenced by fillOptions(), and startMapdServer().

◆ string_leaves

std::vector<LeafHostInfo> CommandLineOptions::string_leaves

Definition at line 131 of file CommandLineOptions.h.

Referenced by startMapdServer().

◆ system_parameters

SystemParameters CommandLineOptions::system_parameters

◆ udf_compiler_options

std::vector<std::string> CommandLineOptions::udf_compiler_options

Definition at line 113 of file CommandLineOptions.h.

Referenced by fillAdvancedOptions(), parse_command_line(), and startMapdServer().

◆ udf_compiler_path

std::string CommandLineOptions::udf_compiler_path = {""}

Definition at line 112 of file CommandLineOptions.h.

Referenced by fillAdvancedOptions(), parse_command_line(), and startMapdServer().

◆ udf_file_name

std::string CommandLineOptions::udf_file_name = {""}

Definition at line 111 of file CommandLineOptions.h.

Referenced by fillAdvancedOptions(), parse_command_line(), and startMapdServer().

◆ use_estimator_result_cache

bool CommandLineOptions::use_estimator_result_cache = true

Definition at line 79 of file CommandLineOptions.h.

Referenced by fillOptions(), and parse_command_line().

◆ verbose_logging

bool CommandLineOptions::verbose_logging = false

Definition at line 55 of file CommandLineOptions.h.

Referenced by fillOptions(), and init_logging().

◆ vm

po::variables_map CommandLineOptions::vm

Definition at line 132 of file CommandLineOptions.h.

Referenced by parse_command_line(), and validate().


The documentation for this class was generated from the following files: