OmniSciDB  f17484ade4
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
CommandLineOptions Class Reference

#include <CommandLineOptions.h>

+ Collaboration diagram for CommandLineOptions:

Public Member Functions

 CommandLineOptions (char const *argv0, bool dist_v5_=false)
 
void fillOptions ()
 
void fillDeveloperOptions ()
 
std::string getNodeIds ()
 
std::vector< std::string > getNodeIdsArray ()
 
boost::optional< int > parse_command_line (int argc, char const *const *argv, const bool should_init_logging=false)
 
void validate ()
 
void validate_base_path ()
 
void init_logging ()
 

Public Attributes

int http_port = 6278
 
int http_binary_port = 6276
 
size_t reserved_gpu_mem = 384 * 1024 * 1024
 
std::string base_path
 
File_Namespace::DiskCacheConfig disk_cache_config
 
std::string cluster_file = {"cluster.conf"}
 
std::string cluster_topology_file = {"cluster_topology.conf"}
 
std::string license_path = {""}
 
std::string encryption_key_store_path = {}
 
bool verbose_logging = false
 
bool jit_debug = false
 
bool intel_jit_profile = false
 
bool allow_multifrag = true
 
bool read_only = false
 
bool allow_loop_joins = false
 
bool enable_legacy_syntax = true
 
bool log_user_origin = true
 
AuthMetadata authMetadata
 
SystemParameters system_parameters
 
bool enable_rendering = false
 
bool enable_auto_clear_render_mem = false
 
int render_oom_retry_threshold = 0
 
size_t render_mem_bytes = 1000000000
 
size_t max_concurrent_render_sessions = 500
 
bool render_compositor_use_last_gpu = true
 
bool renderer_prefer_igpu = false
 
unsigned renderer_vulkan_timeout_ms = 60000
 
bool renderer_use_parallel_executors = true
 
bool enable_watchdog = true
 
bool enable_dynamic_watchdog = false
 
size_t watchdog_none_encoded_string_translation_limit = 1000000
 
bool enable_runtime_query_interrupt = true
 
bool enable_non_kernel_time_query_interrupt = true
 
bool use_estimator_result_cache = true
 
double running_query_interrupt_freq = 0.1
 
unsigned pending_query_interrupt_freq = 1000
 
unsigned dynamic_watchdog_time_limit = 10000
 
std::string disk_cache_level = ""
 
bool enable_data_recycler = true
 
bool use_hashtable_cache = true
 
size_t hashtable_cache_total_bytes = 4294967296
 
size_t max_cacheable_hashtable_size_bytes = 2147483648
 
bool optimize_cuda_block_and_grid_sizes = false
 
size_t num_reader_threads = 0
 
std::string db_query_file = {""}
 
bool exit_after_warmup = false
 
int idle_session_duration = kMinsPerHour
 
int max_session_duration = kMinsPerMonth
 
std::string udf_file_name = {""}
 
std::string udf_compiler_path = {""}
 
std::vector< std::string > udf_compiler_options
 
std::string allowed_import_paths {}
 
std::string allowed_export_paths {}
 
std::string compressor = std::string(BLOSC_LZ4HC_COMPNAME)
 
po::options_description help_desc_
 
po::options_description developer_desc_
 
logger::LogOptions log_options_
 
std::string exe_name
 
po::positional_options_description positional_options
 
std::vector< LeafHostInfodb_leaves
 
std::vector< LeafHostInfostring_leaves
 
po::variables_map vm
 
std::string clusterIds_arg
 
const bool dist_v5_
 

Static Public Attributes

static const std::string nodeIds_token = {"node_id"}
 
static const std::string cluster_command_line_arg {"cluster_topology"}
 

Private Attributes

bool enable_runtime_udfs = true
 
bool enable_runtime_udf = true
 
bool enable_udf_registration_for_all_users = false
 

Detailed Description

Definition at line 40 of file CommandLineOptions.h.

Constructor & Destructor Documentation

CommandLineOptions::CommandLineOptions ( char const *  argv0,
bool  dist_v5_ = false 
)
inline

Definition at line 42 of file CommandLineOptions.h.

References fillDeveloperOptions(), and fillOptions().

43  : log_options_(argv0), exe_name(argv0), dist_v5_(dist_v5_) {
44  fillOptions();
46  }
logger::LogOptions log_options_

+ Here is the call graph for this function:

Member Function Documentation

void CommandLineOptions::fillDeveloperOptions ( )

Definition at line 726 of file CommandLineOptions.cpp.

References g_allow_invalid_literal_buffer_reads, g_allow_query_step_cpu_retry, g_approx_quantile_buffer, g_approx_quantile_centroids, g_bitmap_memory_limit, g_columnar_large_projections, g_columnar_large_projections_threshold, g_cpu_sub_task_size, g_cpu_threads_override, g_enable_auto_metadata_update, g_enable_automatic_ir_metadata, g_enable_bump_allocator, g_enable_columnar_output, g_enable_cpu_sub_tasks, g_enable_dev_table_functions, g_enable_direct_columnarization, g_enable_filter_function, g_enable_foreign_table_scheduled_refresh, g_enable_geo_ops_on_uncompressed_coords, g_enable_http_binary_server, g_enable_idp_temporary_users, g_enable_lazy_fetch, g_enable_left_join_filter_hoisting, g_enable_parallel_window_partition_compute, g_enable_parallel_window_partition_sort, g_enable_seconds_refresh, g_enable_smem_group_by, g_enable_smem_grouped_non_count_agg, g_enable_smem_non_grouped_agg, g_enable_window_functions, g_estimator_failure_max_groupby_size, g_fraction_code_cache_to_evict, g_gpu_smem_threshold, g_large_ndv_multiplier, g_large_ndv_threshold, g_max_log_length, g_max_memory_allocation_size, g_min_memory_allocation_size, g_num_tuple_threshold_switch_to_baseline, g_optimize_row_initialization, g_parallel_top_max, g_parallel_top_min, g_query_engine_cuda_streams, g_ratio_num_hash_entry_to_num_tuple_switch_to_baseline, g_skip_intermediate_count, g_streaming_topn_max, g_strip_join_covered_quals, g_use_table_device_offset, g_vacuum_min_selectivity, and g_window_function_aggregation_tree_fanout.

Referenced by CommandLineOptions().

726  {
727  po::options_description& desc = developer_desc_;
728 
729  desc.add_options()("dev-options", "Print internal developer options.");
730  desc.add_options()(
731  "enable-calcite-view-optimize",
734  ->implicit_value(true),
735  "Enable additional calcite (query plan) optimizations when a view is part of the "
736  "query.");
737  desc.add_options()("enable-columnar-output",
738  po::value<bool>(&g_enable_columnar_output)
739  ->default_value(g_enable_columnar_output)
740  ->implicit_value(true),
741  "Enable columnar output for intermediate/final query steps.");
742  desc.add_options()("enable-left-join-filter-hoisting",
743  po::value<bool>(&g_enable_left_join_filter_hoisting)
744  ->default_value(g_enable_left_join_filter_hoisting)
745  ->implicit_value(true),
746  "Enable hoisting left hand side filters through left joins.");
747  desc.add_options()("optimize-row-init",
748  po::value<bool>(&g_optimize_row_initialization)
749  ->default_value(g_optimize_row_initialization)
750  ->implicit_value(true),
751  "Optimize row initialization.");
752  desc.add_options()("enable-legacy-syntax",
753  po::value<bool>(&enable_legacy_syntax)
754  ->default_value(enable_legacy_syntax)
755  ->implicit_value(true),
756  "Enable legacy syntax.");
757  desc.add_options()(
758  "enable-multifrag",
759  po::value<bool>(&allow_multifrag)
760  ->default_value(allow_multifrag)
761  ->implicit_value(true),
762  "Enable execution over multiple fragments in a single round-trip to GPU.");
763  desc.add_options()("enable-lazy-fetch",
764  po::value<bool>(&g_enable_lazy_fetch)
765  ->default_value(g_enable_lazy_fetch)
766  ->implicit_value(true),
767  "Enable lazy fetch columns in query results.");
768  desc.add_options()("enable-shared-mem-group-by",
769  po::value<bool>(&g_enable_smem_group_by)
770  ->default_value(g_enable_smem_group_by)
771  ->implicit_value(true),
772  "Enable using GPU shared memory for some GROUP BY queries.");
773  desc.add_options()("num-executors",
774  po::value<int>(&system_parameters.num_executors)
775  ->default_value(system_parameters.num_executors),
776  "Number of executors to run in parallel.");
777  desc.add_options()(
778  "num-tuple-threshold-switch-to-baseline",
779  po::value<size_t>(&g_num_tuple_threshold_switch_to_baseline)
781  ->implicit_value(100000),
782  "Control a threshold to switch perfect hash join to baseline hash join by "
783  "comparing a hash entry range of the join column to the input table cardinality."
784  "This condition checks the following: |INPUT_TABLE| < {THIS_THRESHOLD}"
785  "We switch hash table layout when this condition and the condition related to "
786  "\'col-range-to-num-hash-entries-threshold-switch-to-baseline\' are satisfied "
787  "together.");
788  desc.add_options()(
789  "ratio-num-hash-entry-to-num-tuple-switch-to-baseline",
792  ->implicit_value(100),
793  "Control a threshold to switch perfect hash join to baseline hash join by "
794  "comparing a hash entry range of the join column to the input table cardinality."
795  "This condition checks the following: HASH_ENTRY_RANGE / |INPUT_TABLE| < "
796  "{THIS_THRESHOLD}"
797  "We switch hash table layout when this condition and the condition related to "
798  "\'num-tuple-threshold-switch-to-baseline\' are satisfied together.");
799  desc.add_options()(
800  "gpu-shared-mem-threshold",
801  po::value<size_t>(&g_gpu_smem_threshold)->default_value(g_gpu_smem_threshold),
802  "GPU shared memory threshold (in bytes). If query requires larger buffers than "
803  "this threshold, we disable those optimizations. 0 (default) means no static cap.");
804  desc.add_options()(
805  "enable-shared-mem-grouped-non-count-agg",
806  po::value<bool>(&g_enable_smem_grouped_non_count_agg)
807  ->default_value(g_enable_smem_grouped_non_count_agg)
808  ->implicit_value(true),
809  "Enable using GPU shared memory for grouped non-count aggregate queries.");
810  desc.add_options()("enable-shared-mem-non-grouped-agg",
811  po::value<bool>(&g_enable_smem_non_grouped_agg)
812  ->default_value(g_enable_smem_non_grouped_agg)
813  ->implicit_value(true),
814  "Enable using GPU shared memory for non-grouped aggregate queries.");
815  desc.add_options()("enable-direct-columnarization",
816  po::value<bool>(&g_enable_direct_columnarization)
817  ->default_value(g_enable_direct_columnarization)
818  ->implicit_value(true),
819  "Enables/disables a more optimized columnarization method "
820  "for intermediate steps in multi-step queries.");
821  desc.add_options()(
822  "offset-device-by-table-id",
823  po::value<bool>(&g_use_table_device_offset)
824  ->default_value(g_use_table_device_offset)
825  ->implicit_value(true),
826  "Enables/disables offseting the chosen device ID by the table ID for a given "
827  "fragment. This improves balance of fragments across GPUs.");
828  desc.add_options()("enable-window-functions",
829  po::value<bool>(&g_enable_window_functions)
830  ->default_value(g_enable_window_functions)
831  ->implicit_value(true),
832  "Enable window function support.");
833  desc.add_options()("enable-parallel-window-partition-compute",
836  ->implicit_value(true),
837  "Enable parallel window function partition computation.");
838  desc.add_options()("enable-parallel-window-partition-sort",
841  ->implicit_value(true),
842  "Enable parallel window function partition sorting.");
843  desc.add_options()(
844  "window-function-frame-aggregation-tree-fanout",
845  po::value<size_t>(&g_window_function_aggregation_tree_fanout)->default_value(8),
846  "A tree fanout for aggregation tree used to compute aggregation over "
847  "window frame");
848  desc.add_options()("enable-dev-table-functions",
849  po::value<bool>(&g_enable_dev_table_functions)
850  ->default_value(g_enable_dev_table_functions)
851  ->implicit_value(true),
852  "Enable dev (test or alpha) table functions. Also "
853  "requires --enable-table-functions to be turned on");
854 
855  desc.add_options()("enable-geo-ops-on-uncompressed-coords",
858  ->implicit_value(true),
859  "Enable faster geo operations on uncompressed coords");
860  desc.add_options()(
861  "jit-debug-ir",
862  po::value<bool>(&jit_debug)->default_value(jit_debug)->implicit_value(true),
863  "Enable runtime debugger support for the JIT. Note that this flag is "
864  "incompatible "
865  "with the `ENABLE_JIT_DEBUG` build flag. The generated code can be found at "
866  "`/tmp/mapdquery`.");
867  desc.add_options()(
868  "intel-jit-profile",
869  po::value<bool>(&intel_jit_profile)
870  ->default_value(intel_jit_profile)
871  ->implicit_value(true),
872  "Enable runtime support for the JIT code profiling using Intel VTune.");
873  desc.add_options()(
874  "enable-cpu-sub-tasks",
875  po::value<bool>(&g_enable_cpu_sub_tasks)
876  ->default_value(g_enable_cpu_sub_tasks)
877  ->implicit_value(true),
878  "Enable parallel processing of a single data fragment on CPU. This can improve CPU "
879  "load balance and decrease reduction overhead.");
880  desc.add_options()(
881  "cpu-sub-task-size",
882  po::value<size_t>(&g_cpu_sub_task_size)->default_value(g_cpu_sub_task_size),
883  "Set CPU sub-task size in rows.");
884  desc.add_options()(
885  "cpu-threads",
886  po::value<unsigned>(&g_cpu_threads_override)->default_value(g_cpu_threads_override),
887  "Set max CPU concurrent threads. Values <= 0 will use default of 2X the number of "
888  "hardware threads.");
889  desc.add_options()(
890  "skip-intermediate-count",
891  po::value<bool>(&g_skip_intermediate_count)
892  ->default_value(g_skip_intermediate_count)
893  ->implicit_value(true),
894  "Skip pre-flight counts for intermediate projections with no filters.");
895  desc.add_options()("strip-join-covered-quals",
896  po::value<bool>(&g_strip_join_covered_quals)
897  ->default_value(g_strip_join_covered_quals)
898  ->implicit_value(true),
899  "Remove quals from the filtered count if they are covered by a "
900  "join condition (currently only ST_Contains).");
901 
902  desc.add_options()("min-cpu-slab-size",
903  po::value<size_t>(&system_parameters.min_cpu_slab_size)
904  ->default_value(system_parameters.min_cpu_slab_size),
905  "Min slab size (size of memory allocations) for CPU buffer pool.");
906  desc.add_options()(
907  "max-cpu-slab-size",
908  po::value<size_t>(&system_parameters.max_cpu_slab_size)
909  ->default_value(system_parameters.max_cpu_slab_size),
910  "Max CPU buffer pool slab size (size of memory allocations). Note if "
911  "there is not enough free memory to accomodate the target slab size, smaller "
912  "slabs will be allocated, down to the minimum size specified by "
913  "min-cpu-slab-size.");
914  desc.add_options()("min-gpu-slab-size",
915  po::value<size_t>(&system_parameters.min_gpu_slab_size)
916  ->default_value(system_parameters.min_gpu_slab_size),
917  "Min slab size (size of memory allocations) for GPU buffer pools.");
918  desc.add_options()(
919  "max-gpu-slab-size",
920  po::value<size_t>(&system_parameters.max_gpu_slab_size)
921  ->default_value(system_parameters.max_gpu_slab_size),
922  "Max GPU buffer pool slab size (size of memory allocations). Note if "
923  "there is not enough free memory to accomodate the target slab size, smaller "
924  "slabs will be allocated, down to the minimum size speified by "
925  "min-gpu-slab-size.");
926 
927  desc.add_options()(
928  "max-output-projection-allocation-bytes",
929  po::value<size_t>(&g_max_memory_allocation_size)
930  ->default_value(g_max_memory_allocation_size),
931  "Maximum allocation size for a fixed output buffer allocation for projection "
932  "queries with no pre-flight count. Default is the maximum slab size (sizes "
933  "greater "
934  "than the maximum slab size have no affect). Requires bump allocator.");
935  desc.add_options()(
936  "min-output-projection-allocation-bytes",
937  po::value<size_t>(&g_min_memory_allocation_size)
938  ->default_value(g_min_memory_allocation_size),
939  "Minimum allocation size for a fixed output buffer allocation for projection "
940  "queries with no pre-flight count. If an allocation of this size cannot be "
941  "obtained, the query will be retried with different execution parameters and/or "
942  "on "
943  "CPU (if allow-cpu-retry is enabled). Requires bump allocator.");
944  desc.add_options()("enable-bump-allocator",
945  po::value<bool>(&g_enable_bump_allocator)
946  ->default_value(g_enable_bump_allocator)
947  ->implicit_value(true),
948  "Enable the bump allocator for projection queries on "
949  "GPU. The bump allocator will "
950  "allocate a fixed size buffer for each query, track the "
951  "number of rows passing the "
952  "kernel during query execution, and copy back only the "
953  "rows that passed the kernel "
954  "to CPU after execution. When disabled, pre-flight "
955  "count queries are used to size "
956  "the output buffer for projection queries.");
957  desc.add_options()(
958  "code-cache-eviction-percent",
959  po::value<float>(&g_fraction_code_cache_to_evict)
960  ->default_value(g_fraction_code_cache_to_evict),
961  "Percentage of the GPU code cache to evict if an out of memory error is "
962  "encountered while attempting to place generated code on the GPU.");
963 
964  desc.add_options()("ssl-cert",
965  po::value<std::string>(&system_parameters.ssl_cert_file)
966  ->default_value(std::string("")),
967  "SSL Validated public certficate.");
968 
969  desc.add_options()("ssl-private-key",
970  po::value<std::string>(&system_parameters.ssl_key_file)
971  ->default_value(std::string("")),
972  "SSL private key file.");
973  // Note ssl_trust_store is passed through to Calcite via system_parameters
974  // todo(jack): add ensure ssl-trust-store exists if cert and private key in use
975  desc.add_options()("ssl-trust-store",
976  po::value<std::string>(&system_parameters.ssl_trust_store)
977  ->default_value(std::string("")),
978  "SSL public CA certifcates (java trust store) to validate "
979  "TLS connections (passed through to the Calcite server).");
980 
981  desc.add_options()(
982  "ssl-trust-password",
983  po::value<std::string>(&system_parameters.ssl_trust_password)
984  ->default_value(std::string("")),
985  "SSL password for java trust store provided via --ssl-trust-store parameter.");
986 
987  desc.add_options()(
988  "ssl-trust-ca",
989  po::value<std::string>(&system_parameters.ssl_trust_ca_file)
990  ->default_value(std::string("")),
991  "SSL public CA certificates to validate TLS connection(as a client).");
992 
993  desc.add_options()(
994  "ssl-trust-ca-server",
995  po::value<std::string>(&authMetadata.ca_file_name)->default_value(std::string("")),
996  "SSL public CA certificates to validate TLS connection(as a server).");
997 
998  desc.add_options()("ssl-keystore",
999  po::value<std::string>(&system_parameters.ssl_keystore)
1000  ->default_value(std::string("")),
1001  "SSL server credentials as a java key store (passed "
1002  "through to the Calcite server).");
1003 
1004  desc.add_options()("ssl-keystore-password",
1005  po::value<std::string>(&system_parameters.ssl_keystore_password)
1006  ->default_value(std::string("")),
1007  "SSL password for java keystore, provide by via --ssl-keystore.");
1008 
1009  desc.add_options()(
1010  "udf",
1011  po::value<std::string>(&udf_file_name),
1012  "Load user defined extension functions from this file at startup. The file is "
1013  "expected to be a C/C++ file with extension .cpp.");
1014 
1015  desc.add_options()("udf-compiler-path",
1016  po::value<std::string>(&udf_compiler_path),
1017  "Provide absolute path to clang++ used in udf compilation.");
1018 
1019  desc.add_options()("udf-compiler-options",
1020  po::value<std::vector<std::string>>(&udf_compiler_options),
1021  "Specify compiler options to tailor udf compilation.");
1022 
1023 #ifdef ENABLE_GEOS
1024  desc.add_options()("libgeos-so-filename",
1025  po::value<std::string>(&libgeos_so_filename),
1026  "Specify libgeos shared object filename to be used for "
1027  "geos-backed geo opertations.");
1028 #endif
1029  desc.add_options()(
1030  "large-ndv-threshold",
1031  po::value<int64_t>(&g_large_ndv_threshold)->default_value(g_large_ndv_threshold));
1032  desc.add_options()(
1033  "large-ndv-multiplier",
1034  po::value<size_t>(&g_large_ndv_multiplier)->default_value(g_large_ndv_multiplier));
1035  desc.add_options()("approx_quantile_buffer",
1036  po::value<size_t>(&g_approx_quantile_buffer)
1037  ->default_value(g_approx_quantile_buffer));
1038  desc.add_options()("approx_quantile_centroids",
1039  po::value<size_t>(&g_approx_quantile_centroids)
1040  ->default_value(g_approx_quantile_centroids));
1041  desc.add_options()(
1042  "bitmap-memory-limit",
1043  po::value<int64_t>(&g_bitmap_memory_limit)->default_value(g_bitmap_memory_limit),
1044  "Limit for count distinct bitmap memory use. The limit is computed by taking the "
1045  "size of the group by buffer (entry count in Query Memory Descriptor) and "
1046  "multiplying it by the number of count distinct expression and the size of bitmap "
1047  "required for each. For approx_count_distinct this is typically 8192 bytes.");
1048  desc.add_options()(
1049  "enable-filter-function",
1050  po::value<bool>(&g_enable_filter_function)
1051  ->default_value(g_enable_filter_function)
1052  ->implicit_value(true),
1053  "Enable the filter function protection feature for the SQL JIT compiler. "
1054  "Normally should be on but techs might want to disable for troubleshooting.");
1055  desc.add_options()(
1056  "enable-idp-temporary-users",
1057  po::value<bool>(&g_enable_idp_temporary_users)
1058  ->default_value(g_enable_idp_temporary_users)
1059  ->implicit_value(true),
1060  "Enable temporary users for SAML and LDAP logins on read-only servers. "
1061  "Normally should be on but techs might want to disable for troubleshooting.");
1062  desc.add_options()("enable-foreign-table-scheduled-refresh",
1065  ->implicit_value(true),
1066  "Enable scheduled foreign table refresh.");
1067  desc.add_options()(
1068  "enable-seconds-refresh-interval",
1069  po::value<bool>(&g_enable_seconds_refresh)
1070  ->default_value(g_enable_seconds_refresh)
1071  ->implicit_value(true),
1072  "Enable foreign table seconds refresh interval for testing purposes.");
1073  desc.add_options()("enable-auto-metadata-update",
1074  po::value<bool>(&g_enable_auto_metadata_update)
1075  ->default_value(g_enable_auto_metadata_update)
1076  ->implicit_value(true),
1077  "Enable automatic metadata update.");
1078  desc.add_options()(
1079  "parallel-top-min",
1080  po::value<size_t>(&g_parallel_top_min)->default_value(g_parallel_top_min),
1081  "For ResultSets requiring a heap sort, the number of rows necessary to trigger "
1082  "parallelTop() to sort.");
1083  desc.add_options()(
1084  "parallel-top-max",
1085  po::value<size_t>(&g_parallel_top_max)->default_value(g_parallel_top_max),
1086  "For ResultSets requiring a heap sort, the maximum number of rows allowed by "
1087  "watchdog.");
1088  desc.add_options()(
1089  "streaming-top-n-max",
1090  po::value<size_t>(&g_streaming_topn_max)->default_value(g_streaming_topn_max),
1091  "The maximum number of rows allowing streaming top-N sorting.");
1092  desc.add_options()("vacuum-min-selectivity",
1093  po::value<float>(&g_vacuum_min_selectivity)
1094  ->default_value(g_vacuum_min_selectivity),
1095  "Minimum selectivity for automatic vacuuming. "
1096  "This specifies the percentage (with a value of 0 "
1097  "implying 0% and a value of 1 implying 100%) of "
1098  "deleted rows in a fragment at which to perform "
1099  "automatic vacuuming. A number greater than 1 can "
1100  "be used to disable automatic vacuuming.");
1101  desc.add_options()("enable-automatic-ir-metadata",
1102  po::value<bool>(&g_enable_automatic_ir_metadata)
1103  ->default_value(g_enable_automatic_ir_metadata)
1104  ->implicit_value(true),
1105  "Enable automatic IR metadata (debug builds only).");
1106  desc.add_options()(
1107  "max-log-length",
1108  po::value<size_t>(&g_max_log_length)->default_value(g_max_log_length),
1109  "The maximum number of characters that a log message can has. If the log message "
1110  "is longer than this, we only record \'g_max_log_message_length\' characters.");
1111  desc.add_options()(
1112  "estimator-failure-max-groupby-size",
1113  po::value<size_t>(&g_estimator_failure_max_groupby_size)
1114  ->default_value(g_estimator_failure_max_groupby_size),
1115  "Maximum size of the groupby buffer if the estimator fails. By default we use the "
1116  "number of tuples in the table up to this value.");
1117  desc.add_options()("columnar-large-projections",
1118  po::value<bool>(&g_columnar_large_projections)
1119  ->default_value(g_columnar_large_projections)
1120  ->implicit_value(true),
1121  "Prefer columnar output if projection size is >= "
1122  "threshold set by --columnar-large-projections-threshold "
1123  "(default 1,000,000 rows).");
1124  desc.add_options()(
1125  "columnar-large-projections-threshold",
1126  po::value<size_t>(&g_columnar_large_projections_threshold)
1127  ->default_value(g_columnar_large_projections_threshold),
1128  "Threshold (in minimum number of rows) to prefer columnar output for projections. "
1129  "Requires --columnar-large-projections to be set.");
1130 
1131  desc.add_options()(
1132  "allow-query-step-cpu-retry",
1133  po::value<bool>(&g_allow_query_step_cpu_retry)
1134  ->default_value(g_allow_query_step_cpu_retry)
1135  ->implicit_value(true),
1136  R"(Allow certain query steps to retry on CPU, even when allow-cpu-retry is disabled)");
1137  desc.add_options()("enable-http-binary-server",
1138  po::value<bool>(&g_enable_http_binary_server)
1139  ->default_value(g_enable_http_binary_server)
1140  ->implicit_value(true),
1141  "Enable binary over HTTP Thrift server");
1142 
1143  desc.add_options()("enable-query-engine-cuda-streams",
1144  po::value<bool>(&g_query_engine_cuda_streams)
1145  ->default_value(g_query_engine_cuda_streams)
1146  ->implicit_value(true),
1147  "Enable Query Engine CUDA streams");
1148 
1149  desc.add_options()(
1150  "allow-invalid-literal-buffer-reads",
1151  po::value<bool>(&g_allow_invalid_literal_buffer_reads)
1152  ->default_value(g_allow_invalid_literal_buffer_reads)
1153  ->implicit_value(true),
1154  "For backwards compatibility. Enabling may cause invalid query results.");
1155 
1156 #ifdef HAVE_TORCH_TFS
1157  desc.add_options()("torch-lib-path",
1158  po::value<std::string>(&torch_lib_path),
1159  "Absolute path to custom LibTorch shared library location to be "
1160  "loaded at runtime. (If not provided, the library will be searched "
1161  "for in the system's default library path.)");
1162 #endif
1163 }
int64_t g_large_ndv_threshold
bool g_use_table_device_offset
bool g_enable_parallel_window_partition_sort
bool g_enable_left_join_filter_hoisting
Definition: Execute.cpp:101
bool g_enable_smem_group_by
size_t g_parallel_top_max
Definition: ResultSet.cpp:50
size_t g_num_tuple_threshold_switch_to_baseline
Definition: Execute.cpp:104
size_t g_cpu_sub_task_size
Definition: Execute.cpp:84
bool g_strip_join_covered_quals
Definition: Execute.cpp:110
bool g_enable_direct_columnarization
Definition: Execute.cpp:127
bool g_enable_lazy_fetch
Definition: Execute.cpp:129
std::string udf_compiler_path
bool g_skip_intermediate_count
bool enable_calcite_view_optimize
unsigned g_cpu_threads_override
bool g_enable_auto_metadata_update
size_t g_streaming_topn_max
Definition: ResultSet.cpp:51
bool g_enable_geo_ops_on_uncompressed_coords
Definition: Execute.cpp:119
bool g_enable_smem_non_grouped_agg
Definition: Execute.cpp:143
size_t g_parallel_top_min
Definition: ResultSet.cpp:49
std::string ssl_trust_ca_file
bool g_enable_columnar_output
Definition: Execute.cpp:100
size_t g_ratio_num_hash_entry_to_num_tuple_switch_to_baseline
Definition: Execute.cpp:105
std::string ssl_trust_store
bool g_enable_idp_temporary_users
Definition: SysCatalog.cpp:63
size_t g_window_function_aggregation_tree_fanout
bool g_enable_http_binary_server
int64_t g_bitmap_memory_limit
size_t g_max_memory_allocation_size
Definition: Execute.cpp:121
size_t g_approx_quantile_buffer
Definition: Execute.cpp:164
size_t g_max_log_length
Definition: Execute.cpp:169
bool g_enable_dev_table_functions
Definition: Execute.cpp:118
std::string ca_file_name
Definition: AuthMetadata.h:31
std::string ssl_key_file
AuthMetadata authMetadata
bool g_enable_window_functions
Definition: Execute.cpp:114
size_t g_min_memory_allocation_size
Definition: Execute.cpp:122
bool g_enable_seconds_refresh
size_t g_estimator_failure_max_groupby_size
bool g_enable_smem_grouped_non_count_agg
Definition: Execute.cpp:140
bool g_enable_automatic_ir_metadata
Definition: Execute.cpp:167
std::vector< std::string > udf_compiler_options
bool g_enable_foreign_table_scheduled_refresh
float g_vacuum_min_selectivity
bool g_enable_filter_function
Definition: Execute.cpp:85
float g_fraction_code_cache_to_evict
bool g_allow_invalid_literal_buffer_reads
Definition: ConstantIR.cpp:140
std::string ssl_keystore_password
std::string ssl_trust_password
bool g_enable_bump_allocator
Definition: Execute.cpp:125
bool g_enable_parallel_window_partition_compute
bool g_enable_cpu_sub_tasks
Definition: Execute.cpp:83
std::string ssl_keystore
bool g_allow_query_step_cpu_retry
Definition: Execute.cpp:88
size_t g_approx_quantile_centroids
Definition: Execute.cpp:165
bool g_optimize_row_initialization
Definition: Execute.cpp:102
bool g_columnar_large_projections
size_t g_columnar_large_projections_threshold
bool g_query_engine_cuda_streams
Definition: QueryEngine.h:9
po::options_description developer_desc_
size_t g_large_ndv_multiplier
SystemParameters system_parameters
size_t g_gpu_smem_threshold
Definition: Execute.cpp:135
std::string ssl_cert_file

+ Here is the caller graph for this function:

void CommandLineOptions::fillOptions ( )

Definition at line 96 of file CommandLineOptions.cpp.

References g_allow_auto_resultset_caching, g_allow_cpu_retry, g_allow_query_step_skipping, g_auto_resultset_caching_threshold, g_bbox_intersect_max_table_size_bytes, g_bbox_intersect_target_entries_per_bin, g_bigint_count, g_cache_string_hash, g_enable_add_metadata_columns, g_enable_bbox_intersect_hashjoin, g_enable_debug_timer, g_enable_distance_rangejoin, g_enable_executor_resource_mgr, g_enable_filter_push_down, g_enable_fsi, g_enable_fsi_regex_import, g_enable_hashjoin_many_to_many, g_enable_interop, g_enable_legacy_delimited_import, g_enable_logs_system_tables, g_enable_ml_functions, g_enable_string_functions, g_enable_stringdict_parallel, g_enable_system_tables, g_enable_table_functions, g_enable_thrift_logs, g_enable_union, g_executor_resource_mgr_allow_cpu_gpu_kernel_concurrency, g_executor_resource_mgr_allow_cpu_kernel_concurrency, g_executor_resource_mgr_allow_cpu_result_mem_oversubscription_concurrency, g_executor_resource_mgr_allow_cpu_slot_oversubscription_concurrency, g_executor_resource_mgr_cpu_result_mem_bytes, g_executor_resource_mgr_cpu_result_mem_ratio, g_executor_resource_mgr_max_available_resource_use_ratio, g_executor_resource_mgr_per_query_max_cpu_result_mem_ratio, g_executor_resource_mgr_per_query_max_cpu_slots_ratio, g_filter_push_down_high_frac, g_filter_push_down_low_frac, g_filter_push_down_passing_row_ubound, g_from_table_reordering, g_hll_precision_bits, g_inner_join_fragment_skipping, Catalog_Namespace::g_log_user_id, g_logs_system_tables_max_files_count, g_max_cacheable_query_resultset_size_bytes, g_max_import_threads, g_null_div_by_zero, g_pmem_path, g_pmem_size, g_query_resultset_cache_total_bytes, g_restrict_ml_model_metadata_to_superusers, g_trivial_loop_join_threshold, g_uniform_request_ids_per_thrift_call, g_use_chunk_metadata_cache, g_use_query_resultset_cache, and run_benchmark_import::required.

Referenced by CommandLineOptions().

96  {
97  po::options_description& desc = help_desc_;
98 
99  desc.add_options()("help,h", "Show available options.");
100  desc.add_options()(
101  "allow-cpu-retry",
102  po::value<bool>(&g_allow_cpu_retry)
103  ->default_value(g_allow_cpu_retry)
104  ->implicit_value(true),
105  R"(Allow the queries which failed on GPU to retry on CPU, even when watchdog is enabled.)");
106  desc.add_options()("allow-loop-joins",
107  po::value<bool>(&allow_loop_joins)
108  ->default_value(allow_loop_joins)
109  ->implicit_value(true),
110  "Enable loop joins.");
111  desc.add_options()("bigint-count",
112  po::value<bool>(&g_bigint_count)
113  ->default_value(g_bigint_count)
114  ->implicit_value(true),
115  "Use 64-bit count.");
116 
117  desc.add_options()(
118  "enable-executor-resource-mgr",
119  po::value<bool>(&g_enable_executor_resource_mgr)
120  ->default_value(g_enable_executor_resource_mgr)
121  ->implicit_value(true),
122  "Enable executor resource manager to track execution resources and selectively "
123  "gate concurrency based on resource availability.");
124 
125  // Note we allow executor-cpu-result-mem-ratio to have values > 0 to allow
126  // oversubscription of memory when warranted, but user should be careful with this as
127  // too high a value can cause OOM errors.
128  desc.add_options()(
129  "executor-cpu-result-mem-ratio",
132  "Set executor resource manager reserved memory for query result sets as a ratio "
133  "greater than 0, representing the fraction of the system memory not allocated for "
134  "the CPU buffer pool. Values of 1.0 are permitted to allow oversubscription when "
135  "warranted, but too high a value can cause out-of-memory errors. Requires "
136  "--executor-resource-mgr to be set");
137 
138  desc.add_options()(
139  "executor-cpu-result-mem-bytes",
142  "Set executor resource manager reserved memory for query result sets in bytes, "
143  "this overrides the default reservation of 80% the size of the system memory that "
144  "is not allocated for the CPU buffer pool. Use 0 for auto. Requires "
145  "--enable-executor-resource-mgr to be set.");
146 
147  // Note we allow executor-per-query-max-cpu-threads-ratio to have values > 1 to allow
148  // oversubscription of threads when warranted, given we may be overly pessimistic about
149  // kernel core occupation for some classes of queries. Care should be taken however with
150  // setting this value too high as thrashing and thread starvation can result.
151  desc.add_options()(
152  "executor-per-query-max-cpu-threads-ratio",
155  "Set max fraction of executor resource manager total CPU slots/threads that can be "
156  "allocated for a single query. Requires --enable-executor-resource-mgr to be set.");
157 
158  // Note we allow executor-per-query-max-cpu-result-mem-ratio to have values > 0 to allow
159  // oversubscription of memory when warranted, but user should be careful with this as
160  // too high a value can cause OOM errors.
161  desc.add_options()(
162  "executor-per-query-max-cpu-result-mem-ratio",
165  "Set max fraction of executor resource manager total CPU result memory reservation "
166  "that can be "
167  "allocated for a single query. Requires --enable-executor-resource-mgr to be set.");
168 
169  desc.add_options()(
170  "allow-cpu-kernel-concurrency",
173  ->implicit_value(true),
174  "Allow for multiple queries to run execution kernels concurrently on CPU. Requires "
175  "--enable-executor-resource-mgr to be set.");
176 
177  desc.add_options()(
178  "allow-cpu-gpu-kernel-concurrency",
181  ->implicit_value(true),
182  "Allow multiple queries to run execution kernels concurrently on CPU while a "
183  "GPU query is executing. Requires --enable-executor-resource-mgr to be set.");
184 
185  // Below controls whether multiple concurrent queries in conjunction can oversubscribe
186  // CPU slots/threads Single query CPU slot oversubscription should be controlled with
187  // --executor-per-query-max-cpu-threads-ratio (i.e. by setting it to > 1.0)
188 
189  desc.add_options()(
190  "allow-cpu-thread-oversubscription-concurrency",
191  po::value<bool>(
193  ->default_value(
195  ->implicit_value(true),
196  "Allow for concurrent query kernel execution even if it results in "
197  "oversubscription of CPU threads. Caution should be used when turning this on as "
198  "it can lead to thread exhaustion. Requires --enable-executor-resource-mgr to be "
199  "set.");
200 
201  // Below controls whether multiple concurrent queries in conjunction can oversubscribe
202  // CPU result memory. Single query CPU result memory oversubscription should be
203  // controlled with
204  // --executor-per-query-cpu-result-mem-ratio (i.e. by setting it to > 1.0)
205 
206  desc.add_options()(
207  "allow-cpu-result-mem-oversubscription-concurrency",
208  po::value<bool>(
210  ->default_value(
212  ->implicit_value(true),
213  "Allow for concurrent query kernel execution even if it results in "
214  "oversubscription of CPU memory. Caution should be used when turning this on as it "
215  "can lead to out-of-memory errors. Requires --enable-executor-resource-mgr to be "
216  "set.");
217 
218  desc.add_options()(
219  "executor-max-available-resource-use-ratio",
222  "Set max proportion (0 < ratio <= 1.0) of available resources that should be "
223  "granted to a query. Requires --executor-resource-mgr to be set");
224 
225  desc.add_options()("calcite-max-mem",
226  po::value<size_t>(&system_parameters.calcite_max_mem)
227  ->default_value(system_parameters.calcite_max_mem),
228  "Max memory available to calcite JVM.");
229  if (!dist_v5_) {
230  desc.add_options()("calcite-port",
231  po::value<int>(&system_parameters.calcite_port)
232  ->default_value(system_parameters.calcite_port),
233  "Calcite port number.");
234  }
235  desc.add_options()("config",
236  po::value<std::string>(&system_parameters.config_file),
237  "Path to server configuration file.");
238  desc.add_options()("cpu-buffer-mem-bytes",
239  po::value<size_t>(&system_parameters.cpu_buffer_mem_bytes)
240  ->default_value(system_parameters.cpu_buffer_mem_bytes),
241  "Size of memory reserved for CPU buffers, in bytes.");
242 
243  desc.add_options()("cpu-only",
244  po::value<bool>(&system_parameters.cpu_only)
245  ->default_value(system_parameters.cpu_only)
246  ->implicit_value(true),
247  "Run on CPU only, even if GPUs are available.");
248  desc.add_options()("cuda-block-size",
249  po::value<size_t>(&system_parameters.cuda_block_size)
250  ->default_value(system_parameters.cuda_block_size),
251  "Size of block to use on NVIDIA GPU.");
252  desc.add_options()("cuda-grid-size",
253  po::value<size_t>(&system_parameters.cuda_grid_size)
254  ->default_value(system_parameters.cuda_grid_size),
255  "Size of grid to use on NVIDIA GPU.");
256  desc.add_options()("optimize-cuda-block-and-grid-sizes",
257  po::value<bool>(&optimize_cuda_block_and_grid_sizes)
258  ->default_value(false)
259  ->implicit_value(true));
260 
261  if (!dist_v5_) {
262  desc.add_options()(
263  "data",
264  po::value<std::string>(&base_path)->required()->default_value("storage"),
265  "Directory path to HeavyDB data storage (catalogs, raw data, log files, etc).");
266  positional_options.add("data", 1);
267  }
268  desc.add_options()("db-query-list",
269  po::value<std::string>(&db_query_file),
270  "Path to file containing HeavyDB warmup queries.");
271  desc.add_options()(
272  "exit-after-warmup",
273  po::value<bool>(&exit_after_warmup)->default_value(false)->implicit_value(true),
274  "Exit after HeavyDB warmup queries.");
275  desc.add_options()("dynamic-watchdog-time-limit",
276  po::value<unsigned>(&dynamic_watchdog_time_limit)
277  ->default_value(dynamic_watchdog_time_limit)
278  ->implicit_value(10000),
279  "Dynamic watchdog time limit, in milliseconds.");
280  desc.add_options()("enable-data-recycler",
281  po::value<bool>(&enable_data_recycler)
282  ->default_value(enable_data_recycler)
283  ->implicit_value(true),
284  "Use data recycler.");
285  desc.add_options()("use-hashtable-cache",
286  po::value<bool>(&use_hashtable_cache)
287  ->default_value(use_hashtable_cache)
288  ->implicit_value(true),
289  "Use hashtable cache.");
290  desc.add_options()("use-query-resultset-cache",
291  po::value<bool>(&g_use_query_resultset_cache)
292  ->default_value(g_use_query_resultset_cache)
293  ->implicit_value(true),
294  "Use query resultset cache.");
295  desc.add_options()("use-chunk-metadata-cache",
296  po::value<bool>(&g_use_chunk_metadata_cache)
297  ->default_value(g_use_chunk_metadata_cache)
298  ->implicit_value(true),
299  "Use chunk metadata cache.");
300  desc.add_options()(
301  "hashtable-cache-total-bytes",
302  po::value<size_t>(&hashtable_cache_total_bytes)
303  ->default_value(hashtable_cache_total_bytes)
304  ->implicit_value(4294967296),
305  "Size of total memory space for hashtable cache, in bytes (default: 4GB).");
306  desc.add_options()("max-cacheable-hashtable-size-bytes",
307  po::value<size_t>(&max_cacheable_hashtable_size_bytes)
308  ->default_value(max_cacheable_hashtable_size_bytes)
309  ->implicit_value(2147483648),
310  "The maximum size of hashtable that is available to cache, in "
311  "bytes (default: 2GB).");
312  desc.add_options()(
313  "query-resultset-cache-total-bytes",
314  po::value<size_t>(&g_query_resultset_cache_total_bytes)
315  ->default_value(g_query_resultset_cache_total_bytes),
316  "Size of total memory space for query resultset cache, in bytes (default: 4GB).");
317  desc.add_options()("max-query-resultset-size-bytes",
320  "The maximum size of query resultset that is available to cache, in "
321  "bytes (default: 2GB).");
322  desc.add_options()("allow-auto-query-resultset-caching",
323  po::value<bool>(&g_allow_auto_resultset_caching)
324  ->default_value(g_allow_auto_resultset_caching)
325  ->implicit_value(true),
326  "Allow automatic query resultset caching when the size of "
327  "query resultset is smaller or equal to the threshold defined "
328  "by `auto-resultset-caching-threshold-bytes`, in bytes (to "
329  "enable this, query resultset recycler "
330  "should be enabled, default: 1048576 bytes (or 1MB)).");
331  desc.add_options()(
332  "auto-resultset-caching-threshold-bytes",
333  po::value<size_t>(&g_auto_resultset_caching_threshold)
334  ->default_value(g_auto_resultset_caching_threshold),
335  "A threshold that allows caching query resultset automatically if the size of "
336  "resultset is less than it, in bytes (default: 1MB).");
337  desc.add_options()("allow-query-step-skipping",
338  po::value<bool>(&g_allow_query_step_skipping)
339  ->default_value(g_allow_query_step_skipping)
340  ->implicit_value(true),
341  "Allow query step skipping when multi-step query has at least "
342  "one cached query resultset.");
343  desc.add_options()("enable-debug-timer",
344  po::value<bool>(&g_enable_debug_timer)
345  ->default_value(g_enable_debug_timer)
346  ->implicit_value(true),
347  "Enable debug timer logging.");
348  desc.add_options()("enable-dynamic-watchdog",
349  po::value<bool>(&enable_dynamic_watchdog)
350  ->default_value(enable_dynamic_watchdog)
351  ->implicit_value(true),
352  "Enable dynamic watchdog.");
353  desc.add_options()("enable-filter-push-down",
354  po::value<bool>(&g_enable_filter_push_down)
355  ->default_value(g_enable_filter_push_down)
356  ->implicit_value(true),
357  "Enable filter push down through joins.");
358  desc.add_options()(
359  "enable-bbox-intersect-hashjoin",
360  po::value<bool>(&g_enable_bbox_intersect_hashjoin)
361  ->default_value(g_enable_bbox_intersect_hashjoin)
362  ->implicit_value(true),
363  "Enable the bounding box intersect hash join framework to enable post-filtering of "
364  "pairs of geometries before actually comptuing geometry function.");
365  desc.add_options()("enable-hashjoin-many-to-many",
366  po::value<bool>(&g_enable_hashjoin_many_to_many)
367  ->default_value(g_enable_hashjoin_many_to_many)
368  ->implicit_value(true),
369  "Enable the bounding box intersect hash join framework to more "
370  "spatial join operators for pairs of geometry types corresponding "
371  "to many-to-many relationship.");
372  desc.add_options()("enable-distance-rangejoin",
373  po::value<bool>(&g_enable_distance_rangejoin)
374  ->default_value(g_enable_distance_rangejoin)
375  ->implicit_value(true),
376  "Enable accelerating point distance joins with a hash table. "
377  "This rewrites ST_Distance when using an upperbound (<= X).");
378  desc.add_options()("enable-runtime-query-interrupt",
379  po::value<bool>(&enable_runtime_query_interrupt)
380  ->default_value(enable_runtime_query_interrupt)
381  ->implicit_value(true),
382  "Enable runtime query interrupt.");
383  desc.add_options()("enable-non-kernel-time-query-interrupt",
386  ->implicit_value(true),
387  "Enable non-kernel time query interrupt.");
388  desc.add_options()("pending-query-interrupt-freq",
389  po::value<unsigned>(&pending_query_interrupt_freq)
390  ->default_value(pending_query_interrupt_freq)
391  ->implicit_value(1000),
392  "A frequency of checking the request of pending query "
393  "interrupt from user (in millisecond).");
394  desc.add_options()("running-query-interrupt-freq",
395  po::value<double>(&running_query_interrupt_freq)
396  ->default_value(running_query_interrupt_freq)
397  ->implicit_value(0.5),
398  "A frequency of checking the request of running query "
399  "interrupt from user (0.0 (less frequent) ~ (more frequent) 1.0).");
400  desc.add_options()("use-estimator-result-cache",
401  po::value<bool>(&use_estimator_result_cache)
402  ->default_value(use_estimator_result_cache)
403  ->implicit_value(true),
404  "Use estimator result cache.");
405  if (!dist_v5_) {
406  desc.add_options()(
407  "enable-string-dict-hash-cache",
408  po::value<bool>(&g_cache_string_hash)
409  ->default_value(g_cache_string_hash)
410  ->implicit_value(true),
411  "Cache string hash values in the string dictionary server during import.");
412  }
413  desc.add_options()("enable-thrift-logs",
414  po::value<bool>(&g_enable_thrift_logs)
415  ->default_value(g_enable_thrift_logs)
416  ->implicit_value(true),
417  "Enable writing messages directly from thrift to stdout/stderr.");
418  desc.add_options()("enable-watchdog",
419  po::value<bool>(&enable_watchdog)
420  ->default_value(enable_watchdog)
421  ->implicit_value(true),
422  "Enable watchdog.");
423  desc.add_options()("watchdog-none-encoded-string-translation-limit",
426  "Max number of none-encoded strings allowed to be translated "
427  "to dictionary-encoded with watchdog enabled");
428  desc.add_options()("filter-push-down-low-frac",
429  po::value<float>(&g_filter_push_down_low_frac)
430  ->default_value(g_filter_push_down_low_frac)
431  ->implicit_value(g_filter_push_down_low_frac),
432  "Lower threshold for selectivity of filters that are pushed down.");
433  desc.add_options()("filter-push-down-high-frac",
434  po::value<float>(&g_filter_push_down_high_frac)
435  ->default_value(g_filter_push_down_high_frac)
436  ->implicit_value(g_filter_push_down_high_frac),
437  "Higher threshold for selectivity of filters that are pushed down.");
438  desc.add_options()("filter-push-down-passing-row-ubound",
439  po::value<size_t>(&g_filter_push_down_passing_row_ubound)
441  ->implicit_value(g_filter_push_down_passing_row_ubound),
442  "Upperbound on the number of rows that should pass the filter "
443  "if the selectivity is less than "
444  "the high fraction threshold.");
445  desc.add_options()("from-table-reordering",
446  po::value<bool>(&g_from_table_reordering)
447  ->default_value(g_from_table_reordering)
448  ->implicit_value(true),
449  "Enable automatic table reordering in FROM clause.");
450  desc.add_options()("gpu-buffer-mem-bytes",
451  po::value<size_t>(&system_parameters.gpu_buffer_mem_bytes)
452  ->default_value(system_parameters.gpu_buffer_mem_bytes),
453  "Size of memory reserved for GPU buffers, in bytes, per GPU.");
454  desc.add_options()("gpu-input-mem-limit",
455  po::value<double>(&system_parameters.gpu_input_mem_limit)
456  ->default_value(system_parameters.gpu_input_mem_limit),
457  "Force query to CPU when input data memory usage exceeds this "
458  "percentage of available GPU memory.");
459  desc.add_options()(
460  "hll-precision-bits",
461  po::value<int>(&g_hll_precision_bits)
462  ->default_value(g_hll_precision_bits)
463  ->implicit_value(g_hll_precision_bits),
464  "Number of bits used from the hash value used to specify the bucket number.");
465  if (!dist_v5_) {
466  desc.add_options()("http-port",
467  po::value<int>(&http_port)->default_value(http_port),
468  "HTTP port number.");
469  desc.add_options()("http-binary-port",
470  po::value<int>(&http_binary_port)->default_value(http_binary_port),
471  "HTTP binary port number.");
472  }
473  desc.add_options()(
474  "idle-session-duration",
475  po::value<int>(&idle_session_duration)->default_value(idle_session_duration),
476  "Maximum duration of idle session.");
477  desc.add_options()("inner-join-fragment-skipping",
478  po::value<bool>(&g_inner_join_fragment_skipping)
479  ->default_value(g_inner_join_fragment_skipping)
480  ->implicit_value(true),
481  "Enable/disable inner join fragment skipping. This feature is "
482  "considered stable and is enabled by default. This "
483  "parameter will be removed in a future release.");
484  desc.add_options()(
485  "max-session-duration",
486  po::value<int>(&max_session_duration)->default_value(max_session_duration),
487  "Maximum duration of active session.");
488  desc.add_options()("num-sessions",
489  po::value<int>(&system_parameters.num_sessions)
490  ->default_value(system_parameters.num_sessions),
491  "Maximum number of active session.");
492  desc.add_options()("null-div-by-zero",
493  po::value<bool>(&g_null_div_by_zero)
494  ->default_value(g_null_div_by_zero)
495  ->implicit_value(true),
496  "Return null on division by zero instead of throwing an exception.");
497  desc.add_options()(
498  "num-reader-threads",
499  po::value<size_t>(&num_reader_threads)->default_value(num_reader_threads),
500  "Number of reader threads to use.");
501  desc.add_options()(
502  "max-import-threads",
503  po::value<size_t>(&g_max_import_threads)->default_value(g_max_import_threads),
504  "Max number of default import threads to use (num hardware threads will be used "
505  "instead if lower). Can be overriden with copy statement threads option).");
506  desc.add_options()(
507  "bbox-intersect-max-table-size-bytes",
508  po::value<size_t>(&g_bbox_intersect_max_table_size_bytes)
509  ->default_value(g_bbox_intersect_max_table_size_bytes),
510  "The maximum size in bytes of the hash table for bounding box intersect.");
511  desc.add_options()("bbox-intersect-target-entries-per-bin",
512  po::value<double>(&g_bbox_intersect_target_entries_per_bin)
514  "The target number of entries per bin for bounding box intersect");
515  if (!dist_v5_) {
516  desc.add_options()("port,p",
517  po::value<int>(&system_parameters.omnisci_server_port)
518  ->default_value(system_parameters.omnisci_server_port),
519  "TCP Port number.");
520  }
521  desc.add_options()("num-gpus",
522  po::value<int>(&system_parameters.num_gpus)
523  ->default_value(system_parameters.num_gpus),
524  "Number of gpus to use.");
525  desc.add_options()(
526  "read-only",
527  po::value<bool>(&read_only)->default_value(read_only)->implicit_value(true),
528  "Enable read-only mode.");
529 
530  desc.add_options()(
531  "res-gpu-mem",
532  po::value<size_t>(&reserved_gpu_mem)->default_value(reserved_gpu_mem),
533  "Reduces GPU memory available to the HeavyDB allocator by this amount. Used for "
534  "compiled code cache and ancillary GPU functions and other processes that may also "
535  "be using the GPU concurrent with HeavyDB.");
536 
537  desc.add_options()("start-gpu",
538  po::value<int>(&system_parameters.start_gpu)
539  ->default_value(system_parameters.start_gpu),
540  "First gpu to use.");
541  desc.add_options()("trivial-loop-join-threshold",
542  po::value<unsigned>(&g_trivial_loop_join_threshold)
543  ->default_value(g_trivial_loop_join_threshold)
544  ->implicit_value(1000),
545  "The maximum number of rows in the inner table of a loop join "
546  "considered to be trivially small.");
547  desc.add_options()(
548  "uniform-request-ids-per-thrift-call",
549  po::value<bool>(&g_uniform_request_ids_per_thrift_call)
551  ->implicit_value(true),
552  "If true (default) then assign the same request_id to thrift calls that were "
553  "initiated by the same external thrift call. If false then assign different "
554  "request_ids and log the parent/child relationships.");
555  desc.add_options()("verbose",
556  po::value<bool>(&verbose_logging)
557  ->default_value(verbose_logging)
558  ->implicit_value(true),
559  "Write additional debug log messages to server logs.");
560  desc.add_options()(
561  "enable-runtime-udf",
562  po::value<bool>(&enable_runtime_udf)
563  ->default_value(enable_runtime_udf)
564  ->implicit_value(true),
565  "DEPRECATED. Please use `enable-runtime-udfs` instead as this flag will be removed "
566  "in the near future.");
567  desc.add_options()(
568  "enable-runtime-udfs",
569  po::value<bool>(&enable_runtime_udfs)
570  ->default_value(enable_runtime_udfs)
571  ->implicit_value(true),
572  "Enable runtime UDF registration by passing signatures and corresponding LLVM IR "
573  "to the `register_runtime_udf` endpoint. For use with the Python Remote Backend "
574  "Compiler server, packaged separately.");
575  desc.add_options()("enable-udf-registration-for-all-users",
576  po::value<bool>(&enable_udf_registration_for_all_users)
578  ->implicit_value(true),
579  "Allow all users, not just superusers, to register runtime "
580  "UDFs/UDTFs. Option only valid if "
581  "`--enable-runtime-udfs` is set to true.");
582  desc.add_options()("version,v", "Print Version Number.");
583  desc.add_options()("enable-string-functions",
584  po::value<bool>(&g_enable_string_functions)
585  ->default_value(g_enable_string_functions)
586  ->implicit_value(true),
587  "Enable experimental string functions.");
588  desc.add_options()("enable-experimental-string-functions",
589  po::value<bool>(&g_enable_string_functions)
590  ->default_value(g_enable_string_functions)
591  ->implicit_value(true),
592  "DEPRECATED. String functions are now enabled by default, "
593  "but can still be controlled with --enable-string-functions.");
594  desc.add_options()(
595  "enable-fsi",
596  po::value<bool>(&g_enable_fsi)->default_value(g_enable_fsi)->implicit_value(true),
597  "Enable foreign storage interface.");
598 
599  desc.add_options()("enable-legacy-delimited-import",
600  po::value<bool>(&g_enable_legacy_delimited_import)
601  ->default_value(g_enable_legacy_delimited_import)
602  ->implicit_value(true),
603  "Use legacy importer for delimited sources.");
604 #ifdef ENABLE_IMPORT_PARQUET
605  desc.add_options()("enable-legacy-parquet-import",
606  po::value<bool>(&g_enable_legacy_parquet_import)
607  ->default_value(g_enable_legacy_parquet_import)
608  ->implicit_value(true),
609  "Use legacy importer for parquet sources.");
610 #endif
611  desc.add_options()("enable-fsi-regex-import",
612  po::value<bool>(&g_enable_fsi_regex_import)
613  ->default_value(g_enable_fsi_regex_import)
614  ->implicit_value(true),
615  "Use FSI importer for regex parsed sources.");
616 
617  desc.add_options()("enable-add-metadata-columns",
618  po::value<bool>(&g_enable_add_metadata_columns)
619  ->default_value(g_enable_add_metadata_columns)
620  ->implicit_value(true),
621  "Enable add_metadata_columns COPY FROM WITH option (Beta).");
622 
623  desc.add_options()("disk-cache-path",
624  po::value<std::string>(&disk_cache_config.path),
625  "Specify the path for the disk cache.");
626 
627  desc.add_options()(
628  "disk-cache-level",
629  po::value<std::string>(&(disk_cache_level))->default_value("foreign_tables"),
630  "Specify level of disk cache. Valid options are 'foreign_tables', "
631  "'local_tables', 'none', and 'all'.");
632 
633  desc.add_options()("disk-cache-size",
634  po::value<size_t>(&(disk_cache_config.size_limit)),
635  "Specify a maximum size for the disk cache in bytes.");
636 
637  desc.add_options()(
638  "enable-interoperability",
639  po::value<bool>(&g_enable_interop)
640  ->default_value(g_enable_interop)
641  ->implicit_value(true),
642  "Enable offloading of query portions to an external execution engine.");
643  desc.add_options()("enable-union",
644  po::value<bool>(&g_enable_union)
645  ->default_value(g_enable_union)
646  ->implicit_value(true),
647  "DEPRECATED. UNION ALL is enabled by default. Please remove "
648  "use of this option, as it may be disabled in the future.");
649  desc.add_options()(
650  "calcite-service-timeout",
651  po::value<size_t>(&system_parameters.calcite_timeout)
652  ->default_value(system_parameters.calcite_timeout),
653  "Calcite server timeout (milliseconds). Increase this on systems with frequent "
654  "schema changes or when running large numbers of parallel queries.");
655  desc.add_options()("calcite-service-keepalive",
656  po::value<size_t>(&system_parameters.calcite_keepalive)
657  ->default_value(system_parameters.calcite_keepalive)
658  ->implicit_value(true),
659  "Enable keepalive on Calcite connections.");
660  desc.add_options()(
661  "stringdict-parallelizm",
662  po::value<bool>(&g_enable_stringdict_parallel)
663  ->default_value(g_enable_stringdict_parallel)
664  ->implicit_value(true),
665  "Allow StringDictionary to parallelize loads using multiple threads");
666  desc.add_options()("log-user-id",
667  po::value<bool>(&Catalog_Namespace::g_log_user_id)
668  ->default_value(Catalog_Namespace::g_log_user_id)
669  ->implicit_value(true),
670  "Log userId integer in place of the userName (when available).");
671  desc.add_options()("log-user-origin",
672  po::value<bool>(&log_user_origin)
673  ->default_value(log_user_origin)
674  ->implicit_value(true),
675  "Lookup the origin of inbound connections by IP address/DNS "
676  "name, and print this information as part of stdlog.");
677  desc.add_options()("allowed-import-paths",
678  po::value<std::string>(&allowed_import_paths),
679  "List of allowed root paths that can be used in import operations.");
680  desc.add_options()("allowed-export-paths",
681  po::value<std::string>(&allowed_export_paths),
682  "List of allowed root paths that can be used in export operations.");
683  desc.add_options()("enable-system-tables",
684  po::value<bool>(&g_enable_system_tables)
685  ->default_value(g_enable_system_tables)
686  ->implicit_value(true),
687  "Enable use of system tables.");
688  desc.add_options()("enable-table-functions",
689  po::value<bool>(&g_enable_table_functions)
690  ->default_value(g_enable_table_functions)
691  ->implicit_value(true),
692  "Enable system table functions support.");
693  desc.add_options()("enable-ml-functions",
694  po::value<bool>(&g_enable_ml_functions)
695  ->default_value(g_enable_ml_functions)
696  ->implicit_value(true),
697  "Enable ML support.");
698  desc.add_options()("restrict-ml-model-metadata-to-superusers",
701  ->implicit_value(true),
702  "RESTRICT SHOW MODEL and SHOW MODEL DETAILS to superusers only.");
703  desc.add_options()("enable-logs-system-tables",
704  po::value<bool>(&g_enable_logs_system_tables)
705  ->default_value(g_enable_logs_system_tables)
706  ->implicit_value(true),
707  "Enable use of logs system tables.");
708  desc.add_options()(
709  "logs-system-tables-max-files-count",
710  po::value<size_t>(&g_logs_system_tables_max_files_count)
711  ->default_value(g_logs_system_tables_max_files_count),
712  "Maximum number of log files that will be processed by each logs system table.");
713 #ifdef ENABLE_MEMKIND
714  desc.add_options()("enable-tiered-cpu-mem",
715  po::value<bool>(&g_enable_tiered_cpu_mem)
716  ->default_value(g_enable_tiered_cpu_mem)
717  ->implicit_value(true),
718  "Enable additional tiers of CPU memory (PMEM, etc...)");
719  desc.add_options()("pmem-size", po::value<size_t>(&g_pmem_size)->default_value(0));
720  desc.add_options()("pmem-path", po::value<std::string>(&g_pmem_path));
721 #endif
722 
723  desc.add(log_options_.get_options());
724 }
size_t g_pmem_size
float g_filter_push_down_low_frac
Definition: Execute.cpp:97
bool g_use_query_resultset_cache
Definition: Execute.cpp:153
double g_executor_resource_mgr_per_query_max_cpu_slots_ratio
Definition: Execute.cpp:175
logger::LogOptions log_options_
bool g_enable_logs_system_tables
Definition: Catalog.cpp:100
bool g_enable_legacy_delimited_import
Definition: ParserNode.cpp:83
bool g_allow_query_step_skipping
Definition: Execute.cpp:156
size_t g_logs_system_tables_max_files_count
bool g_enable_debug_timer
Definition: Logger.cpp:17
double g_bbox_intersect_target_entries_per_bin
Definition: Execute.cpp:109
size_t g_filter_push_down_passing_row_ubound
Definition: Execute.cpp:99
unsigned g_trivial_loop_join_threshold
Definition: Execute.cpp:90
bool enable_non_kernel_time_query_interrupt
int g_hll_precision_bits
double g_executor_resource_mgr_cpu_result_mem_ratio
Definition: Execute.cpp:173
po::options_description help_desc_
bool g_executor_resource_mgr_allow_cpu_gpu_kernel_concurrency
Definition: Execute.cpp:181
std::string config_file
bool g_inner_join_fragment_skipping
Definition: Execute.cpp:92
bool g_use_chunk_metadata_cache
Definition: Execute.cpp:154
bool g_enable_string_functions
bool g_null_div_by_zero
Definition: Execute.cpp:89
bool g_enable_interop
bool g_enable_executor_resource_mgr
Definition: Execute.cpp:171
bool g_restrict_ml_model_metadata_to_superusers
Definition: Execute.cpp:117
bool g_from_table_reordering
Definition: Execute.cpp:91
bool g_enable_hashjoin_many_to_many
Definition: Execute.cpp:107
bool g_enable_system_tables
Definition: SysCatalog.cpp:64
std::string g_pmem_path
float g_filter_push_down_high_frac
Definition: Execute.cpp:98
bool g_enable_distance_rangejoin
Definition: Execute.cpp:106
bool g_executor_resource_mgr_allow_cpu_result_mem_oversubscription_concurrency
Definition: Execute.cpp:187
bool g_bigint_count
bool g_allow_auto_resultset_caching
Definition: Execute.cpp:155
std::string allowed_export_paths
bool g_executor_resource_mgr_allow_cpu_slot_oversubscription_concurrency
Definition: Execute.cpp:184
size_t g_query_resultset_cache_total_bytes
Definition: Execute.cpp:159
size_t g_max_cacheable_query_resultset_size_bytes
Definition: Execute.cpp:160
bool g_executor_resource_mgr_allow_cpu_kernel_concurrency
Definition: Execute.cpp:180
unsigned pending_query_interrupt_freq
bool g_enable_fsi_regex_import
Definition: ParserNode.cpp:87
bool g_enable_bbox_intersect_hashjoin
Definition: Execute.cpp:103
bool g_cache_string_hash
bool g_enable_ml_functions
Definition: Execute.cpp:116
double g_executor_resource_mgr_per_query_max_cpu_result_mem_ratio
Definition: Execute.cpp:176
bool g_uniform_request_ids_per_thrift_call
Definition: DBHandler.cpp:125
size_t g_executor_resource_mgr_cpu_result_mem_bytes
Definition: Execute.cpp:174
bool g_enable_filter_push_down
Definition: Execute.cpp:96
boost::program_options::options_description const & get_options() const
po::positional_options_description positional_options
double g_executor_resource_mgr_max_available_resource_use_ratio
Definition: Execute.cpp:188
bool g_enable_union
bool g_allow_cpu_retry
Definition: Execute.cpp:87
File_Namespace::DiskCacheConfig disk_cache_config
bool enable_udf_registration_for_all_users
std::string allowed_import_paths
bool g_enable_stringdict_parallel
std::string disk_cache_level
size_t max_cacheable_hashtable_size_bytes
size_t watchdog_none_encoded_string_translation_limit
bool g_enable_fsi
Definition: Catalog.cpp:96
bool g_enable_thrift_logs
Definition: HeavyDB.cpp:293
bool g_enable_add_metadata_columns
Definition: ParserNode.cpp:89
size_t g_max_import_threads
Definition: Importer.cpp:105
size_t g_auto_resultset_caching_threshold
Definition: Execute.cpp:161
size_t g_bbox_intersect_max_table_size_bytes
Definition: Execute.cpp:108
bool g_enable_table_functions
Definition: Execute.cpp:115
unsigned dynamic_watchdog_time_limit
SystemParameters system_parameters

+ Here is the caller graph for this function:

std::string CommandLineOptions::getNodeIds ( )
std::vector<std::string> CommandLineOptions::getNodeIdsArray ( )
void CommandLineOptions::init_logging ( )

Definition at line 88 of file CommandLineOptions.cpp.

References logger::DEBUG1, and logger::init().

Referenced by EmbeddedDatabase::DBEngineImpl::init().

88  {
91  }
95 }
logger::LogOptions log_options_
void init(LogOptions const &log_opts)
Definition: Logger.cpp:364
Severity severity_
Definition: Logger.h:214
void set_base_path(std::string const &base_path)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

boost::optional< int > CommandLineOptions::parse_command_line ( int  argc,
char const *const *  argv,
const bool  should_init_logging = false 
)

Definition at line 1603 of file CommandLineOptions.cpp.

References SystemParameters::ALLOWED_ALL_USERS, SystemParameters::ALLOWED_SUPERUSERS_ONLY, BASELINE_HT, BBOX_INTERSECT_HT, construct_runtime_udf_registration_policy(), SystemParameters::DISALLOWED, logger::ERROR, migrations::MigrationMgr::executeRebrandMigration(), g_allow_auto_resultset_caching, g_allow_query_step_skipping, g_auto_resultset_caching_threshold, g_dynamic_watchdog_time_limit, g_enable_data_recycler, g_enable_dynamic_watchdog, g_enable_executor_resource_mgr, g_enable_filter_push_down, g_enable_fsi, g_enable_fsi_regex_import, g_enable_legacy_delimited_import, g_enable_non_kernel_time_query_interrupt, g_enable_runtime_query_interrupt, g_enable_system_tables, g_enable_union, g_enable_watchdog, g_executor_resource_mgr_allow_cpu_gpu_kernel_concurrency, g_executor_resource_mgr_allow_cpu_kernel_concurrency, g_executor_resource_mgr_allow_cpu_result_mem_oversubscription_concurrency, g_executor_resource_mgr_allow_cpu_slot_oversubscription_concurrency, g_executor_resource_mgr_cpu_result_mem_bytes, g_executor_resource_mgr_cpu_result_mem_ratio, g_executor_resource_mgr_max_available_resource_use_ratio, g_executor_resource_mgr_per_query_max_cpu_result_mem_ratio, g_executor_resource_mgr_per_query_max_cpu_slots_ratio, g_from_table_reordering, g_hashtable_cache_total_bytes, g_max_cacheable_hashtable_size_bytes, g_max_cacheable_query_resultset_size_bytes, g_max_import_threads, g_multi_instance, g_optimize_cuda_block_and_grid_sizes, g_parallel_top_max, g_parallel_top_min, g_pending_query_interrupt_freq, g_query_resultset_cache_total_bytes, g_running_query_interrupt_freq, g_use_chunk_metadata_cache, g_use_estimator_result_cache, g_use_hashtable_cache, g_use_query_resultset_cache, g_watchdog_none_encoded_string_translation_limit, BoundingBoxIntersectJoinHashTable::getHashTableCache(), PerfectJoinHashTable::getHashTableCache(), BaselineJoinHashTable::getHashTableCache(), logger::INFO, shared::kCatalogDirectoryName, shared::kDataDirectoryName, shared::kLockfilesDirectoryName, LOG, MAPD_RELEASE, migrations::MigrationMgr::migrationEnabled(), PERFECT_HT, run, anonymous_namespace{CommandLineOptions.cpp}::sanitize_config_file(), DataRecycler< CACHED_ITEM_TYPE, META_INFO_TYPE >::setMaxCacheItemSize(), DataRecycler< CACHED_ITEM_TYPE, META_INFO_TYPE >::setTotalCacheSize(), migrations::MigrationMgr::takeMigrationLock(), anonymous_namespace{CommandLineOptions.cpp}::trim_and_check_file_exists(), and UNREACHABLE.

Referenced by EmbeddedDatabase::DBEngineImpl::init(), and main().

1606  {
1607  po::options_description all_desc("All options");
1608  all_desc.add(help_desc_).add(developer_desc_);
1609 
1610  try {
1611  po::store(po::command_line_parser(argc, argv)
1612  .options(all_desc)
1613  .positional(positional_options)
1614  .run(),
1615  vm);
1616  po::notify(vm);
1617 
1618  if (vm.count("help")) {
1619  std::cerr << "Usage: heavydb <data directory path> [-p <port number>] "
1620  "[--http-port <http port number>] [--flush-log] [--version|-v]"
1621  << std::endl
1622  << std::endl;
1623  std::cout << help_desc_ << std::endl;
1624  return 0;
1625  }
1626  if (vm.count("dev-options")) {
1627  std::cout << "Usage: heavydb <data directory path> [-p <port number>] "
1628  "[--http-port <http port number>] [--flush-log] [--version|-v]"
1629  << std::endl
1630  << std::endl;
1631  std::cout << developer_desc_ << std::endl;
1632  return 0;
1633  }
1634  if (vm.count("version")) {
1635  std::cout << "HeavyDB Version: " << MAPD_RELEASE << std::endl;
1636  return 0;
1637  }
1638 
1639  if (vm.count("config")) {
1640  std::ifstream settings_file(system_parameters.config_file);
1641 
1642  auto sanitized_settings = sanitize_config_file(settings_file);
1643 
1644  po::store(po::parse_config_file(sanitized_settings, all_desc, false), vm);
1645  po::notify(vm);
1646  settings_file.close();
1647  }
1648 
1649  if (!g_enable_union) {
1650  std::cerr
1651  << "The enable-union option is DEPRECATED and is now enabled by default. "
1652  "Please remove use of this option, as it may be disabled in the future."
1653  << std::endl;
1654  }
1655 
1656  // Trim base path before executing migration
1657  boost::algorithm::trim_if(base_path, boost::is_any_of("\"'"));
1658  if (!boost::filesystem::exists(base_path)) {
1659  std::cerr << "Storage folder (--data) not found: " << base_path << std::endl;
1660  std::cerr << "Need to run initheavy before heavydb." << std::endl;
1661  return 1;
1662  }
1663 
1664  // Execute rebrand migration before accessing any system files.
1665  std::string lockfiles_path = base_path + "/" + shared::kLockfilesDirectoryName;
1666  if (!boost::filesystem::exists(lockfiles_path)) {
1667  if (!boost::filesystem::create_directory(lockfiles_path)) {
1668  std::cerr << "Cannot create " + shared::kLockfilesDirectoryName +
1669  " subdirectory under "
1670  << base_path << std::endl;
1671  return 1;
1672  }
1673  }
1674  std::string lockfiles_path2 = lockfiles_path + "/" + shared::kCatalogDirectoryName;
1675  if (!boost::filesystem::exists(lockfiles_path2)) {
1676  if (!boost::filesystem::create_directory(lockfiles_path2)) {
1677  std::cerr << "Cannot create " + shared::kLockfilesDirectoryName + "/" +
1678  shared::kCatalogDirectoryName + " subdirectory under "
1679  << base_path << std::endl;
1680  return 1;
1681  }
1682  }
1683  std::string lockfiles_path3 = lockfiles_path + "/" + shared::kDataDirectoryName;
1684  if (!boost::filesystem::exists(lockfiles_path3)) {
1685  if (!boost::filesystem::create_directory(lockfiles_path3)) {
1686  std::cerr << "Cannot create " + shared::kLockfilesDirectoryName + "/" +
1687  shared::kDataDirectoryName + " subdirectory under "
1688  << base_path << std::endl;
1689  return 1;
1690  }
1691  }
1695  }
1696 
1697  if (!vm["enable-runtime-udf"].defaulted()) {
1698  if (!vm["enable-runtime-udfs"].defaulted()) {
1699  std::cerr << "Usage Error: Both enable-runtime-udf and enable-runtime-udfs "
1700  "specified. Please remove use of the enable-runtime-udfs flag, "
1701  "as it will be deprecated in the future."
1702  << std::endl;
1703  return 1;
1704  } else {
1706  std::cerr << "The enable-runtime-udf flag has been deprecated and replaced "
1707  "with enable-runtime-udfs. Please remove use of this option "
1708  "as it will be disabled in the future."
1709  << std::endl;
1710  }
1711  }
1715 
1716  if (should_init_logging) {
1717  init_logging();
1718  }
1719 
1721  return 1;
1722  }
1723  if (!trim_and_check_file_exists(authMetadata.ca_file_name, "ca file name")) {
1724  return 1;
1725  }
1727  "ssl trust store")) {
1728  return 1;
1729  }
1731  return 1;
1732  }
1734  return 1;
1735  }
1737  return 1;
1738  }
1739 
1754  if (g_use_hashtable_cache) {
1767  }
1769 
1770  if (g_multi_instance) {
1771  LOG(INFO) << "Disabling FSI and Disk Cache as they are not currently supported "
1772  "with multi-instance.";
1773  // Fsi & disk cache currently unsupported in multi-instance. The other option
1774  // configs are required because they require fsi support.
1775  g_enable_fsi = false;
1776  g_enable_fsi_regex_import = false;
1777  g_enable_system_tables = false;
1779 #ifdef ENABLE_IMPORT_PARQUET
1780  g_enable_legacy_parquet_import = true;
1781 #endif
1782  disk_cache_level = "none";
1783  }
1784 
1785  } catch (po::error& e) {
1786  std::cerr << "Usage Error: " << e.what() << std::endl;
1787  return 1;
1788  }
1789 
1790  if (g_hll_precision_bits < 1 || g_hll_precision_bits > 16) {
1791  std::cerr << "hll-precision-bits must be between 1 and 16." << std::endl;
1792  return 1;
1793  }
1794 
1795  if (!g_from_table_reordering) {
1796  LOG(INFO) << " From clause table reordering is disabled";
1797  }
1798 
1800  LOG(INFO) << " Filter push down for JOIN is enabled";
1801  }
1802 
1803  if (vm.count("udf")) {
1804  boost::algorithm::trim_if(udf_file_name, boost::is_any_of("\"'"));
1805 
1806  if (!boost::filesystem::exists(udf_file_name)) {
1807  LOG(ERROR) << " User defined function file " << udf_file_name << " does not exist.";
1808  return 1;
1809  }
1810 
1811  LOG(INFO) << " User provided extension functions loaded from " << udf_file_name;
1812  }
1813 
1814  if (vm.count("udf-compiler-path")) {
1815  boost::algorithm::trim_if(udf_compiler_path, boost::is_any_of("\"'"));
1816  }
1817 
1818 #ifdef HAVE_TORCH_TFS
1819  if (vm.count("torch-lib-path")) {
1820  boost::algorithm::trim_if(torch_lib_path, boost::is_any_of("\"'"));
1821  }
1822 #endif
1823 
1824  auto trim_string = [](std::string& s) {
1825  boost::algorithm::trim_if(s, boost::is_any_of("\"'"));
1826  };
1827 
1828  if (vm.count("udf-compiler-options")) {
1829  std::for_each(udf_compiler_options.begin(), udf_compiler_options.end(), trim_string);
1830  }
1831 
1832  boost::algorithm::trim_if(system_parameters.ha_brokers, boost::is_any_of("\"'"));
1833  boost::algorithm::trim_if(system_parameters.ha_group_id, boost::is_any_of("\"'"));
1834  boost::algorithm::trim_if(system_parameters.ha_shared_data, boost::is_any_of("\"'"));
1835  boost::algorithm::trim_if(system_parameters.ha_unique_server_id,
1836  boost::is_any_of("\"'"));
1837 
1838  if (!system_parameters.ha_group_id.empty()) {
1839  LOG(INFO) << " HA group id " << system_parameters.ha_group_id;
1840  if (system_parameters.ha_unique_server_id.empty()) {
1841  LOG(ERROR) << "Starting server in HA mode --ha-unique-server-id must be set ";
1842  return 5;
1843  } else {
1844  LOG(INFO) << " HA unique server id " << system_parameters.ha_unique_server_id;
1845  }
1846  if (system_parameters.ha_brokers.empty()) {
1847  LOG(ERROR) << "Starting server in HA mode --ha-brokers must be set ";
1848  return 6;
1849  } else {
1850  LOG(INFO) << " HA brokers " << system_parameters.ha_brokers;
1851  }
1852  if (system_parameters.ha_shared_data.empty()) {
1853  LOG(ERROR) << "Starting server in HA mode --ha-shared-data must be set ";
1854  return 7;
1855  } else {
1856  LOG(INFO) << " HA shared data is " << system_parameters.ha_shared_data;
1857  }
1858  }
1859 
1860  boost::algorithm::trim_if(system_parameters.master_address, boost::is_any_of("\"'"));
1861  if (!system_parameters.master_address.empty()) {
1862  if (!read_only) {
1863  LOG(ERROR) << "The master-address setting is only allowed in read-only mode";
1864  return 9;
1865  }
1866  LOG(INFO) << " Master Address is " << system_parameters.master_address;
1867  LOG(INFO) << " Master Port is " << system_parameters.master_port;
1868  }
1869 
1870  if (g_max_import_threads < 1) {
1871  std::cerr << "max-import-threads must be >= 1 (was set to " << g_max_import_threads
1872  << ")." << std::endl;
1873  return 8;
1874  } else {
1875  LOG(INFO) << " Max import threads " << g_max_import_threads;
1876  }
1877 
1879  LOG(INFO) << " cuda block size " << system_parameters.cuda_block_size;
1880  }
1882  LOG(INFO) << " cuda grid size " << system_parameters.cuda_grid_size;
1883  }
1884  LOG(INFO) << " Min CPU buffer pool slab size (in bytes) "
1886  LOG(INFO) << " Max CPU buffer pool slab size (in bytes) "
1888  LOG(INFO) << " Min GPU buffer pool slab size (in bytes) "
1890  LOG(INFO) << " Max GPU buffer pool slab size (in bytes) "
1892  LOG(INFO) << " calcite JVM max memory (in MB) " << system_parameters.calcite_max_mem;
1893  LOG(INFO) << " HeavyDB Server Port " << system_parameters.omnisci_server_port;
1894  LOG(INFO) << " HeavyDB Calcite Port " << system_parameters.calcite_port;
1895  LOG(INFO) << " Enable Calcite view optimize "
1897  LOG(INFO) << " Allow Local Auth Fallback: "
1898  << (authMetadata.allowLocalAuthFallback ? "enabled" : "disabled");
1899  LOG(INFO) << " ParallelTop min threshold: " << g_parallel_top_min;
1900  LOG(INFO) << " ParallelTop watchdog max: " << g_parallel_top_max;
1901 
1902  LOG(INFO) << " Enable Data Recycler: "
1903  << (g_enable_data_recycler ? "enabled" : "disabled");
1904  if (g_enable_data_recycler) {
1905  LOG(INFO) << " \t Use hashtable cache: "
1906  << (g_use_hashtable_cache ? "enabled" : "disabled");
1907  if (g_use_hashtable_cache) {
1908  LOG(INFO) << " \t\t Total amount of bytes that hashtable cache keeps: "
1909  << g_hashtable_cache_total_bytes / (1024 * 1024) << " MB.";
1910  LOG(INFO) << " \t\t Per-hashtable size limit: "
1911  << g_max_cacheable_hashtable_size_bytes / (1024 * 1024) << " MB.";
1912  }
1913  LOG(INFO) << " \t Use query resultset cache: "
1914  << (g_use_query_resultset_cache ? "enabled" : "disabled");
1916  LOG(INFO) << " \t\t Total amount of bytes that query resultset cache keeps: "
1917  << g_query_resultset_cache_total_bytes / (1024 * 1024) << " MB.";
1918  LOG(INFO) << " \t\t Per-query resultset size limit: "
1919  << g_max_cacheable_query_resultset_size_bytes / (1024 * 1024) << " MB.";
1920  }
1921  LOG(INFO) << " \t\t Use auto query resultset caching: "
1922  << (g_allow_auto_resultset_caching ? "enabled" : "disabled");
1924  LOG(INFO) << " \t\t\t The maximum bytes of a query resultset which is "
1925  "automatically cached: "
1926  << g_auto_resultset_caching_threshold << " Bytes.";
1927  }
1928  LOG(INFO) << " \t\t Use query step skipping: "
1929  << (g_allow_query_step_skipping ? "enabled" : "disabled");
1930  LOG(INFO) << " \t Use chunk metadata cache: "
1931  << (g_use_chunk_metadata_cache ? "enabled" : "disabled");
1932  }
1933  LOG(INFO) << "Executor Resource Manager: "
1934  << (g_enable_executor_resource_mgr ? "enabled" : "disabled");
1936  LOG(INFO) << "\tCPU kernel concurrency: "
1938  : "disabled");
1939  LOG(INFO) << "\tCPU-GPU kernel concurrency: "
1941  : "disabled");
1943  LOG(INFO) << "\tCPU result set reserved allocation: "
1944  << g_executor_resource_mgr_cpu_result_mem_bytes / (1024 * 1024) << " MB";
1945  } else {
1946  LOG(INFO) << "\tCPU result set reserved ratio of CPU buffer pool size: "
1948  }
1949  LOG(INFO) << "\tPer-query max CPU threads ratio: "
1951  LOG(INFO) << "\tPer-query max CPU result memory ratio of allocated total: "
1953  LOG(INFO) << "\tAllow concurrent CPU thread/slot oversubscription: "
1955  ? "enabled"
1956  : "disabled");
1957  LOG(INFO)
1958  << "\tAllow concurrent CPU result memory oversubscription: "
1960  ? "enabled"
1961  : "disabled");
1962  LOG(INFO) << "\tPer-query Max available resource utilization ratio: "
1964  }
1965 
1966  const std::string udf_reg_policy_log_prefix{"Runtime UDF/UDTF Registration Policy: "};
1969  LOG(INFO) << udf_reg_policy_log_prefix << " DISALLOWED";
1970  break;
1971  }
1973  LOG(INFO) << udf_reg_policy_log_prefix << " ALLOWED for superusers only";
1974  break;
1975  }
1977  LOG(INFO) << udf_reg_policy_log_prefix << " ALLOWED for all users";
1978  break;
1979  }
1980  default: {
1981  UNREACHABLE() << "Unrecognized option for Runtime UDF/UDTF registration policy.";
1982  }
1983  }
1984 
1985  boost::algorithm::trim_if(authMetadata.distinguishedName, boost::is_any_of("\"'"));
1986  boost::algorithm::trim_if(authMetadata.uri, boost::is_any_of("\"'"));
1987  boost::algorithm::trim_if(authMetadata.ldapQueryUrl, boost::is_any_of("\"'"));
1988  boost::algorithm::trim_if(authMetadata.ldapRoleRegex, boost::is_any_of("\"'"));
1989  boost::algorithm::trim_if(authMetadata.ldapSuperUserRole, boost::is_any_of("\"'"));
1990 
1991  return boost::none;
1992 }
std::string distinguishedName
Definition: AuthMetadata.h:25
double g_running_query_interrupt_freq
Definition: Execute.cpp:134
size_t g_parallel_top_max
Definition: ResultSet.cpp:50
const std::string kDataDirectoryName
std::string ldapQueryUrl
Definition: AuthMetadata.h:26
bool g_use_query_resultset_cache
Definition: Execute.cpp:153
bool g_multi_instance
Definition: heavyai_locks.h:22
double g_executor_resource_mgr_per_query_max_cpu_slots_ratio
Definition: Execute.cpp:175
SystemParameters::RuntimeUdfRegistrationPolicy construct_runtime_udf_registration_policy(const bool enable_runtime_udfs, const bool enable_udf_registration_for_all_users)
bool trim_and_check_file_exists(std::string &filename, const std::string desc)
std::string ha_shared_data
std::string udf_compiler_path
bool g_enable_legacy_delimited_import
Definition: ParserNode.cpp:83
unsigned g_pending_query_interrupt_freq
Definition: Execute.cpp:133
#define LOG(tag)
Definition: Logger.h:285
bool enable_calcite_view_optimize
bool g_allow_query_step_skipping
Definition: Execute.cpp:156
std::string ldapRoleRegex
Definition: AuthMetadata.h:27
#define UNREACHABLE()
Definition: Logger.h:338
bool g_enable_dynamic_watchdog
Definition: Execute.cpp:81
size_t g_hashtable_cache_total_bytes
Definition: Execute.cpp:157
bool g_enable_non_kernel_time_query_interrupt
Definition: Execute.cpp:131
void setMaxCacheItemSize(CacheItemType item_type, size_t new_max_cache_item_size)
Definition: DataRecycler.h:613
bool enable_non_kernel_time_query_interrupt
double g_executor_resource_mgr_cpu_result_mem_ratio
Definition: Execute.cpp:173
bool g_enable_data_recycler
Definition: Execute.cpp:151
po::options_description help_desc_
bool g_executor_resource_mgr_allow_cpu_gpu_kernel_concurrency
Definition: Execute.cpp:181
std::string config_file
bool g_use_chunk_metadata_cache
Definition: Execute.cpp:154
size_t g_max_cacheable_hashtable_size_bytes
Definition: Execute.cpp:158
size_t g_watchdog_none_encoded_string_translation_limit
Definition: Execute.cpp:82
std::string ha_brokers
bool g_enable_executor_resource_mgr
Definition: Execute.cpp:171
size_t g_parallel_top_min
Definition: ResultSet.cpp:49
std::string ssl_trust_ca_file
std::string ssl_trust_store
bool g_from_table_reordering
Definition: Execute.cpp:91
bool g_enable_system_tables
Definition: SysCatalog.cpp:64
static HashtableRecycler * getHashTableCache()
static bool migrationEnabled()
Definition: MigrationMgr.h:47
bool g_executor_resource_mgr_allow_cpu_result_mem_oversubscription_concurrency
Definition: Execute.cpp:187
std::string uri
Definition: AuthMetadata.h:24
bool g_enable_watchdog
bool g_allow_auto_resultset_caching
Definition: Execute.cpp:155
std::string ha_unique_server_id
std::string ca_file_name
Definition: AuthMetadata.h:31
void setTotalCacheSize(CacheItemType item_type, size_t new_total_cache_size)
Definition: DataRecycler.h:606
std::string ssl_key_file
AuthMetadata authMetadata
bool g_optimize_cuda_block_and_grid_sizes
Definition: Execute.cpp:162
bool g_executor_resource_mgr_allow_cpu_slot_oversubscription_concurrency
Definition: Execute.cpp:184
size_t g_query_resultset_cache_total_bytes
Definition: Execute.cpp:159
size_t g_max_cacheable_query_resultset_size_bytes
Definition: Execute.cpp:160
RuntimeUdfRegistrationPolicy runtime_udf_registration_policy
std::string ldapSuperUserRole
Definition: AuthMetadata.h:28
static void executeRebrandMigration(const std::string &base_path)
static void takeMigrationLock(const std::string &base_path)
bool g_executor_resource_mgr_allow_cpu_kernel_concurrency
Definition: Execute.cpp:180
unsigned pending_query_interrupt_freq
bool g_enable_fsi_regex_import
Definition: ParserNode.cpp:87
std::stringstream sanitize_config_file(std::ifstream &in)
std::vector< std::string > udf_compiler_options
const std::string kCatalogDirectoryName
double g_executor_resource_mgr_per_query_max_cpu_result_mem_ratio
Definition: Execute.cpp:176
size_t g_executor_resource_mgr_cpu_result_mem_bytes
Definition: Execute.cpp:174
bool g_enable_filter_push_down
Definition: Execute.cpp:96
bool g_use_estimator_result_cache
Definition: Execute.cpp:132
bool allowLocalAuthFallback
Definition: AuthMetadata.h:32
po::positional_options_description positional_options
static HashtableRecycler * getHashTableCache()
double g_executor_resource_mgr_max_available_resource_use_ratio
Definition: Execute.cpp:188
bool g_enable_union
std::string ssl_keystore
const std::string kLockfilesDirectoryName
bool enable_udf_registration_for_all_users
static const std::string MAPD_RELEASE
Definition: release.h:42
std::string disk_cache_level
static bool run
po::variables_map vm
size_t max_cacheable_hashtable_size_bytes
size_t watchdog_none_encoded_string_translation_limit
unsigned g_dynamic_watchdog_time_limit
Definition: Execute.cpp:86
bool g_enable_fsi
Definition: Catalog.cpp:96
std::string ha_group_id
po::options_description developer_desc_
bool g_enable_runtime_query_interrupt
Definition: Execute.cpp:130
size_t g_max_import_threads
Definition: Importer.cpp:105
bool g_use_hashtable_cache
Definition: Execute.cpp:152
size_t g_auto_resultset_caching_threshold
Definition: Execute.cpp:161
std::string master_address
unsigned dynamic_watchdog_time_limit
SystemParameters system_parameters
std::string ssl_cert_file

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void CommandLineOptions::validate ( )

Definition at line 1206 of file CommandLineOptions.cpp.

References anonymous_namespace{CommandLineOptions.cpp}::addOptionalFileToBlacklist(), ddl_utils::FilePathBlacklist::addToBlacklist(), File_Namespace::all, test_exceptions::data_path, File_Namespace::fsi, heavyai::ftruncate(), g_allow_invalid_literal_buffer_reads, g_enable_debug_timer, g_enable_executor_resource_mgr, g_enable_fsi, g_enable_fsi_regex_import, g_enable_legacy_delimited_import, g_enable_logs_system_tables, g_enable_ml_functions, g_enable_s3_fsi, g_enable_system_tables, g_enable_table_functions, g_executor_resource_mgr_allow_cpu_gpu_kernel_concurrency, g_executor_resource_mgr_allow_cpu_kernel_concurrency, g_executor_resource_mgr_allow_cpu_result_mem_oversubscription_concurrency, g_executor_resource_mgr_allow_cpu_slot_oversubscription_concurrency, g_executor_resource_mgr_cpu_result_mem_ratio, g_executor_resource_mgr_max_available_resource_use_ratio, g_executor_resource_mgr_per_query_max_cpu_result_mem_ratio, g_executor_resource_mgr_per_query_max_cpu_slots_ratio, Catalog_Namespace::g_log_user_id, g_logs_system_tables_max_files_count, g_multi_instance, g_pmem_path, g_pmem_size, g_read_only, g_vacuum_min_selectivity, File_Namespace::CachingFileMgr::getMinimumSize(), logger::INFO, ddl_utils::FilePathWhitelist::initialize(), shared::kCatalogDirectoryName, shared::kDataDirectoryName, shared::kDefaultDiskCacheDirName, shared::kDefaultLicenseFileName, shared::kDefaultLogDirName, shared::kSystemCatalogName, LOG, File_Namespace::non_fsi, File_Namespace::none, heavyai::safe_close(), heavyai::safe_fcntl(), heavyai::safe_open(), heavyai::safe_write(), import_export::ForeignDataImporter::setDefaultImportPath(), to_string(), VLOG, and logger::WARNING.

Referenced by main().

1206  {
1207  boost::algorithm::trim_if(base_path, boost::is_any_of("\"'"));
1208  const auto data_path = boost::filesystem::path(base_path) / shared::kDataDirectoryName;
1209  if (!boost::filesystem::exists(data_path)) {
1210  throw std::runtime_error("HeavyDB data directory does not exist at '" + base_path +
1211  "'");
1212  }
1213 
1214 // TODO: support lock on Windows
1215 #ifndef _WIN32
1216  {
1217  // If we aren't sharing the data directory, take and hold a write lock on
1218  // heavydb_pid.lck to prevent other processes from trying to share our dir.
1219  // TODO(sy): Probably need to get rid of this PID file because it doesn't make much
1220  // sense to store only one server's PID when we have the --multi-instance option.
1221  auto exe_filename = boost::filesystem::path(exe_name).filename().string();
1222  const std::string lock_file =
1223  (boost::filesystem::path(base_path) / std::string(exe_filename + "_pid.lck"))
1224  .string();
1225  auto pid = std::to_string(getpid());
1226  if (!g_multi_instance) {
1227  VLOG(1) << "taking [" << lock_file << "] read+write lock until process exit";
1228  } else {
1229  VLOG(1) << "taking [" << lock_file << "] read-only lock until process exit";
1230  }
1231 
1232  int fd;
1233  fd = heavyai::safe_open(lock_file.c_str(), O_RDWR | O_CREAT, 0664);
1234  if (fd == -1) {
1235  throw std::runtime_error("failed to open lockfile: " + lock_file + ": " +
1236  std::string(strerror(errno)) + " (" +
1237  std::to_string(errno) + ")");
1238  }
1239 
1240  struct flock fl;
1241  memset(&fl, 0, sizeof(fl));
1242  fl.l_type = !g_multi_instance ? F_WRLCK : F_RDLCK;
1243  fl.l_whence = SEEK_SET;
1244  int cmd;
1245 #ifdef __linux__
1246  // cmd = F_OFD_SETLK; // TODO(sy): broken on centos
1247  cmd = F_SETLK;
1248 #else
1249  cmd = F_SETLK;
1250 #endif // __linux__
1251  int ret = heavyai::safe_fcntl(fd, cmd, &fl);
1252  if (ret == -1 && (errno == EACCES || errno == EAGAIN)) { // locked by someone else
1253  heavyai::safe_close(fd);
1254  throw std::runtime_error(
1255  "another HeavyDB server instance is already using data directory: " +
1256  base_path);
1257  } else if (ret == -1) {
1258  auto errno0 = errno;
1259  heavyai::safe_close(fd);
1260  throw std::runtime_error("failed to lock lockfile: " + lock_file + ": " +
1261  std::string(strerror(errno0)) + " (" +
1262  std::to_string(errno0) + ")");
1263  }
1264 
1265  if (!g_multi_instance) {
1266  if (heavyai::ftruncate(fd, 0) == -1) {
1267  auto errno0 = errno;
1268  heavyai::safe_close(fd);
1269  throw std::runtime_error("failed to truncate lockfile: " + lock_file + ": " +
1270  std::string(strerror(errno0)) + " (" +
1271  std::to_string(errno0) + ")");
1272  }
1273  if (heavyai::safe_write(fd, pid.c_str(), pid.length()) == -1) {
1274  auto errno0 = errno;
1275  heavyai::safe_close(fd);
1276  throw std::runtime_error("failed to write lockfile: " + lock_file + ": " +
1277  std::string(strerror(errno0)) + " (" +
1278  std::to_string(errno0) + ")");
1279  }
1280  }
1281 
1282  // Intentionally leak the file descriptor. Lock will be held until process exit.
1283  }
1284 #endif // _WIN32
1285 
1286  boost::algorithm::trim_if(db_query_file, boost::is_any_of("\"'"));
1287  if (db_query_file.length() > 0 && !boost::filesystem::exists(db_query_file)) {
1288  throw std::runtime_error("File containing DB queries " + db_query_file +
1289  " does not exist.");
1290  }
1291  const auto db_file = boost::filesystem::path(base_path) /
1293  if (!boost::filesystem::exists(db_file)) {
1294  { // check old system catalog existsense
1295  const auto db_file =
1296  boost::filesystem::path(base_path) / shared::kCatalogDirectoryName / "mapd";
1297  if (!boost::filesystem::exists(db_file)) {
1298  throw std::runtime_error("System catalog " + shared::kSystemCatalogName +
1299  " does not exist.");
1300  }
1301  }
1302  }
1303  if (license_path.length() == 0) {
1305  }
1306 
1307  // add all parameters to be displayed on startup
1308  LOG(INFO) << "HeavyDB started with data directory at '" << base_path << "'";
1309  if (vm.count("license-path")) {
1310  LOG(INFO) << "License key path set to '" << license_path << "'";
1311  }
1313  LOG(INFO) << " Server read-only mode is " << read_only << " (--read-only)";
1314  if (g_multi_instance) {
1315  LOG(INFO) << " Multiple servers per --data directory is " << g_multi_instance
1316  << " (--multi-instance)";
1317  }
1318  if (g_read_only && g_multi_instance) {
1319  throw std::runtime_error(
1320  "You may not use the --read-only and --multi-instance configuration flags "
1321  "simultaneously.");
1322  }
1324  LOG(WARNING) << " Allowing invalid reads from the literal buffer. May cause invalid "
1325  "query results! (--allow-invalid-literal-buffer-reads)";
1326  }
1327 #if DISABLE_CONCURRENCY
1328  LOG(INFO) << " Threading layer: serial";
1329 #elif ENABLE_TBB
1330  LOG(INFO) << " Threading layer: TBB";
1331 #else
1332  LOG(INFO) << " Threading layer: std";
1333 #endif
1334  LOG(INFO) << " Watchdog is set to " << enable_watchdog;
1335  LOG(INFO) << " Dynamic Watchdog is set to " << enable_dynamic_watchdog;
1337  LOG(INFO) << " Dynamic Watchdog timeout is set to " << dynamic_watchdog_time_limit;
1338  }
1339  LOG(INFO) << " Runtime query interrupt is set to " << enable_runtime_query_interrupt;
1341  LOG(INFO) << " A frequency of checking pending query interrupt request is set to "
1342  << pending_query_interrupt_freq << " (in ms.)";
1343  LOG(INFO) << " A frequency of checking running query interrupt request is set to "
1344  << running_query_interrupt_freq << " (0.0 ~ 1.0)";
1345  }
1346  LOG(INFO) << " Non-kernel time query interrupt is set to "
1348 
1349  LOG(INFO) << " Debug Timer is set to " << g_enable_debug_timer;
1350  LOG(INFO) << " LogUserId is set to " << Catalog_Namespace::g_log_user_id;
1351  LOG(INFO) << " Maximum idle session duration " << idle_session_duration;
1352  LOG(INFO) << " Maximum active session duration " << max_session_duration;
1353  LOG(INFO) << " Maximum number of sessions " << system_parameters.num_sessions;
1354 
1355  LOG(INFO) << "Legacy delimited import is set to " << g_enable_legacy_delimited_import;
1356 #ifdef ENABLE_IMPORT_PARQUET
1357  LOG(INFO) << "Legacy parquet import is set to " << g_enable_legacy_parquet_import;
1358 #endif
1359  LOG(INFO) << "FSI regex parsed import is set to " << g_enable_fsi_regex_import;
1360 
1361  LOG(INFO) << "Allowed import paths is set to " << allowed_import_paths;
1362  LOG(INFO) << "Allowed export paths is set to " << allowed_export_paths;
1365 
1375  g_enable_s3_fsi = false;
1376 
1378 #ifdef ENABLE_IMPORT_PARQUET
1379  !g_enable_legacy_parquet_import ||
1380 #endif
1382  g_enable_fsi =
1383  true; // a requirement for FSI import code-paths is for FSI to be enabled
1384  LOG(INFO) << "FSI has been enabled as a side effect of enabling non-legacy import.";
1385  }
1386 
1387  const bool executor_resource_mgr_cpu_result_mem_ratio_flag_set =
1388  vm["executor-cpu-result-mem-ratio"].defaulted() ? false : true;
1389  const bool executor_resource_mgr_cpu_result_mem_bytes_flag_set =
1390  vm["executor-cpu-result-mem-bytes"].defaulted() ? false : true;
1391  const bool executor_resource_mgr_per_query_max_cpu_thread_ratio_flag_set =
1392  vm["executor-per-query-max-cpu-threads-ratio"].defaulted() ? false : true;
1393  const bool executor_resource_mgr_per_query_max_cpu_result_mem_ratio_flag_set =
1394  vm["executor-per-query-max-cpu-result-mem-ratio"].defaulted() ? false : true;
1395  const bool executor_resource_mgr_cpu_kernel_concurrency_flag_set =
1396  vm["allow-cpu-kernel-concurrency"].defaulted() ? false : true;
1397  const bool executor_resource_mgr_cpu_gpu_kernel_concurrency_flag_set =
1398  vm["allow-cpu-gpu-kernel-concurrency"].defaulted() ? false : true;
1399  const bool executor_resource_mgr_cpu_thread_oversubscription_concurrency_flag_set =
1400  vm["allow-cpu-thread-oversubscription-concurrency"].defaulted() ? false : true;
1401  const bool executor_resource_mgr_cpu_result_mem_oversubscription_concurrency_flag_set =
1402  vm["allow-cpu-result-mem-oversubscription-concurrency"].defaulted() ? false : true;
1403 
1405  if (executor_resource_mgr_cpu_result_mem_bytes_flag_set) {
1406  throw std::runtime_error(
1407  "Cannot set executor-cpu-result-mem-bytes without enable-executor-resource-mgr "
1408  "option enabled");
1409  }
1410  if (executor_resource_mgr_cpu_result_mem_ratio_flag_set) {
1411  throw std::runtime_error(
1412  "Cannot set executor-cpu-result-mem-ratio without enable-executor-resource-mgr "
1413  "option enabled");
1414  }
1415  if (executor_resource_mgr_per_query_max_cpu_thread_ratio_flag_set) {
1416  throw std::runtime_error(
1417  "Cannot set executor-per-query-max-cpu-slots-ratio without "
1418  "enable-executor-resource-mgr option enabled");
1419  }
1420  if (executor_resource_mgr_per_query_max_cpu_result_mem_ratio_flag_set) {
1421  throw std::runtime_error(
1422  "Cannot set executor-per-query-max-cpu-result-mem-ratio without "
1423  "enable-executor-resource-mgr option enabled");
1424  }
1425  if (executor_resource_mgr_cpu_kernel_concurrency_flag_set) {
1426  throw std::runtime_error(
1427  "Cannot set allow-cpu-kernel-concurrency without "
1428  "enable-executor-resource-mgr option enabled");
1429  }
1430  if (executor_resource_mgr_cpu_gpu_kernel_concurrency_flag_set) {
1431  throw std::runtime_error(
1432  "Cannot set allow-cpu-gpu-kernel-concurrency without "
1433  "enable-executor-resource-mgr option enabled");
1434  }
1435  if (executor_resource_mgr_cpu_thread_oversubscription_concurrency_flag_set) {
1436  throw std::runtime_error(
1437  "Cannot set allow-cpu-thread-oversubscription-concurrency without "
1438  "enable-executor-resource-mgr option enabled");
1439  }
1440  if (executor_resource_mgr_cpu_result_mem_oversubscription_concurrency_flag_set) {
1441  throw std::runtime_error(
1442  "Cannot set allow-cpu-thread-result-mem-concurrency without "
1443  "enable-executor-resource-mgr option enabled");
1444  }
1445  }
1446  if (executor_resource_mgr_cpu_result_mem_bytes_flag_set &&
1447  executor_resource_mgr_cpu_result_mem_ratio_flag_set) {
1448  throw std::runtime_error(
1449  "Setting both executor-cpu-result-mem-bytes and executor-cpu-result-mem-ratio is "
1450  "not allowed as the flags are mutually exclusive.");
1451  }
1455  throw std::runtime_error(
1456  "allow-cpu-thread-oversubscription-concurrency cannot be set without at least "
1457  "one of allow-cpu-kernel-concurrency or allow-cpu-gpu-kernel-concurrency being "
1458  "set.");
1459  }
1461  throw std::runtime_error(
1462  "allow-cpu-result-mem-oversubscription-concurrency cannot be set without at "
1463  "least one of allow-cpu-kernel-concurrency or allow-cpu-gpu-kernel-concurrency "
1464  "being set.");
1465  }
1466  }
1467 
1469  throw std::runtime_error(
1470  "Invalid value for executor-cpu-result-mem-ratio, must be greater than 0.");
1471  }
1473  throw std::runtime_error(
1474  "Invalid value for executor-per-query-max-cpu-slots-ratio, must be greater than "
1475  "0.");
1476  }
1478  throw std::runtime_error(
1479  "Invalid value for executor-per-query-max-cpu-result-mem-ratio, must be greater "
1480  "than "
1481  "0.");
1482  }
1485  throw std::runtime_error(
1486  "Invalid value for executor-max-available-resource-use-ratio, must be greater "
1487  "than "
1488  "0. and less than or equal to 1.0");
1489  }
1490 
1491 #ifndef HAVE_SYSTEM_TFS
1493  g_enable_table_functions = false;
1494  LOG(INFO) << "System table functions turned off due to HeavyDB being built without "
1495  "table function support.";
1496  }
1497 #endif // HAVE_SYSTEM_TFS
1499  g_enable_ml_functions = false;
1500  LOG(INFO) << "ML functions turned off due to `--enable-table-functions` being set to "
1501  "false. Please enable table functions to use ML functionality.";
1502  }
1503 
1504  if (disk_cache_level == "foreign_tables") {
1505  if (g_enable_fsi) {
1507  LOG(INFO) << "Disk cache enabled for foreign tables only";
1508  } else {
1509  LOG(INFO) << "Cannot enable disk cache for fsi when fsi is disabled. Defaulted to "
1510  "disk cache disabled";
1511  }
1512  } else if (disk_cache_level == "all") {
1514  LOG(INFO) << "Disk cache enabled for all tables";
1515  } else if (disk_cache_level == "local_tables") {
1517  LOG(INFO) << "Disk cache enabled for non-FSI tables";
1518  } else if (disk_cache_level == "none") {
1520  LOG(INFO) << "Disk cache disabled";
1521  } else {
1522  throw std::runtime_error{
1523  "Unexpected \"disk-cache-level\" value: " + disk_cache_level +
1524  ". Valid options are 'foreign_tables', "
1525  "'local_tables', 'none', and 'all'."};
1526  }
1527 
1529  throw std::runtime_error{"disk-cache-size must be at least " +
1531  }
1532 
1533  if (disk_cache_config.path.empty()) {
1535  }
1537 
1540 
1541  // If passed in, blacklist all security config files
1550 
1551  if (g_vacuum_min_selectivity < 0) {
1552  throw std::runtime_error{"vacuum-min-selectivity cannot be less than 0."};
1553  }
1554  LOG(INFO) << "Vacuum Min Selectivity: " << g_vacuum_min_selectivity;
1555 
1556  LOG(INFO) << "Enable system tables is set to " << g_enable_system_tables;
1557  if (g_enable_system_tables) {
1558  // System tables currently reuse FSI infrastructure and therefore, require FSI to be
1559  // enabled
1560  if (!g_enable_fsi) {
1561  g_enable_fsi = true;
1562  LOG(INFO) << "FSI has been enabled as a side effect of enabling system tables";
1563  }
1564  }
1565  LOG(INFO) << "Enable FSI is set to " << g_enable_fsi;
1566  LOG(INFO) << "Enable logs system tables set to " << g_enable_logs_system_tables;
1567 
1569  throw std::runtime_error{
1570  "Invalid value provided for the \"logs-system-tables-max-files-count\" "
1571  "option. Value must be greater than 0."};
1572  }
1573  LOG(INFO) << "Maximum number of logs system table files set to "
1575 
1576 #ifdef ENABLE_MEMKIND
1577  if (g_enable_tiered_cpu_mem) {
1578  if (g_pmem_path == "") {
1579  throw std::runtime_error{"pmem-path must be set to use tiered cpu memory"};
1580  }
1581  if (g_pmem_size == 0) {
1582  throw std::runtime_error{"pmem-size must be set to use tiered cpu memory"};
1583  }
1584  if (!std::filesystem::exists(g_pmem_path.c_str())) {
1585  throw std::runtime_error{"path to PMem directory (" + g_pmem_path +
1586  ") does not exist."};
1587  }
1588  }
1589 #endif
1590 }
size_t g_pmem_size
int safe_open(const char *path, int flags, mode_t mode) noexcept
Definition: heavyai_fs.cpp:90
const std::string kDataDirectoryName
bool g_multi_instance
Definition: heavyai_locks.h:22
double g_executor_resource_mgr_per_query_max_cpu_slots_ratio
Definition: Execute.cpp:175
bool g_enable_logs_system_tables
Definition: Catalog.cpp:100
static void initialize(const std::string &data_dir, const std::string &allowed_import_paths, const std::string &allowed_export_paths)
Definition: DdlUtils.cpp:878
const std::string kDefaultDiskCacheDirName
bool g_enable_legacy_delimited_import
Definition: ParserNode.cpp:83
#define LOG(tag)
Definition: Logger.h:285
size_t g_logs_system_tables_max_files_count
bool g_enable_debug_timer
Definition: Logger.cpp:17
const std::string kDefaultLogDirName
const std::string kSystemCatalogName
void addOptionalFileToBlacklist(std::string &filename)
bool enable_non_kernel_time_query_interrupt
double g_executor_resource_mgr_cpu_result_mem_ratio
Definition: Execute.cpp:173
bool g_executor_resource_mgr_allow_cpu_gpu_kernel_concurrency
Definition: Execute.cpp:181
std::string to_string(char const *&&v)
bool g_enable_executor_resource_mgr
Definition: Execute.cpp:171
std::string ssl_trust_ca_file
ssize_t safe_write(const int fd, const void *buffer, const size_t buffer_size) noexcept
Definition: heavyai_fs.cpp:144
std::string ssl_trust_store
bool g_enable_s3_fsi
Definition: Catalog.cpp:97
static void setDefaultImportPath(const std::string &base_path)
bool g_enable_system_tables
Definition: SysCatalog.cpp:64
std::string g_pmem_path
bool g_executor_resource_mgr_allow_cpu_result_mem_oversubscription_concurrency
Definition: Execute.cpp:187
std::string allowed_export_paths
std::string ca_file_name
Definition: AuthMetadata.h:31
std::string ssl_key_file
AuthMetadata authMetadata
bool g_executor_resource_mgr_allow_cpu_slot_oversubscription_concurrency
Definition: Execute.cpp:184
bool g_read_only
Definition: heavyai_locks.h:21
bool g_executor_resource_mgr_allow_cpu_kernel_concurrency
Definition: Execute.cpp:180
unsigned pending_query_interrupt_freq
bool g_enable_fsi_regex_import
Definition: ParserNode.cpp:87
float g_vacuum_min_selectivity
const std::string kCatalogDirectoryName
bool g_enable_ml_functions
Definition: Execute.cpp:116
bool g_allow_invalid_literal_buffer_reads
Definition: ConstantIR.cpp:140
double g_executor_resource_mgr_per_query_max_cpu_result_mem_ratio
Definition: Execute.cpp:176
const std::string kDefaultLicenseFileName
double g_executor_resource_mgr_max_available_resource_use_ratio
Definition: Execute.cpp:188
std::string ssl_keystore
File_Namespace::DiskCacheConfig disk_cache_config
int32_t ftruncate(const int32_t fd, int64_t length)
Definition: heavyai_fs.cpp:86
static void addToBlacklist(const std::string &path)
Definition: DdlUtils.cpp:925
std::string allowed_import_paths
std::string disk_cache_level
po::variables_map vm
int safe_fcntl(int fd, int cmd, struct flock *fl) noexcept
Definition: heavyai_fs.cpp:112
int safe_close(int fd) noexcept
Definition: heavyai_fs.cpp:101
bool g_enable_fsi
Definition: Catalog.cpp:96
#define VLOG(n)
Definition: Logger.h:388
bool g_enable_table_functions
Definition: Execute.cpp:115
unsigned dynamic_watchdog_time_limit
SystemParameters system_parameters
std::string ssl_cert_file

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void CommandLineOptions::validate_base_path ( )

Definition at line 1199 of file CommandLineOptions.cpp.

Referenced by main().

1199  {
1200  boost::algorithm::trim_if(base_path, boost::is_any_of("\"'"));
1201  if (!boost::filesystem::exists(base_path)) {
1202  throw std::runtime_error("HeavyDB base directory does not exist at " + base_path);
1203  }
1204 }

+ Here is the caller graph for this function:

Member Data Documentation

bool CommandLineOptions::allow_loop_joins = false
bool CommandLineOptions::allow_multifrag = true
std::string CommandLineOptions::allowed_export_paths {}

Definition at line 119 of file CommandLineOptions.h.

std::string CommandLineOptions::allowed_import_paths {}

Definition at line 118 of file CommandLineOptions.h.

AuthMetadata CommandLineOptions::authMetadata
std::string CommandLineOptions::base_path
const std::string CommandLineOptions::cluster_command_line_arg {"cluster_topology"}
static

Definition at line 148 of file CommandLineOptions.h.

std::string CommandLineOptions::cluster_file = {"cluster.conf"}

Definition at line 52 of file CommandLineOptions.h.

std::string CommandLineOptions::cluster_topology_file = {"cluster_topology.conf"}

Definition at line 53 of file CommandLineOptions.h.

std::string CommandLineOptions::clusterIds_arg

Definition at line 144 of file CommandLineOptions.h.

std::string CommandLineOptions::compressor = std::string(BLOSC_LZ4HC_COMPNAME)

Definition at line 132 of file CommandLineOptions.h.

std::vector<LeafHostInfo> CommandLineOptions::db_leaves
std::string CommandLineOptions::db_query_file = {""}

path to file containing warmup queries list

Definition at line 100 of file CommandLineOptions.h.

Referenced by startHeavyDBServer().

po::options_description CommandLineOptions::developer_desc_

Definition at line 135 of file CommandLineOptions.h.

File_Namespace::DiskCacheConfig CommandLineOptions::disk_cache_config
std::string CommandLineOptions::disk_cache_level = ""

Definition at line 85 of file CommandLineOptions.h.

const bool CommandLineOptions::dist_v5_

Definition at line 156 of file CommandLineOptions.h.

unsigned CommandLineOptions::dynamic_watchdog_time_limit = 10000

Definition at line 84 of file CommandLineOptions.h.

bool CommandLineOptions::enable_auto_clear_render_mem = false
bool CommandLineOptions::enable_data_recycler = true

Definition at line 87 of file CommandLineOptions.h.

bool CommandLineOptions::enable_dynamic_watchdog = false

Definition at line 77 of file CommandLineOptions.h.

bool CommandLineOptions::enable_legacy_syntax = true
bool CommandLineOptions::enable_non_kernel_time_query_interrupt = true

Definition at line 80 of file CommandLineOptions.h.

bool CommandLineOptions::enable_rendering = false
bool CommandLineOptions::enable_runtime_query_interrupt = true

Definition at line 79 of file CommandLineOptions.h.

bool CommandLineOptions::enable_runtime_udf = true
private

Definition at line 163 of file CommandLineOptions.h.

bool CommandLineOptions::enable_runtime_udfs = true
private

Definition at line 159 of file CommandLineOptions.h.

bool CommandLineOptions::enable_udf_registration_for_all_users = false
private

Definition at line 164 of file CommandLineOptions.h.

bool CommandLineOptions::enable_watchdog = true

Definition at line 76 of file CommandLineOptions.h.

std::string CommandLineOptions::encryption_key_store_path = {}

Definition at line 55 of file CommandLineOptions.h.

std::string CommandLineOptions::exe_name

Definition at line 137 of file CommandLineOptions.h.

bool CommandLineOptions::exit_after_warmup = false

exit after warmup

Definition at line 104 of file CommandLineOptions.h.

Referenced by startHeavyDBServer().

size_t CommandLineOptions::hashtable_cache_total_bytes = 4294967296

Definition at line 89 of file CommandLineOptions.h.

po::options_description CommandLineOptions::help_desc_

Definition at line 134 of file CommandLineOptions.h.

int CommandLineOptions::http_binary_port = 6276

Definition at line 48 of file CommandLineOptions.h.

Referenced by startHeavyDBServer().

int CommandLineOptions::http_port = 6278

Definition at line 47 of file CommandLineOptions.h.

Referenced by startHeavyDBServer().

int CommandLineOptions::idle_session_duration = kMinsPerHour

Inactive session tolerance in mins (60 mins)

Definition at line 108 of file CommandLineOptions.h.

Referenced by EmbeddedDatabase::DBEngineImpl::init(), and startHeavyDBServer().

bool CommandLineOptions::intel_jit_profile = false
bool CommandLineOptions::jit_debug = false
std::string CommandLineOptions::license_path = {""}

Definition at line 54 of file CommandLineOptions.h.

logger::LogOptions CommandLineOptions::log_options_

Definition at line 136 of file CommandLineOptions.h.

bool CommandLineOptions::log_user_origin = true

Definition at line 63 of file CommandLineOptions.h.

Referenced by startHeavyDBServer().

size_t CommandLineOptions::max_cacheable_hashtable_size_bytes = 2147483648

Definition at line 90 of file CommandLineOptions.h.

size_t CommandLineOptions::max_concurrent_render_sessions = 500
int CommandLineOptions::max_session_duration = kMinsPerMonth

Maximum session life in mins (43,200 mins == 30 Days) (https://pages.nist.gov/800-63-3/sp800-63b.html#aal3reauth)

Definition at line 113 of file CommandLineOptions.h.

Referenced by EmbeddedDatabase::DBEngineImpl::init(), and startHeavyDBServer().

const std::string CommandLineOptions::nodeIds_token = {"node_id"}
static

Definition at line 148 of file CommandLineOptions.h.

size_t CommandLineOptions::num_reader_threads = 0

Number of threads used when loading data

Definition at line 96 of file CommandLineOptions.h.

Referenced by EmbeddedDatabase::DBEngineImpl::init(), and startHeavyDBServer().

bool CommandLineOptions::optimize_cuda_block_and_grid_sizes = false

Definition at line 91 of file CommandLineOptions.h.

unsigned CommandLineOptions::pending_query_interrupt_freq = 1000

Definition at line 83 of file CommandLineOptions.h.

po::positional_options_description CommandLineOptions::positional_options

Definition at line 138 of file CommandLineOptions.h.

bool CommandLineOptions::read_only = false
bool CommandLineOptions::render_compositor_use_last_gpu = true
size_t CommandLineOptions::render_mem_bytes = 1000000000
int CommandLineOptions::render_oom_retry_threshold = 0
bool CommandLineOptions::renderer_prefer_igpu = false
bool CommandLineOptions::renderer_use_parallel_executors = true

Definition at line 75 of file CommandLineOptions.h.

Referenced by startHeavyDBServer().

unsigned CommandLineOptions::renderer_vulkan_timeout_ms = 60000
size_t CommandLineOptions::reserved_gpu_mem = 384 * 1024 * 1024
double CommandLineOptions::running_query_interrupt_freq = 0.1

Definition at line 82 of file CommandLineOptions.h.

std::vector<LeafHostInfo> CommandLineOptions::string_leaves
SystemParameters CommandLineOptions::system_parameters
std::vector<std::string> CommandLineOptions::udf_compiler_options
std::string CommandLineOptions::udf_compiler_path = {""}
std::string CommandLineOptions::udf_file_name = {""}
bool CommandLineOptions::use_estimator_result_cache = true

Definition at line 81 of file CommandLineOptions.h.

bool CommandLineOptions::use_hashtable_cache = true

Definition at line 88 of file CommandLineOptions.h.

bool CommandLineOptions::verbose_logging = false

Definition at line 56 of file CommandLineOptions.h.

po::variables_map CommandLineOptions::vm

Definition at line 143 of file CommandLineOptions.h.

size_t CommandLineOptions::watchdog_none_encoded_string_translation_limit = 1000000

Definition at line 78 of file CommandLineOptions.h.


The documentation for this class was generated from the following files: