OmniSciDB  c1a53651b2
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
CommandLineOptions.h
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #pragma once
18 
19 #include <blosc.h>
20 #include <cstddef>
21 
22 #include <boost/algorithm/string.hpp>
23 #include <boost/algorithm/string/trim.hpp>
24 #include <boost/filesystem.hpp>
25 #include <boost/locale/generator.hpp>
26 #include <boost/make_shared.hpp>
27 #include <boost/program_options.hpp>
28 
29 #include "Catalog/AuthMetadata.h"
35 
36 namespace po = boost::program_options;
37 
38 class LeafHostInfo;
39 
41  public:
42  CommandLineOptions(char const* argv0, bool dist_v5_ = false)
43  : log_options_(argv0), exe_name(argv0), dist_v5_(dist_v5_) {
44  fillOptions();
46  }
47  int http_port = 6278;
48  int http_binary_port = 6276;
49  size_t reserved_gpu_mem = 384 * 1024 * 1024;
50  std::string base_path;
52  std::string cluster_file = {"cluster.conf"};
53  std::string cluster_topology_file = {"cluster_topology.conf"};
54  std::string license_path = {""};
55  std::string encryption_key_store_path = {};
56  bool verbose_logging = false;
57  bool jit_debug = false;
58  bool intel_jit_profile = false;
59  bool allow_multifrag = true;
60  bool read_only = false;
61  bool allow_loop_joins = false;
62  bool enable_legacy_syntax = true;
63  bool log_user_origin = true;
65 
67  bool enable_rendering = false;
69  int render_oom_retry_threshold = 0; // in milliseconds
70  size_t render_mem_bytes = 1000000000;
74  bool renderer_prefer_igpu = false;
75  unsigned renderer_vulkan_timeout_ms = 60000; // in milliseconds
77  bool enable_watchdog = true;
83  double running_query_interrupt_freq = 0.1; // 0.0 ~ 1.0
84  unsigned pending_query_interrupt_freq = 1000; // in milliseconds
85  unsigned dynamic_watchdog_time_limit = 10000;
86  std::string disk_cache_level = "";
87 
88  bool enable_data_recycler = true;
89  bool use_hashtable_cache = true;
90  size_t hashtable_cache_total_bytes = 4294967296; // 4GB
91  size_t max_cacheable_hashtable_size_bytes = 2147483648; // 2GB
93 
97  size_t num_reader_threads = 0;
101  std::string db_query_file = {""};
105  bool exit_after_warmup = false;
115  std::string udf_file_name = {""};
116  std::string udf_compiler_path = {""};
117  std::vector<std::string> udf_compiler_options;
118 
119  std::string allowed_import_paths{};
120  std::string allowed_export_paths{};
121 
122 #ifdef ENABLE_GEOS
123  std::string libgeos_so_filename = {"libgeos_c.so"};
124 #endif
125 
126  void fillOptions();
127  void fillAdvancedOptions();
128 
129  std::string compressor = std::string(BLOSC_LZ4HC_COMPNAME);
130 
131  po::options_description help_desc;
132  po::options_description developer_desc;
134  std::string exe_name;
135  po::positional_options_description positional_options;
136 
137  public:
138  std::vector<LeafHostInfo> db_leaves;
139  std::vector<LeafHostInfo> string_leaves;
140  po::variables_map vm;
141  std::string clusterIds_arg;
142 
143  std::string getNodeIds();
144  std::vector<std::string> getNodeIdsArray();
145  static const std::string nodeIds_token;
146 
147  boost::optional<int> parse_command_line(int argc,
148  char const* const* argv,
149  const bool should_init_logging = false);
150  void validate();
151  void validate_base_path();
152  void init_logging();
153  const bool dist_v5_;
154 
155  private:
156  bool enable_runtime_udfs = true;
157  // To store deprecated --enable-runtime-udf flag, replaced by --enable-runtime-udfs
158  // If the --enable-runtime-udf flag is specified, the contents of enable_runtime_udf
159  // are transferred to enable_runtime_udfs
160  bool enable_runtime_udf = true;
162 };
163 
164 extern bool g_enable_watchdog;
165 extern bool g_enable_dynamic_watchdog;
166 extern unsigned g_dynamic_watchdog_time_limit;
167 extern unsigned g_trivial_loop_join_threshold;
169 extern bool g_from_table_reordering;
170 extern bool g_enable_filter_push_down;
171 extern bool g_allow_cpu_retry;
172 extern bool g_allow_query_step_cpu_retry;
173 extern bool g_null_div_by_zero;
174 extern bool g_bigint_count;
176 extern float g_filter_push_down_low_frac;
177 extern float g_filter_push_down_high_frac;
179 extern bool g_enable_columnar_output;
181 extern bool g_enable_overlaps_hashjoin;
183 extern bool g_enable_distance_rangejoin;
184 extern size_t g_overlaps_max_table_size_bytes;
186 extern bool g_strip_join_covered_quals;
187 extern size_t g_constrained_by_in_threshold;
188 extern size_t g_big_group_threshold;
189 extern bool g_enable_window_functions;
193 extern bool g_enable_table_functions;
194 extern bool g_enable_dev_table_functions;
196 
197 extern size_t g_max_memory_allocation_size;
198 extern double g_bump_allocator_step_reduction;
201 extern unsigned g_pending_query_interrupt_freq;
202 extern double g_running_query_interrupt_freq;
204 extern size_t g_gpu_smem_threshold;
207 extern bool g_use_estimator_result_cache;
208 extern bool g_enable_lazy_fetch;
209 
210 extern int64_t g_omni_kafka_seek;
211 extern size_t g_leaf_count;
212 extern size_t g_compression_limit_bytes;
213 extern bool g_skip_intermediate_count;
214 extern bool g_enable_bump_allocator;
216 extern size_t g_min_memory_allocation_size;
217 extern bool g_enable_string_functions;
218 extern bool g_enable_fsi;
219 extern bool g_enable_s3_fsi;
221 #ifdef ENABLE_IMPORT_PARQUET
222 extern bool g_enable_legacy_parquet_import;
223 #endif
224 extern bool g_enable_fsi_regex_import;
226 extern bool g_enable_interop;
227 extern bool g_enable_union;
228 extern bool g_enable_cpu_sub_tasks;
229 extern size_t g_cpu_sub_task_size;
230 extern unsigned g_cpu_threads_override;
231 extern bool g_enable_filter_function;
232 extern size_t g_max_import_threads;
234 extern bool g_allow_s3_server_privileges;
235 extern float g_vacuum_min_selectivity;
236 extern bool g_read_only;
238 extern size_t g_enable_parallel_linearization;
239 extern size_t g_max_log_length;
240 #ifdef ENABLE_MEMKIND
241 extern bool g_enable_tiered_cpu_mem;
242 extern size_t g_pmem_size;
243 extern std::string g_pmem_path;
244 #endif
245 extern bool g_enable_data_recycler;
246 extern bool g_use_hashtable_cache;
247 extern size_t g_hashtable_cache_total_bytes;
249 extern bool g_use_query_resultset_cache;
252 extern bool g_use_chunk_metadata_cache;
255 extern bool g_allow_query_step_skipping;
256 extern bool g_query_engine_cuda_streams;
257 extern bool g_multi_instance;
CommandLineOptions(char const *argv0, bool dist_v5_=false)
bool g_enable_parallel_window_partition_sort
double g_running_query_interrupt_freq
Definition: Execute.cpp:129
size_t g_pmem_size
float g_filter_push_down_low_frac
Definition: Execute.cpp:96
size_t g_constrained_by_in_threshold
Definition: Execute.cpp:108
bool g_use_query_resultset_cache
Definition: Execute.cpp:148
bool g_multi_instance
Definition: heavyai_locks.h:21
double g_bump_allocator_step_reduction
Definition: Execute.cpp:121
size_t g_cpu_sub_task_size
Definition: Execute.cpp:83
logger::LogOptions log_options_
bool g_strip_join_covered_quals
Definition: Execute.cpp:107
unsigned renderer_vulkan_timeout_ms
std::vector< LeafHostInfo > string_leaves
bool g_enable_direct_columnarization
Definition: Execute.cpp:122
bool g_enable_lazy_fetch
Definition: Execute.cpp:124
std::string udf_compiler_path
bool g_enable_legacy_delimited_import
Definition: ParserNode.cpp:82
bool g_skip_intermediate_count
unsigned g_pending_query_interrupt_freq
Definition: Execute.cpp:128
po::options_description help_desc
bool g_allow_query_step_skipping
Definition: Execute.cpp:151
static constexpr int64_t kMinsPerHour
unsigned g_cpu_threads_override
bool g_enable_auto_metadata_update
size_t g_filter_push_down_passing_row_ubound
Definition: Execute.cpp:98
boost::optional< int > parse_command_line(int argc, char const *const *argv, const bool should_init_logging=false)
bool g_enable_dynamic_watchdog
Definition: Execute.cpp:80
size_t g_hashtable_cache_total_bytes
Definition: Execute.cpp:152
unsigned g_trivial_loop_join_threshold
Definition: Execute.cpp:89
bool g_enable_geo_ops_on_uncompressed_coords
Definition: Execute.cpp:114
bool g_enable_non_kernel_time_query_interrupt
Definition: Execute.cpp:126
bool enable_non_kernel_time_query_interrupt
This file includes the class specification for the cache used by the Foreign Storage Interface (FSI)...
bool g_enable_data_recycler
Definition: Execute.cpp:146
size_t g_compression_limit_bytes
Definition: Compressor.cpp:35
std::string encryption_key_store_path
bool g_enable_overlaps_hashjoin
Definition: Execute.cpp:102
bool g_inner_join_fragment_skipping
Definition: Execute.cpp:91
bool g_use_chunk_metadata_cache
Definition: Execute.cpp:149
size_t g_max_cacheable_hashtable_size_bytes
Definition: Execute.cpp:153
bool g_enable_string_functions
bool g_enable_smem_non_grouped_agg
Definition: Execute.cpp:138
static constexpr int64_t kMinsPerMonth
size_t g_lockfile_lock_extension_milliseconds
Definition: heavyai_locks.h:23
size_t g_watchdog_none_encoded_string_translation_limit
Definition: Execute.cpp:81
bool g_null_div_by_zero
Definition: Execute.cpp:88
bool g_enable_interop
bool g_enable_columnar_output
Definition: Execute.cpp:99
size_t max_concurrent_render_sessions
bool g_enable_s3_fsi
Definition: Catalog.cpp:97
bool g_from_table_reordering
Definition: Execute.cpp:90
size_t g_window_function_aggregation_tree_fanout
std::string getNodeIds()
bool g_enable_hashjoin_many_to_many
Definition: Execute.cpp:104
std::vector< LeafHostInfo > db_leaves
std::vector< std::string > getNodeIdsArray()
std::string g_pmem_path
size_t g_big_group_threshold
Definition: Execute.cpp:110
float g_filter_push_down_high_frac
Definition: Execute.cpp:97
bool g_enable_distance_rangejoin
Definition: Execute.cpp:103
bool g_bigint_count
bool g_enable_watchdog
size_t g_max_memory_allocation_size
Definition: Execute.cpp:116
double g_overlaps_target_entries_per_bin
Definition: Execute.cpp:106
bool g_allow_auto_resultset_caching
Definition: Execute.cpp:150
std::string allowed_export_paths
size_t g_max_log_length
Definition: Execute.cpp:164
size_t g_overlaps_max_table_size_bytes
Definition: Execute.cpp:105
bool g_enable_dev_table_functions
Definition: Execute.cpp:113
AuthMetadata authMetadata
bool g_optimize_cuda_block_and_grid_sizes
Definition: Execute.cpp:157
size_t g_query_resultset_cache_total_bytes
Definition: Execute.cpp:154
bool g_enable_window_functions
Definition: Execute.cpp:111
size_t g_max_cacheable_query_resultset_size_bytes
Definition: Execute.cpp:155
size_t g_min_memory_allocation_size
Definition: Execute.cpp:117
bool g_read_only
Definition: File.cpp:40
unsigned pending_query_interrupt_freq
bool g_enable_fsi_regex_import
Definition: ParserNode.cpp:86
bool g_enable_smem_grouped_non_count_agg
Definition: Execute.cpp:135
bool g_enable_automatic_ir_metadata
Definition: Execute.cpp:162
std::vector< std::string > udf_compiler_options
float g_vacuum_min_selectivity
static const std::string nodeIds_token
bool g_enable_filter_function
Definition: Execute.cpp:84
bool g_allow_invalid_literal_buffer_reads
Definition: ConstantIR.cpp:140
bool g_enable_filter_push_down
Definition: Execute.cpp:95
int64_t g_omni_kafka_seek
bool g_use_estimator_result_cache
Definition: Execute.cpp:127
bool g_enable_bump_allocator
Definition: Execute.cpp:120
bool g_enable_parallel_window_partition_compute
po::positional_options_description positional_options
bool g_enable_union
bool g_enable_cpu_sub_tasks
Definition: Execute.cpp:82
Functions used to work with HyperLogLog records.
bool g_allow_query_step_cpu_retry
Definition: Execute.cpp:87
bool g_allow_cpu_retry
Definition: Execute.cpp:86
File_Namespace::DiskCacheConfig disk_cache_config
bool enable_udf_registration_for_all_users
po::options_description developer_desc
std::string allowed_import_paths
std::string disk_cache_level
bool g_optimize_row_initialization
Definition: Execute.cpp:101
po::variables_map vm
size_t max_cacheable_hashtable_size_bytes
size_t watchdog_none_encoded_string_translation_limit
size_t g_leaf_count
Definition: ParserNode.cpp:77
unsigned g_dynamic_watchdog_time_limit
Definition: Execute.cpp:85
bool g_enable_fsi
Definition: Catalog.cpp:96
bool g_query_engine_cuda_streams
Definition: QueryEngine.h:9
bool g_allow_s3_server_privileges
Definition: S3Archive.cpp:34
bool g_enable_add_metadata_columns
Definition: ParserNode.cpp:88
bool g_enable_runtime_query_interrupt
Definition: Execute.cpp:125
size_t g_max_import_threads
Definition: Importer.cpp:106
size_t g_enable_parallel_linearization
Definition: Execute.cpp:144
std::string cluster_topology_file
bool g_use_hashtable_cache
Definition: Execute.cpp:147
size_t g_auto_resultset_caching_threshold
Definition: Execute.cpp:156
bool g_enable_table_functions
Definition: Execute.cpp:112
unsigned dynamic_watchdog_time_limit
SystemParameters system_parameters
size_t g_gpu_smem_threshold
Definition: Execute.cpp:130