OmniSciDB  085a039ca4
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
shared Namespace Reference

Namespaces

 anonymous_namespace{file_path_util.cpp}
 

Classes

class  FileNotFoundException
 
class  NoRegexFilterMatchException
 
class  FileOrderBase
 
class  FileOrderLocal
 
struct  PrintContainer
 
struct  is_std_container
 
struct  is_std_container< std::deque< T, A > >
 
struct  is_std_container< std::list< T, A > >
 
struct  is_std_container< std::set< T, A > >
 
struct  is_std_container< std::unordered_set< T, A > >
 
struct  is_std_container< std::vector< T, A > >
 
struct  DivUMod
 

Typedefs

using LocalFileComparator = std::function< bool(const std::string &, const std::string &)>
 

Functions

std::string decode_base64_uri (const std::string &data, bool trim_nulls)
 
std::string decode_base64 (const std::string &val, bool trim_nulls)
 
std::string decode_base64 (const std::string &val)
 
static std::string encode_base64 (const std::string &val)
 
void validate_sort_options (const std::optional< std::string > &sort_by, const std::optional< std::string > &sort_regex)
 
std::vector< std::string > local_glob_filter_sort_files (const std::string &file_path, const std::optional< std::string > &filter_regex, const std::optional< std::string > &sort_by, const std::optional< std::string > &sort_regex, const bool recurse)
 
bool file_or_glob_path_exists (const std::string &path)
 
void throw_file_not_found (const std::string &file_path)
 
void throw_no_filter_match (const std::string &pattern)
 
bool is_compressed_mime_type (const std::string &mime_type)
 
bool is_compressed_file_extension (const std::string &location)
 
void execute_over_contiguous_indices (const std::vector< size_t > &indices, std::function< void(const size_t, const size_t)> to_execute)
 
bool isPowOfTwo (unsigned n)
 
unsigned getExpOfTwo (unsigned n)
 
size_t formatDate (char *buf, size_t const max, int64_t const unixtime)
 
size_t formatDateTime (char *buf, size_t const max, int64_t const timestamp, int const dimension, bool use_iso_format)
 
size_t formatHMS (char *buf, size_t const max, int64_t const unixtime)
 
std::string convert_temporal_to_iso_format (const SQLTypeInfo &type_info, int64_t unix_time)
 
template<typename K , typename V , typename comp >
V & get_from_map (std::map< K, V, comp > &map, const K &key)
 
template<typename K , typename V , typename comp >
const V & get_from_map (const std::map< K, V, comp > &map, const K &key)
 
template<typename T >
size_t append_move (std::vector< T > &destination, std::vector< T > &&source)
 
template<typename... Ts, typename T >
bool dynamic_castable_to_any (T const *ptr)
 
template<typename CONTAINER >
PrintContainer< CONTAINER > printContainer (CONTAINER &container)
 
template<typename OSTREAM , typename CONTAINER >
OSTREAM & operator<< (OSTREAM &os, PrintContainer< CONTAINER > pc)
 
DivUMod divUMod (int64_t num, int64_t den)
 
uint64_t unsignedMod (int64_t num, int64_t den)
 
template<typename T , typename U >
bool contains (const T &container, const U &element)
 
template<typename... COEFFICIENTS>
DEVICE constexpr double horner (double const x, double const c0, COEFFICIENTS...c)
 
DEVICE double fastAtanh (double const x)
 
DEVICE double fastCos (double const x)
 
DEVICE double fastCosh (double const x)
 
DEVICE double fastSin (double const x)
 
DEVICE double fastSinh (double const x)
 
template<typename T , size_t N>
constexpr std::array< T, N > powersOf (T const a)
 
template<size_t N>
constexpr std::array< double, N > inversePowersOf (double const a)
 
double power10 (unsigned const x)
 
double power10inv (unsigned const x)
 
template<typename TO , typename FROM >
TO reinterpret_bits (FROM const from)
 
template<typename... STR>
constexpr std::array
< std::string_view, sizeof...(STR)> 
string_view_array (STR &&...str)
 
template<typename OUTPUT , typename INPUT , typename FUNC >
OUTPUT transform (INPUT const &input, FUNC const &func)
 
std::shared_ptr
< apache::thrift::TConfiguration > 
default_tconfig ()
 

Variables

const std::string FILE_SORT_ORDER_BY_KEY = "FILE_SORT_ORDER_BY"
 
const std::string FILE_SORT_REGEX_KEY = "FILE_SORT_REGEX"
 
const std::string PATHNAME_ORDER_TYPE = "PATHNAME"
 
const std::string DATE_MODIFIED_ORDER_TYPE = "DATE_MODIFIED"
 
const std::string REGEX_ORDER_TYPE = "REGEX"
 
const std::string REGEX_DATE_ORDER_TYPE = "REGEX_DATE"
 
const std::string REGEX_NUMBER_ORDER_TYPE = "REGEX_NUMBER"
 
const std::array< std::string, 5 > supported_file_sort_order_types
 
const std::array< std::string, 2 > non_regex_sort_order_types
 
const std::array< std::string, 3 > regex_sort_order_types
 
const std::function< bool(const
std::string &, const
std::string &)> 
common_regex_date_comp_
 
const std::function< bool(const
std::string &, const
std::string &)> 
common_regex_number_comp_
 
const std::string kSystemCatalogName = "system_catalog"
 
const std::string kDefaultDbName = "heavyai"
 
const std::string kRootUsername = "admin"
 
const int kRootUserId = 0
 
const std::string kRootUserIdStr = "0"
 
const std::string kDefaultRootPasswd = "HyperInteractive"
 
const int32_t kTempUserIdRange = 1000000000
 
const std::string kInfoSchemaDbName = "information_schema"
 
const std::string kInfoSchemaMigrationName = "information_schema_db_created"
 
const std::string kDefaultExportDirName = "export"
 
const std::string kDefaultImportDirName = "import"
 
const std::string kDefaultDiskCacheDirName = "disk_cache"
 
const std::string kDefaultKeyFileName = "heavyai.pem"
 
const std::string kDefaultKeyStoreDirName = "key_store"
 
const std::string kDefaultLogDirName = "log"
 
const std::string kCatalogDirectoryName = "catalogs"
 
const std::string kDataDirectoryName = "data"
 
const std::string kDefaultLicenseFileName = "heavyai.license"
 

Typedef Documentation

using shared::LocalFileComparator = typedef std::function<bool(const std::string&, const std::string&)>

Definition at line 43 of file file_path_util.h.

Function Documentation

template<typename T >
size_t shared::append_move ( std::vector< T > &  destination,
std::vector< T > &&  source 
)

Definition at line 78 of file misc.h.

Referenced by RelAlgExecutor::createUnionWorkUnit().

78  {
79  if (source.empty()) {
80  return 0;
81  } else if (destination.empty()) {
82  destination = std::move(source);
83  return destination.size();
84  } else {
85  size_t const source_size = source.size();
86  destination.reserve(destination.size() + source_size);
87  std::move(std::begin(source), std::end(source), std::back_inserter(destination));
88  return source_size;
89  }
90 }

+ Here is the caller graph for this function:

template<typename T , typename U >
bool shared::contains ( const T &  container,
const U &  element 
)
inline

Definition at line 196 of file misc.h.

Referenced by foreign_storage::ParquetDataWrapper::fetchChunkMetadata(), Catalog_Namespace::Catalog::getForeignServersForUser(), TableOptimizer::getFragmentIndexes(), is_compressed_file_extension(), is_compressed_mime_type(), Grantee::reassignObjectOwners(), Catalog_Namespace::SysCatalog::reassignObjectOwners(), Catalog_Namespace::Catalog::reassignOwners(), TableOptimizer::vacuumFragments(), validate_sort_options(), foreign_storage::ForeignDataWrapperFactory::validateDataWrapperType(), and foreign_storage::AbstractFileStorageDataWrapper::validateServerOptions().

196  {
197  if (std::find(container.begin(), container.end(), element) == container.end()) {
198  return false;
199  } else {
200  return true;
201  }
202 }

+ Here is the caller graph for this function:

std::string shared::convert_temporal_to_iso_format ( const SQLTypeInfo type_info,
int64_t  unix_time 
)

Definition at line 109 of file misc.cpp.

References CHECK_EQ, CHECK_GT, formatDate(), formatDateTime(), formatHMS(), SQLTypeInfo::get_precision(), SQLTypeInfo::get_type(), kDATE, kTIME, kTIMESTAMP, SQLTypeInfo::toString(), and UNREACHABLE.

Referenced by import_export::QueryExporterCSV::exportResults(), anonymous_namespace{DBHandler.cpp}::get_refresh_info(), import_export::anonymous_namespace{QueryExporterGDAL.cpp}::insert_array_column(), import_export::anonymous_namespace{QueryExporterGDAL.cpp}::insert_scalar_column(), and import_export::anonymous_namespace{QueryExporterCSV.cpp}::target_value_to_string().

110  {
111  std::string iso_str;
112  if (type_info.get_type() == kTIME) {
113  // Set a buffer size that can contain HH:MM:SS
114  iso_str.resize(8);
115  const auto len = shared::formatHMS(iso_str.data(), iso_str.length() + 1, unix_time);
116  CHECK_EQ(len, iso_str.length());
117  } else if (type_info.get_type() == kDATE) {
118  // Set a buffer size that can contain YYYYYYYYYYYY-mm-dd (int64_t can represent up to
119  // 12 digit years)
120  iso_str.resize(18);
121  const size_t len =
122  shared::formatDate(iso_str.data(), iso_str.length() + 1, unix_time);
123  CHECK_GT(len, static_cast<size_t>(0));
124  iso_str.resize(len);
125  } else if (type_info.get_type() == kTIMESTAMP) {
126  auto precision = type_info.get_precision();
127  // Set a buffer size that can contain the specified timestamp precision
128  // YYYYYYYYYYYY-mm-dd(18) T(1) HH:MM:SS(8) .(precision?) nnnnnnnnn(precision) Z(1)
129  // (int64_t can represent up to 12 digit years with seconds precision)
130  iso_str.resize(18 + 1 + 8 + bool(precision) + precision + 1);
131  const size_t len = shared::formatDateTime(
132  iso_str.data(), iso_str.length() + 1, unix_time, precision, true);
133  CHECK_GT(len, static_cast<size_t>(0));
134  iso_str.resize(len);
135  } else {
136  UNREACHABLE() << "Unexpected column type: " << type_info.toString();
137  }
138  return iso_str;
139 }
#define CHECK_EQ(x, y)
Definition: Logger.h:231
Definition: sqltypes.h:49
#define UNREACHABLE()
Definition: Logger.h:267
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:329
#define CHECK_GT(x, y)
Definition: Logger.h:235
size_t formatHMS(char *buf, size_t const max, int64_t const unixtime)
Definition: misc.cpp:96
std::string toString() const
Definition: sqltypes.h:482
int get_precision() const
Definition: sqltypes.h:332
size_t formatDate(char *buf, size_t const max, int64_t const unixtime)
Definition: misc.cpp:27
Definition: sqltypes.h:53
size_t formatDateTime(char *buf, size_t const max, int64_t const timestamp, int const dimension, bool use_iso_format)
Definition: misc.cpp:45

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string shared::decode_base64 ( const std::string &  val,
bool  trim_nulls 
)
inline

Definition at line 27 of file base64.h.

Referenced by decode_base64(), and decode_base64_uri().

27  {
28  // For some strings, signatature particualary it is important no to trim
29  // '\0' characters
30  using namespace boost::archive::iterators;
31  using It = transform_width<binary_from_base64<std::string::const_iterator>, 8, 6>;
32 
33  if (!trim_nulls) {
34  return std::string(It(val.begin()), It(val.end()));
35  }
36  return boost::algorithm::trim_right_copy_if(
37  std::string(It(std::begin(val)), It(std::end(val))),
38  [](char c) { return c == '\0'; });
39 }

+ Here is the caller graph for this function:

std::string shared::decode_base64 ( const std::string &  val)
inline

Definition at line 41 of file base64.h.

References decode_base64().

41  {
42  return decode_base64(val, true);
43 }
std::string decode_base64(const std::string &val, bool trim_nulls)
Definition: base64.h:27

+ Here is the call graph for this function:

std::string shared::decode_base64_uri ( const std::string &  data,
bool  trim_nulls 
)

Definition at line 21 of file base64.cpp.

References decode_base64(), and transform().

21  {
22  // Allocate a string large enough to hold exta '=' as padding
23  std::string uri_dec;
24  size_t data_len = data.length();
25  // base64_uri encoding removes '=' padding at the end of the string.
26  size_t padding = 4 - (data_len % 4);
27  if (padding == 4) {
28  padding = 0;
29  }
30  uri_dec.resize(data_len + padding);
31 
32  // base64_uri encoding replaces all '+' and '/' with '-' and '_' respectively.
34  data.begin(), data.end(), uri_dec.begin(), [](unsigned char c) -> unsigned char {
35  switch (c) {
36  case '-':
37  return '+';
38  case '_':
39  return '/';
40  default:
41  return c;
42  }
43  });
44  if (padding != 0) {
45  uri_dec.replace(uri_dec.begin() + data_len, uri_dec.end(), padding, '=');
46  }
47  // in the case of a signature from a JWT trim_nulls should be false
48  return decode_base64(uri_dec, trim_nulls);
49 }
OUTPUT transform(INPUT const &input, FUNC const &func)
Definition: misc.h:297
std::string decode_base64(const std::string &val, bool trim_nulls)
Definition: base64.h:27

+ Here is the call graph for this function:

std::shared_ptr<apache::thrift::TConfiguration> shared::default_tconfig ( )
inline

Definition at line 26 of file ThriftConfig.h.

Referenced by ThriftClientConnection::open_buffered_client_transport().

26  {
27  return std::make_shared<apache::thrift::TConfiguration>(
28  std::numeric_limits<int32_t>::max());
29 }

+ Here is the caller graph for this function:

DivUMod shared::divUMod ( int64_t  num,
int64_t  den 
)
inline

Definition at line 177 of file misc.h.

References shared::DivUMod::quot.

Referenced by formatDate(), and formatDateTime().

177  {
178  DivUMod div{num / den, num % den};
179  if (div.rem < 0) {
180  --div.quot;
181  div.rem += den;
182  }
183  return div;
184 }

+ Here is the caller graph for this function:

template<typename... Ts, typename T >
bool shared::dynamic_castable_to_any ( T const *  ptr)

Definition at line 93 of file misc.h.

93  {
94  return (... || dynamic_cast<Ts const*>(ptr));
95 }
static std::string shared::encode_base64 ( const std::string &  val)
inlinestatic

Definition at line 45 of file base64.h.

45  {
46  using namespace boost::archive::iterators;
47  using It = base64_from_binary<transform_width<std::string::const_iterator, 6, 8>>;
48  auto tmp = std::string(It(std::begin(val)), It(std::end(val)));
49  return tmp.append((3 - val.size() % 3) % 3, '=');
50 }
void shared::execute_over_contiguous_indices ( const std::vector< size_t > &  indices,
std::function< void(const size_t, const size_t)>  to_execute 
)

Definition at line 22 of file Iteration.h.

Referenced by DateDaysEncoder< T, V >::appendEncodedDataAtIndices(), FixedLengthEncoder< T, V >::appendEncodedDataAtIndices(), and NoneEncoder< T >::appendEncodedDataAtIndices().

24  {
25  size_t start_pos = 0;
26 
27  while (start_pos < indices.size()) {
28  size_t end_pos = indices.size();
29  for (size_t i = start_pos + 1; i < indices.size(); ++i) {
30  if (indices[i] != indices[i - 1] + 1) {
31  end_pos = i;
32  break;
33  }
34  }
35  to_execute(start_pos, end_pos);
36  start_pos = end_pos;
37  }
38 }

+ Here is the caller graph for this function:

DEVICE double shared::fastAtanh ( double const  x)
inline

Definition at line 216 of file misc.h.

References horner().

Referenced by Transform4326ToUTM::Transform4326ToUTM().

216  {
217  // Mathematica: CoefficientList[Normal@Series[ArcTanh[x],{x,0,16}],x] // InputForm
218  return x * horner(x * x, 1, 1 / 3., 1 / 5., 1 / 7., 1 / 9., 1 / 11., 1 / 13., 1 / 15.);
219 }
DEVICE constexpr double horner(double const x, double const c0, COEFFICIENTS...c)
Definition: misc.h:206

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

DEVICE double shared::fastCos ( double const  x)
inline

Definition at line 222 of file misc.h.

References horner().

Referenced by Transform4326ToUTM::Transform4326ToUTM().

222  {
223  // Mathematica: CoefficientList[Normal@Series[Cos[x],{x,0,16}],x] // InputForm
224  // clang-format off
225  return horner(x * x, 1, -1/2., 1/24., -1/720., 1/40320., -1/3628800.,
226  1/479001600., -1/87178291200., 1/20922789888000.);
227  // clang-format on
228 }
DEVICE constexpr double horner(double const x, double const c0, COEFFICIENTS...c)
Definition: misc.h:206

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

DEVICE double shared::fastCosh ( double const  x)
inline

Definition at line 231 of file misc.h.

References horner().

Referenced by Transform4326ToUTM::calculateY(), TransformUTMTo4326::calculateY(), and TransformUTMTo4326::TransformUTMTo4326().

231  {
232  // Mathematica: CoefficientList[Normal@Series[Cosh[x],{x,0,16}],x] // InputForm
233  // clang-format off
234  return horner(x * x, 1, 1/2., 1/24., 1/720., 1/40320., 1/3628800.,
235  1/479001600., 1/87178291200., 1/20922789888000.);
236  // clang-format on
237 }
DEVICE constexpr double horner(double const x, double const c0, COEFFICIENTS...c)
Definition: misc.h:206

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

DEVICE double shared::fastSin ( double const  x)
inline

Definition at line 240 of file misc.h.

References horner().

Referenced by Transform4326ToUTM::Transform4326ToUTM().

240  {
241  // Mathematica: CoefficientList[Normal@Series[Sin[x],{x,0,16}],x] // InputForm
242  // clang-format off
243  return x * horner(x * x, 1, -1/6., 1/120., -1/5040., 1/362880.,
244  -1/39916800., 1/6227020800., -1/1307674368000.);
245  // clang-format on
246 }
DEVICE constexpr double horner(double const x, double const c0, COEFFICIENTS...c)
Definition: misc.h:206

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

DEVICE double shared::fastSinh ( double const  x)
inline

Definition at line 249 of file misc.h.

References horner().

Referenced by Transform4326ToUTM::calculateX(), TransformUTMTo4326::calculateX(), and TransformUTMTo4326::TransformUTMTo4326().

249  {
250  // Mathematica: CoefficientList[Normal@Series[Sinh[x],{x,0,16}],x] // InputForm
251  // clang-format off
252  return x * horner(x * x, 1, 1/6., 1/120., 1/5040., 1/362880.,
253  1/39916800., 1/6227020800., 1/1307674368000.);
254  // clang-format on
255 }
DEVICE constexpr double horner(double const x, double const c0, COEFFICIENTS...c)
Definition: misc.h:206

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

bool shared::file_or_glob_path_exists ( const std::string &  path)

Definition at line 174 of file file_path_util.cpp.

References heavyai::glob().

Referenced by DBHandler::detect_column_types(), and DBHandler::import_table().

174  {
175  return boost::filesystem::exists(path) || !heavyai::glob(path).empty();
176 }
std::vector< std::string > glob(const std::string &pattern)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

size_t shared::formatDate ( char *  buf,
size_t const  max,
int64_t const  unixtime 
)

Definition at line 27 of file misc.cpp.

References divUMod(), shared::DivUMod::quot, and shared::DivUMod::rem.

Referenced by convert_temporal_to_iso_format(), and DatumToString().

27  {
28  DivUMod const div_day = divUMod(unixtime, 24 * 60 * 60);
29  DivUMod const div_era = divUMod(div_day.quot - 11017, 146097);
30  unsigned const doe = static_cast<unsigned>(div_era.rem);
31  unsigned const yoe = (doe - doe / 1460 + doe / 36524 - (doe == 146096)) / 365;
32  unsigned const doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
33  unsigned const moy = (5 * doy + 2) / 153;
34  static_assert(8 <= sizeof(long long)); // long long needed for snprintf()
35  long long const y = 2000 + div_era.quot * 400 + yoe + (9 < moy);
36  unsigned const m = moy + (9 < moy ? -9 : 3);
37  unsigned const d = doy - (153 * moy + 2) / 5 + 1;
38  int const len = snprintf(buf, max, "%04lld-%02u-%02u", y, m, d);
39  if (0 <= len && static_cast<size_t>(len) < max) {
40  return static_cast<size_t>(len);
41  }
42  return 0;
43 }
DivUMod divUMod(int64_t num, int64_t den)
Definition: misc.h:177

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

size_t shared::formatDateTime ( char *  buf,
size_t const  max,
int64_t const  timestamp,
int const  dimension,
bool  use_iso_format 
)

Definition at line 45 of file misc.cpp.

References divUMod(), shared::DivUMod::quot, and shared::DivUMod::rem.

Referenced by convert_temporal_to_iso_format(), and DatumToString().

49  {
50  constexpr int pow10[10]{
51  1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000};
52  DivUMod const div_hip = divUMod(timestamp, pow10[dimension]);
53  DivUMod const div_day = divUMod(div_hip.quot, 24 * 60 * 60);
54  DivUMod const div_era = divUMod(div_day.quot - 11017, 146097);
55  unsigned const doe = static_cast<unsigned>(div_era.rem);
56  unsigned const yoe = (doe - doe / 1460 + doe / 36524 - (doe == 146096)) / 365;
57  unsigned const doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
58  unsigned const moy = (5 * doy + 2) / 153;
59  static_assert(8 <= sizeof(long long)); // long long needed for snprintf()
60  long long const y = 2000 + div_era.quot * 400 + yoe + (9 < moy);
61  unsigned const m = moy + (9 < moy ? -9 : 3);
62  unsigned const d = doy - (153 * moy + 2) / 5 + 1;
63  unsigned const minutes = static_cast<unsigned>(div_day.rem) / 60;
64  unsigned const ss = div_day.rem % 60;
65  unsigned const hh = minutes / 60;
66  unsigned const mm = minutes % 60;
67  const char* date_time_format;
68  if (use_iso_format) {
69  if (dimension) {
70  date_time_format = "%04lld-%02u-%02uT%02u:%02u:%02u";
71  } else {
72  date_time_format = "%04lld-%02u-%02uT%02u:%02u:%02uZ";
73  }
74  } else {
75  date_time_format = "%04lld-%02u-%02u %02u:%02u:%02u";
76  }
77  int const len = snprintf(buf, max, date_time_format, y, m, d, hh, mm, ss);
78  if (0 <= len && static_cast<size_t>(len) < max) {
79  if (dimension) {
80  auto precision_format = use_iso_format ? ".%0*dZ" : ".%0*d";
81  int const len_frac = snprintf(buf + len,
82  max - len,
83  precision_format,
84  dimension,
85  static_cast<int>(div_hip.rem));
86  if (0 <= len_frac && static_cast<size_t>(len + len_frac) < max) {
87  return static_cast<size_t>(len + len_frac);
88  }
89  } else {
90  return static_cast<size_t>(len);
91  }
92  }
93  return 0;
94 }
DivUMod divUMod(int64_t num, int64_t den)
Definition: misc.h:177

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

size_t shared::formatHMS ( char *  buf,
size_t const  max,
int64_t const  unixtime 
)

Definition at line 96 of file misc.cpp.

References unsignedMod().

Referenced by convert_temporal_to_iso_format(), and DatumToString().

96  {
97  unsigned const seconds = static_cast<unsigned>(unsignedMod(unixtime, 24 * 60 * 60));
98  unsigned const minutes = seconds / 60;
99  unsigned const ss = seconds % 60;
100  unsigned const hh = minutes / 60;
101  unsigned const mm = minutes % 60;
102  int const len = snprintf(buf, max, "%02u:%02u:%02u", hh, mm, ss);
103  if (0 <= len && static_cast<size_t>(len) < max) {
104  return static_cast<size_t>(len);
105  }
106  return 0;
107 }
uint64_t unsignedMod(int64_t num, int64_t den)
Definition: misc.h:187

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename K , typename V , typename comp >
V& shared::get_from_map ( std::map< K, V, comp > &  map,
const K &  key 
)
template<typename K , typename V , typename comp >
const V& shared::get_from_map ( const std::map< K, V, comp > &  map,
const K &  key 
)

Definition at line 69 of file misc.h.

References CHECK.

69  {
70  auto find_it = map.find(key);
71  CHECK(find_it != map.end());
72  return find_it->second;
73 }
#define CHECK(condition)
Definition: Logger.h:223
unsigned shared::getExpOfTwo ( unsigned  n)

Definition at line 23 of file MathUtils.cpp.

23  {
24  unsigned i = 0;
25 
26  while ((n = n >> 1)) {
27  ++i;
28  }
29 
30  return i;
31 }
constexpr double n
Definition: Utm.h:38
template<typename... COEFFICIENTS>
DEVICE constexpr double shared::horner ( double const  x,
double const  c0,
COEFFICIENTS...  c 
)

Definition at line 206 of file misc.h.

Referenced by fastAtanh(), fastCos(), fastCosh(), fastSin(), and fastSinh().

206  {
207  if constexpr (sizeof...(COEFFICIENTS) == 0) { // NOLINT
208  return c0;
209  } else {
210  return horner(x, c...) * x + c0;
211  }
212  return {}; // quiet nvcc warning https://stackoverflow.com/a/64561686/2700898
213 }
DEVICE constexpr double horner(double const x, double const c0, COEFFICIENTS...c)
Definition: misc.h:206

+ Here is the caller graph for this function:

template<size_t N>
constexpr std::array<double, N> shared::inversePowersOf ( double const  a)

Definition at line 265 of file misc.h.

References anonymous_namespace{misc.h}::inversePowersOfImpl().

265  {
266  return inversePowersOfImpl(a, std::make_index_sequence<N>{});
267 }
constexpr double a
Definition: Utm.h:32
constexpr std::array< double, sizeof...(Indices)> inversePowersOfImpl(double const a, std::index_sequence< Indices...>)
Definition: misc.h:51

+ Here is the call graph for this function:

bool shared::is_compressed_file_extension ( const std::string &  location)

Definition at line 50 of file file_type.cpp.

References contains().

Referenced by foreign_storage::LocalMultiFileReader::insertFile(), and foreign_storage::LocalMultiFileReader::LocalMultiFileReader().

50  {
51  static const std::vector<std::string> compressed_exts = {
52  ".7z", ".bz", ".bz2", ".gz", ".rar", ".tar", ".zip", ".tgz"};
53  return shared::contains(compressed_exts, boost::filesystem::extension(location));
54 }
bool contains(const T &container, const U &element)
Definition: misc.h:196

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

bool shared::is_compressed_mime_type ( const std::string &  mime_type)

Definition at line 35 of file file_type.cpp.

References contains().

35  {
36  // strips possible vender (vnd.), personal (prs.) & unregistered (x.) prefixes from the
37  // mime-type
38  static const std::regex mime_prefix("\\/(vnd|prs|x)(\\.|-)");
39  const auto mime_type_no_prefix = std::regex_replace(mime_type, mime_prefix, "/");
40  static const std::vector<std::string> compressed_types = {"application/7z-compressed",
41  "application/bzip",
42  "application/bzip2",
43  "application/gzip",
44  "application/rar",
45  "application/tar",
46  "application/zip"};
47  return shared::contains(compressed_types, mime_type_no_prefix);
48 }
bool contains(const T &container, const U &element)
Definition: misc.h:196

+ Here is the call graph for this function:

bool shared::isPowOfTwo ( unsigned  n)

Definition at line 19 of file MathUtils.cpp.

19  {
20  return (n & (n - 1)) == 0;
21 }
constexpr double n
Definition: Utm.h:38
std::vector< std::string > shared::local_glob_filter_sort_files ( const std::string &  file_path,
const std::optional< std::string > &  filter_regex,
const std::optional< std::string > &  sort_by,
const std::optional< std::string > &  sort_regex,
const bool  recurse 
)

Definition at line 109 of file file_path_util.cpp.

References shared::anonymous_namespace{file_path_util.cpp}::glob_local_recursive_files(), PATHNAME_ORDER_TYPE, and shared::anonymous_namespace{file_path_util.cpp}::regex_file_filter().

Referenced by import_export::DataStreamSink::archivePlumber(), DBHandler::detect_column_types(), ddl_utils::get_expanded_file_paths(), foreign_storage::ParquetDataWrapper::getAllFilePaths(), DBHandler::importGeoTableGlobFilterSort(), and foreign_storage::LocalMultiFileReader::LocalMultiFileReader().

114  {
115  auto result_files = glob_local_recursive_files(file_path, recurse);
116  if (filter_regex.has_value()) {
117  result_files = regex_file_filter(filter_regex.value(), result_files);
118  }
119  // initial lexicographical order ensures a determinisitc ordering for files not matching
120  // sort_regex
121  auto initial_file_order = FileOrderLocal(std::nullopt, PATHNAME_ORDER_TYPE);
122  auto lexi_comp = initial_file_order.getFileComparator();
123  std::stable_sort(result_files.begin(), result_files.end(), lexi_comp);
124 
125  auto file_order = FileOrderLocal(sort_regex, sort_by);
126  auto comp = file_order.getFileComparator();
127  std::stable_sort(result_files.begin(), result_files.end(), comp);
128  return result_files;
129 }
const std::string PATHNAME_ORDER_TYPE
std::vector< std::string > glob_local_recursive_files(const std::string &file_path, const bool recurse)
std::vector< std::string > regex_file_filter(const std::string &pattern, const std::vector< std::string > &file_paths)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename OSTREAM , typename CONTAINER >
OSTREAM& shared::operator<< ( OSTREAM &  os,
PrintContainer< CONTAINER >  pc 
)

Definition at line 126 of file misc.h.

References printContainer().

126  {
127  if (pc.container.empty()) {
128  return os << "()";
129  } else {
130  if constexpr (is_std_container<typename CONTAINER::value_type>::value) { // NOLINT
131  os << '(';
132  for (auto& container : pc.container) {
133  os << printContainer(container);
134  }
135  } else {
136  for (auto itr = pc.container.begin(); itr != pc.container.end(); ++itr) {
137  if constexpr (std::is_pointer_v<typename CONTAINER::value_type>) { // NOLINT
138  os << (itr == pc.container.begin() ? '(' : ' ') << (void const*)*itr;
139  } else {
140  os << (itr == pc.container.begin() ? '(' : ' ') << *itr;
141  }
142  }
143  }
144  return os << ')';
145  }
146 }
PrintContainer< CONTAINER > printContainer(CONTAINER &container)
Definition: misc.h:108

+ Here is the call graph for this function:

double shared::power10 ( unsigned const  x)
inline

Definition at line 270 of file misc.h.

References anonymous_namespace{Utm.h}::N.

Referenced by Analyzer::anonymous_namespace{Analyzer.cpp}::floatFromDecimal().

270  {
271  constexpr unsigned N = 20;
272  constexpr auto pow10 = powersOf<double, N>(10.0);
273  return x < N ? pow10[x] : (pow10[N - 1] * 10) * power10(x - N);
274 }
double power10(unsigned const x)
Definition: misc.h:270
constexpr unsigned N
Definition: Utm.h:110

+ Here is the caller graph for this function:

double shared::power10inv ( unsigned const  x)
inline

Definition at line 277 of file misc.h.

References anonymous_namespace{Utm.h}::N.

Referenced by CodeGenerator::codegenCastToFp().

277  {
278  constexpr unsigned N = 20;
279  constexpr auto pow10inv = inversePowersOf<N>(10.0);
280  return x < N ? pow10inv[x] : (pow10inv[N - 1] / 10) * power10inv(x - N);
281 }
double power10inv(unsigned const x)
Definition: misc.h:277
constexpr unsigned N
Definition: Utm.h:110

+ Here is the caller graph for this function:

template<typename T , size_t N>
constexpr std::array<T, N> shared::powersOf ( T const  a)

Definition at line 259 of file misc.h.

References anonymous_namespace{Utm.h}::a.

259  {
260  return powersOfImpl<T>(a, std::make_index_sequence<N>{});
261 }
constexpr double a
Definition: Utm.h:32
template<typename CONTAINER >
PrintContainer<CONTAINER> shared::printContainer ( CONTAINER &  container)
template<typename TO , typename FROM >
TO shared::reinterpret_bits ( FROM const  from)
inline

Definition at line 285 of file misc.h.

285  {
286  TO to{0};
287  memcpy(&to, &from, sizeof(TO) < sizeof(FROM) ? sizeof(TO) : sizeof(FROM));
288  return to;
289 }
template<typename... STR>
constexpr std::array<std::string_view, sizeof...(STR)> shared::string_view_array ( STR &&...  str)

Definition at line 292 of file misc.h.

292  {
293  return {std::forward<STR>(str)...};
294 }
void shared::throw_file_not_found ( const std::string &  file_path)
inline

Definition at line 79 of file file_path_util.h.

Referenced by shared::anonymous_namespace{file_path_util.cpp}::glob_local_recursive_files().

79  {
80  throw FileNotFoundException{"File or directory \"" + file_path + "\" does not exist."};
81 }

+ Here is the caller graph for this function:

void shared::throw_no_filter_match ( const std::string &  pattern)
inline

Definition at line 89 of file file_path_util.h.

Referenced by shared::anonymous_namespace{file_path_util.cpp}::regex_file_filter(), and foreign_storage::anonymous_namespace{S3FilePathUtil.cpp}::s3_objects_regex_file_filter().

89  {
90  throw NoRegexFilterMatchException{"No files matched the regex file path \"" + pattern +
91  "\"."};
92 }

+ Here is the caller graph for this function:

template<typename OUTPUT , typename INPUT , typename FUNC >
OUTPUT shared::transform ( INPUT const &  input,
FUNC const &  func 
)
uint64_t shared::unsignedMod ( int64_t  num,
int64_t  den 
)
inline

Definition at line 187 of file misc.h.

Referenced by formatHMS().

187  {
188  int64_t mod = num % den;
189  if (mod < 0) {
190  mod += den;
191  }
192  return mod;
193 }

+ Here is the caller graph for this function:

void shared::validate_sort_options ( const std::optional< std::string > &  sort_by,
const std::optional< std::string > &  sort_regex 
)

Definition at line 32 of file file_path_util.cpp.

References contains(), FILE_SORT_ORDER_BY_KEY, FILE_SORT_REGEX_KEY, join(), non_regex_sort_order_types, PATHNAME_ORDER_TYPE, regex_sort_order_types, supported_file_sort_order_types, and to_upper().

Referenced by import_export::DataStreamSink::archivePlumber(), import_export::ForeignDataImporter::importGeneralS3(), DBHandler::importGeoTableGlobFilterSort(), and foreign_storage::AbstractFileStorageDataWrapper::validateTableOptions().

33  {
34  const auto sort_by_str = to_upper(sort_by.value_or(PATHNAME_ORDER_TYPE));
35 
37  throw std::runtime_error{FILE_SORT_ORDER_BY_KEY +
38  " must be one of the following options: " +
40  }
41 
42  if (shared::contains(non_regex_sort_order_types, sort_by_str) &&
43  sort_regex.has_value()) {
44  throw std::runtime_error{"Option \"" + FILE_SORT_REGEX_KEY +
45  "\" must not be set for selected option \"" +
46  FILE_SORT_ORDER_BY_KEY + "='" + sort_by_str + "'\"."};
47  }
48 
49  if (shared::contains(regex_sort_order_types, sort_by_str) && !sort_regex.has_value()) {
50  throw std::runtime_error{"Option \"" + FILE_SORT_REGEX_KEY +
51  "\" must be set for selected option \"" +
52  FILE_SORT_ORDER_BY_KEY + "='" + sort_by_str + "'\"."};
53  }
54 }
bool contains(const T &container, const U &element)
Definition: misc.h:196
const std::array< std::string, 2 > non_regex_sort_order_types
const std::string FILE_SORT_REGEX_KEY
std::string join(T const &container, std::string const &delim)
const std::string PATHNAME_ORDER_TYPE
const std::string FILE_SORT_ORDER_BY_KEY
std::string to_upper(const std::string &str)
const std::array< std::string, 5 > supported_file_sort_order_types
const std::array< std::string, 3 > regex_sort_order_types

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Variable Documentation

const std::function<bool(const std::string&, const std::string&)> shared::common_regex_date_comp_
Initial value:
= [](const std::string& lhs, const std::string& rhs) -> bool {
int64_t lhs_t;
int64_t rhs_t;
try {
lhs_t = dateTimeParse<kDATE>(lhs, 0);
} catch (const std::exception& e) {
lhs_t = 0;
}
try {
rhs_t = dateTimeParse<kDATE>(rhs, 0);
} catch (const std::exception& e) {
rhs_t = 0;
}
return lhs_t < rhs_t;
}

Definition at line 113 of file file_path_util.h.

const std::function<bool(const std::string&, const std::string&)> shared::common_regex_number_comp_
Initial value:
=
[](const std::string& lhs, const std::string& rhs) -> bool {
int64_t lhs_i;
int64_t rhs_i;
try {
lhs_i = stoll(lhs, 0);
} catch (const std::exception& e) {
lhs_i = 0;
}
try {
rhs_i = stoll(rhs, 0);
} catch (const std::exception& e) {
rhs_i = 0;
}
return lhs_i < rhs_i;
}

Definition at line 129 of file file_path_util.h.

const std::string shared::DATE_MODIFIED_ORDER_TYPE = "DATE_MODIFIED"
inline

Definition at line 53 of file file_path_util.h.

const std::string shared::FILE_SORT_ORDER_BY_KEY = "FILE_SORT_ORDER_BY"
inline

Definition at line 49 of file file_path_util.h.

Referenced by validate_sort_options().

const std::string shared::FILE_SORT_REGEX_KEY = "FILE_SORT_REGEX"
inline

Definition at line 50 of file file_path_util.h.

Referenced by validate_sort_options().

const std::string shared::kDefaultDiskCacheDirName = "disk_cache"
inline
const std::string shared::kDefaultExportDirName = "export"
inline
const std::string shared::kDefaultKeyFileName = "heavyai.pem"
inline

Definition at line 37 of file SysDefinitions.h.

Referenced by migrations::MigrationMgr::executeRebrandMigration().

const std::string shared::kDefaultKeyStoreDirName = "key_store"
inline

Definition at line 38 of file SysDefinitions.h.

Referenced by migrations::MigrationMgr::executeRebrandMigration().

const std::string shared::kDefaultLicenseFileName = "heavyai.license"
inline
const std::string shared::kDefaultLogDirName = "log"
inline
const std::string shared::kDefaultRootPasswd = "HyperInteractive"
inline
const std::string shared::kInfoSchemaMigrationName = "information_schema_db_created"
inline
const std::string shared::kRootUserIdStr = "0"
inline

Definition at line 29 of file SysDefinitions.h.

Referenced by Catalog_Namespace::SysCatalog::initDB().

const int32_t shared::kTempUserIdRange = 1000000000
inline

Definition at line 31 of file SysDefinitions.h.

const std::array<std::string, 2> shared::non_regex_sort_order_types
inline
Initial value:
{
const std::string PATHNAME_ORDER_TYPE
const std::string DATE_MODIFIED_ORDER_TYPE

Definition at line 65 of file file_path_util.h.

Referenced by validate_sort_options().

const std::string shared::PATHNAME_ORDER_TYPE = "PATHNAME"
inline
const std::string shared::REGEX_DATE_ORDER_TYPE = "REGEX_DATE"
inline

Definition at line 55 of file file_path_util.h.

const std::string shared::REGEX_NUMBER_ORDER_TYPE = "REGEX_NUMBER"
inline

Definition at line 56 of file file_path_util.h.

const std::string shared::REGEX_ORDER_TYPE = "REGEX"
inline

Definition at line 54 of file file_path_util.h.

const std::array<std::string, 3> shared::regex_sort_order_types
inline
Initial value:
const std::string REGEX_NUMBER_ORDER_TYPE
const std::string REGEX_ORDER_TYPE
const std::string REGEX_DATE_ORDER_TYPE

Definition at line 69 of file file_path_util.h.

Referenced by validate_sort_options().

const std::array<std::string, 5> shared::supported_file_sort_order_types
inline
Initial value:
{
const std::string REGEX_NUMBER_ORDER_TYPE
const std::string REGEX_ORDER_TYPE
const std::string REGEX_DATE_ORDER_TYPE
const std::string PATHNAME_ORDER_TYPE
const std::string DATE_MODIFIED_ORDER_TYPE

Definition at line 58 of file file_path_util.h.

Referenced by validate_sort_options().