OmniSciDB  b28c0d5765
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
heavyai Namespace Reference

Classes

class  MutexInterface
 
class  SharedMutexInterface
 
class  DistributedSharedMutex
 
struct  DeviceIdentifier
 
class  JSON
 
class  InSituFlagsOwnerInterface
 
class  UUID
 

Typedefs

using DeviceGroup = std::vector< DeviceIdentifier >
 
using shared_mutex = std::shared_timed_mutex
 
template<typename T >
using lock_guard = std::lock_guard< T >
 
template<typename T >
using unique_lock = std::unique_lock< T >
 
template<typename T >
using shared_lock = std::shared_lock< T >
 

Enumerations

enum  InSituFlags { InSituFlags::kInSitu = 1u << 0, InSituFlags::kNonInSitu = 1u << 1, InSituFlags::kForcedNonInSitu = (kInSitu | kNonInSitu) }
 

Functions

size_t file_size (const int fd)
 
void * checked_mmap (const int fd, const size_t sz)
 
void checked_munmap (void *addr, size_t length)
 
int msync (void *addr, size_t length, bool async)
 
int fsync (int fd)
 
int open (const char *path, int flags, int mode)
 
void close (const int fd)
 
::FILE * fopen (const char *filename, const char *mode)
 
::FILE * popen (const char *command, const char *type)
 
int32_t pclose (::FILE *fh)
 
int get_page_size ()
 
int32_t ftruncate (const int32_t fd, int64_t length)
 
int safe_open (const char *path, int flags, mode_t mode) noexcept
 
int safe_close (int fd) noexcept
 
int safe_fcntl (int fd, int cmd, struct flock *fl) noexcept
 
ssize_t safe_read (const int fd, void *buffer, const size_t buffer_size) noexcept
 
ssize_t safe_write (const int fd, const void *buffer, const size_t buffer_size) noexcept
 
int32_t safe_ftruncate (const int32_t fd, int64_t length) noexcept
 
std::vector< std::string > glob (const std::string &pattern)
 
std::string get_hostname ()
 
std::string get_root_abs_path ()
 
bool operator== (const JSON &json1, const JSON &json2)
 
bool operator!= (const JSON &json1, const JSON &json2)
 
template<typename T >
bool operator== (const JSON &json, const T &value)
 
template<typename T >
bool operator== (const T &value, const JSON &json)
 
template<typename T >
bool operator!= (const JSON &json, const T &value)
 
template<typename T >
bool operator!= (const T &value, const JSON &json)
 
bool operator== (const UUID &lhs, const UUID &rhs) noexcept
 
bool operator!= (const UUID &lhs, const UUID &rhs) noexcept
 
bool operator< (const UUID &lhs, const UUID &rhs) noexcept
 
std::string to_string (const UUID &uuid)
 

Variables

constexpr UUID empty_uuid {}
 

Typedef Documentation

using heavyai::DeviceGroup = typedef std::vector<DeviceIdentifier>

Definition at line 31 of file DeviceGroup.h.

template<typename T >
using heavyai::lock_guard = typedef std::lock_guard<T>

Definition at line 37 of file heavyai_shared_mutex.h.

template<typename T >
using heavyai::shared_lock = typedef std::shared_lock<T>

Definition at line 41 of file heavyai_shared_mutex.h.

using heavyai::shared_mutex = typedef std::shared_timed_mutex

Definition at line 31 of file heavyai_shared_mutex.h.

template<typename T >
using heavyai::unique_lock = typedef std::unique_lock<T>

Definition at line 39 of file heavyai_shared_mutex.h.

Enumeration Type Documentation

enum heavyai::InSituFlags
strong
Enumerator
kInSitu 
kNonInSitu 
kForcedNonInSitu 

Definition at line 23 of file InSituFlags.h.

Function Documentation

void * heavyai::checked_mmap ( const int  fd,
const size_t  sz 
)

Definition at line 40 of file heavyai_fs.cpp.

References CHECK.

Referenced by StringDictionary::checkAndConditionallyIncreaseOffsetCapacity(), StringDictionary::checkAndConditionallyIncreasePayloadCapacity(), and StringDictionary::StringDictionary().

40  {
41  auto ptr = mmap(nullptr, sz, PROT_WRITE | PROT_READ, MAP_SHARED, fd, 0);
42  CHECK(ptr != reinterpret_cast<void*>(-1));
43 #ifdef __linux__
44 #ifdef MADV_HUGEPAGE
45  madvise(ptr, sz, MADV_RANDOM | MADV_WILLNEED | MADV_HUGEPAGE);
46 #else
47  madvise(ptr, sz, MADV_RANDOM | MADV_WILLNEED);
48 #endif
49 #endif
50  return ptr;
51 }
#define CHECK(condition)
Definition: Logger.h:222

+ Here is the caller graph for this function:

void heavyai::checked_munmap ( void *  addr,
size_t  length 
)

Definition at line 53 of file heavyai_fs.cpp.

References CHECK_EQ.

Referenced by StringDictionary::checkAndConditionallyIncreaseOffsetCapacity(), StringDictionary::checkAndConditionallyIncreasePayloadCapacity(), and StringDictionary::~StringDictionary().

53  {
54  CHECK_EQ(0, munmap(addr, length));
55 }
#define CHECK_EQ(x, y)
Definition: Logger.h:230

+ Here is the caller graph for this function:

void heavyai::close ( const int  fd)

Definition at line 70 of file heavyai_fs.cpp.

Referenced by import_export::DataStreamSink::import_compressed(), safe_close(), and StringDictionary::~StringDictionary().

70  {
71  ::close(fd);
72 }
void close(const int fd)
Definition: heavyai_fs.cpp:70

+ Here is the caller graph for this function:

size_t heavyai::file_size ( const int  fd)
FILE * heavyai::fopen ( const char *  filename,
const char *  mode 
)

Definition at line 74 of file heavyai_fs.cpp.

Referenced by foreign_storage::SingleTextFileReader::checkForMoreRows(), File_Namespace::create(), TableArchiver::dumpTable(), import_export::Detector::importDelimited(), import_export::Importer::importDelimited(), PosixFileArchive::init_for_read(), File_Namespace::open(), anonymous_namespace{TableArchiver.cpp}::rewrite_column_ids_in_page_headers(), and foreign_storage::SingleTextFileReader::SingleTextFileReader().

74  {
75  return ::fopen(filename, mode);
76 }
::FILE * fopen(const char *filename, const char *mode)
Definition: heavyai_fs.cpp:74

+ Here is the caller graph for this function:

int heavyai::fsync ( int  fd)

Definition at line 62 of file heavyai_fs.cpp.

Referenced by StringDictionary::checkpoint(), File_Namespace::FileInfo::syncToDisk(), File_Namespace::TableFileMgr::writeAndSyncEpochToDisk(), File_Namespace::FileMgr::writeAndSyncEpochToDisk(), and File_Namespace::FileMgr::writeAndSyncVersionToDisk().

62  {
63  return ::fsync(fd);
64 }
int fsync(int fd)
Definition: heavyai_fs.cpp:62

+ Here is the caller graph for this function:

int32_t heavyai::ftruncate ( const int32_t  fd,
int64_t  length 
)

Definition at line 86 of file heavyai_fs.cpp.

Referenced by safe_ftruncate(), and CommandLineOptions::validate().

86  {
87  return ::ftruncate(fd, length);
88 }
int32_t ftruncate(const int32_t fd, int64_t length)
Definition: heavyai_fs.cpp:86

+ Here is the caller graph for this function:

std::string heavyai::get_hostname ( )

Definition at line 23 of file heavyai_hostname.cpp.

Referenced by DBHandler::get_server_status(), and DBHandler::get_status().

23  {
24  char hostname[_POSIX_HOST_NAME_MAX];
25  gethostname(hostname, _POSIX_HOST_NAME_MAX);
26  return {hostname};
27 }

+ Here is the caller graph for this function:

int heavyai::get_page_size ( )

Definition at line 29 of file heavyai_fs.cpp.

29  {
30  return getpagesize();
31 }
std::string heavyai::get_root_abs_path ( )

Definition at line 40 of file heavyai_path.cpp.

References CHECK, CHECK_EQ, CHECK_GT, and CHECK_LT.

Referenced by Geospatial::GDAL::init(), Executor::initialize_extension_module_sources(), loadGeo(), and start_calcite_server_as_daemon().

40  {
41 #ifdef ENABLE_EMBEDDED_DATABASE
42  void* const handle = dlopen(DBEngine_LIBNAME, RTLD_LAZY | RTLD_NOLOAD);
43  if (handle) {
44  /* Non-zero handle means that libDBEngine.so has been loaded and
45  the omnisci root path will be determined with respect to the
46  location of the shared library rather than the appliction
47  `/proc/self/exe` path. */
48  const struct link_map* link_map = 0;
49  const int ret = dlinfo(handle, RTLD_DI_LINKMAP, &link_map);
50  CHECK_EQ(ret, 0);
51  CHECK(link_map);
52  /* Despite the dlinfo man page claim that l_name is absolute path,
53  it is so only when the location path to the library is absolute,
54  say, as specified in LD_LIBRARY_PATH. */
55  boost::filesystem::path abs_exe_dir(boost::filesystem::absolute(
56  boost::filesystem::canonical(std::string(link_map->l_name))));
57  abs_exe_dir.remove_filename();
58 #ifdef XCODE
59  const auto mapd_root = abs_exe_dir.parent_path().parent_path();
60 #else
61  const auto mapd_root = abs_exe_dir.parent_path();
62 #endif
63  return mapd_root.string();
64  }
65 #endif
66 #ifdef __APPLE__
67  char abs_exe_path[PROC_PIDPATHINFO_MAXSIZE] = {0};
68  auto path_len = proc_pidpath(getpid(), abs_exe_path, sizeof(abs_exe_path));
69 #else
70  char abs_exe_path[PATH_MAX] = {0};
71  auto path_len = readlink("/proc/self/exe", abs_exe_path, sizeof(abs_exe_path));
72 #endif
73  CHECK_GT(path_len, 0);
74  CHECK_LT(static_cast<size_t>(path_len), sizeof(abs_exe_path));
75  boost::filesystem::path abs_exe_dir(std::string(abs_exe_path, path_len));
76  abs_exe_dir.remove_filename();
77 #ifdef XCODE
78  const auto mapd_root = abs_exe_dir.parent_path().parent_path();
79 #else
80  const auto mapd_root = abs_exe_dir.parent_path();
81 #endif
82  return mapd_root.string();
83 }
#define CHECK_EQ(x, y)
Definition: Logger.h:230
#define CHECK_GT(x, y)
Definition: Logger.h:234
#define CHECK_LT(x, y)
Definition: Logger.h:232
#define CHECK(condition)
Definition: Logger.h:222

+ Here is the caller graph for this function:

std::vector< std::string > heavyai::glob ( const std::string &  pattern)

Definition at line 24 of file heavyai_glob.cpp.

Referenced by shared::file_or_glob_path_exists(), anonymous_namespace{heavyai_glob.cpp}::glob(), and shared::anonymous_namespace{file_path_util.cpp}::glob_local_recursive_files().

24  {
25  std::vector<std::string> results;
26  glob_t glob_result;
27  ::glob(pattern.c_str(), GLOB_BRACE | GLOB_TILDE, nullptr, &glob_result);
28  for (size_t i = 0; i < glob_result.gl_pathc; i++) {
29  results.emplace_back(glob_result.gl_pathv[i]);
30  }
31  globfree(&glob_result);
32  return results;
33 }
std::vector< std::string > glob(const std::string &pattern)

+ Here is the caller graph for this function:

int heavyai::msync ( void *  addr,
size_t  length,
bool  async 
)

Definition at line 57 of file heavyai_fs.cpp.

Referenced by StringDictionary::checkpoint().

57  {
58  // TODO: support MS_INVALIDATE?
59  return ::msync(addr, length, async ? MS_ASYNC : MS_SYNC);
60 }
future< Result > async(Fn &&fn, Args &&...args)
int msync(void *addr, size_t length, bool async)
Definition: heavyai_fs.cpp:57

+ Here is the caller graph for this function:

int heavyai::open ( const char *  path,
int  flags,
int  mode 
)
bool heavyai::operator!= ( const UUID &  lhs,
const UUID &  rhs 
)
inlinenoexcept

Definition at line 102 of file uuid.h.

102  {
103  return !(lhs == rhs);
104 }
bool heavyai::operator!= ( const JSON &  json1,
const JSON &  json2 
)
inline

Definition at line 329 of file json.h.

References heavyai::JSON::vptr_.

329  {
330  return (*json1.vptr_ != *json2.vptr_);
331 }
template<typename T >
bool heavyai::operator!= ( const JSON &  json,
const T &  value 
)
inline

Definition at line 345 of file json.h.

References heavyai::JSON::vptr_.

345  {
346  return (*json.vptr_ != value);
347 }
template<typename T >
bool heavyai::operator!= ( const T &  value,
const JSON &  json 
)
inline

Definition at line 349 of file json.h.

349  {
350  return (json != value);
351 }
bool heavyai::operator< ( const UUID &  lhs,
const UUID &  rhs 
)
inlinenoexcept

Definition at line 106 of file uuid.h.

106  {
107  return lhs.data_ < rhs.data_;
108 }
bool heavyai::operator== ( const UUID &  lhs,
const UUID &  rhs 
)
inlinenoexcept

Definition at line 98 of file uuid.h.

98  {
99  return lhs.data_ == rhs.data_;
100 }
bool heavyai::operator== ( const JSON &  json1,
const JSON &  json2 
)
inline

Definition at line 325 of file json.h.

References heavyai::JSON::vptr_.

325  {
326  return (*json1.vptr_ == *json2.vptr_);
327 }
template<typename T >
bool heavyai::operator== ( const JSON &  json,
const T &  value 
)
inline

Definition at line 336 of file json.h.

References heavyai::JSON::vptr_.

336  {
337  return (*json.vptr_ == value);
338 }
template<typename T >
bool heavyai::operator== ( const T &  value,
const JSON &  json 
)
inline

Definition at line 340 of file json.h.

340  {
341  return (json == value);
342 }
int32_t heavyai::pclose ( ::FILE *  fh)

Definition at line 82 of file heavyai_fs.cpp.

Referenced by anonymous_namespace{UdfCompiler.cpp}::exec_output().

82  {
83  return ::pclose(fh);
84 }
int32_t pclose(::FILE *fh)
Definition: heavyai_fs.cpp:82

+ Here is the caller graph for this function:

FILE * heavyai::popen ( const char *  command,
const char *  type 
)

Definition at line 78 of file heavyai_fs.cpp.

Referenced by anonymous_namespace{UdfCompiler.cpp}::exec_output().

78  {
79  return ::popen(command, type);
80 }
::FILE * popen(const char *command, const char *type)
Definition: heavyai_fs.cpp:78

+ Here is the caller graph for this function:

int heavyai::safe_close ( int  fd)
noexcept

Definition at line 101 of file heavyai_fs.cpp.

References close(), and UNREACHABLE.

Referenced by CommandLineOptions::validate().

101  {
102  for (int ret;;) {
103  ret = ::close(fd);
104  if (ret == -1 && errno == EINTR) { // interrupted by signal
105  continue;
106  }
107  return ret;
108  }
109  UNREACHABLE();
110 }
#define UNREACHABLE()
Definition: Logger.h:266
void close(const int fd)
Definition: heavyai_fs.cpp:70

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

int heavyai::safe_fcntl ( int  fd,
int  cmd,
struct flock *  fl 
)
noexcept

Definition at line 112 of file heavyai_fs.cpp.

References UNREACHABLE.

Referenced by CommandLineOptions::validate().

112  {
113  for (int ret;;) {
114  ret = ::fcntl(fd, cmd, fl);
115  if (ret == -1 && errno == EINTR) { // interrupted by signal
116  continue;
117  }
118  return ret;
119  }
120  UNREACHABLE();
121 }
#define UNREACHABLE()
Definition: Logger.h:266

+ Here is the caller graph for this function:

int32_t heavyai::safe_ftruncate ( const int32_t  fd,
int64_t  length 
)
noexcept

Definition at line 162 of file heavyai_fs.cpp.

References ftruncate(), and UNREACHABLE.

162  {
163  for (int ret;;) {
164  ret = ::ftruncate(fd, length);
165  if (ret == -1 && errno == EINTR) { // interrupted by signal
166  continue;
167  }
168  return ret;
169  }
170  UNREACHABLE();
171 }
#define UNREACHABLE()
Definition: Logger.h:266
int32_t ftruncate(const int32_t fd, int64_t length)
Definition: heavyai_fs.cpp:86

+ Here is the call graph for this function:

int heavyai::safe_open ( const char *  path,
int  flags,
mode_t  mode 
)
noexcept

Definition at line 90 of file heavyai_fs.cpp.

References open(), and UNREACHABLE.

Referenced by CommandLineOptions::validate().

90  {
91  for (int ret;;) {
92  ret = ::open(path, flags, mode);
93  if (ret == -1 && errno == EINTR) { // interrupted by signal
94  continue;
95  }
96  return ret;
97  }
98  UNREACHABLE();
99 }
#define UNREACHABLE()
Definition: Logger.h:266
int open(const char *path, int flags, int mode)
Definition: heavyai_fs.cpp:66

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

ssize_t heavyai::safe_read ( const int  fd,
void *  buffer,
const size_t  buffer_size 
)
noexcept

Definition at line 123 of file heavyai_fs.cpp.

References File_Namespace::read(), and UNREACHABLE.

123  {
124  for (ssize_t ret, sz = 0;;) {
125  ret = ::read(fd, &static_cast<char*>(buffer)[sz], buffer_size - sz);
126  if (ret == -1) {
127  if (errno == EINTR) { // interrupted by signal
128  continue;
129  }
130  return -1;
131  }
132  if (ret == 0) { // EOF
133  return sz;
134  }
135  sz += ret;
136  if (sz == static_cast<ssize_t>(buffer_size)) {
137  return sz;
138  }
139  // either an EOF is coming or interrupted by signal
140  }
141  UNREACHABLE();
142 }
#define UNREACHABLE()
Definition: Logger.h:266
size_t read(FILE *f, const size_t offset, const size_t size, int8_t *buf)
Reads the specified number of bytes from the offset position in file f into buf.
Definition: File.cpp:142

+ Here is the call graph for this function:

ssize_t heavyai::safe_write ( const int  fd,
const void *  buffer,
const size_t  buffer_size 
)
noexcept

Definition at line 144 of file heavyai_fs.cpp.

References UNREACHABLE, and File_Namespace::write().

Referenced by CommandLineOptions::validate().

144  {
145  for (ssize_t ret, sz = 0;;) {
146  ret = ::write(fd, &static_cast<char const*>(buffer)[sz], buffer_size - sz);
147  if (ret == -1) {
148  if (errno == EINTR) { // interrupted by signal
149  continue;
150  }
151  return -1;
152  }
153  sz += ret;
154  if (sz == static_cast<ssize_t>(buffer_size)) {
155  return sz;
156  }
157  // either an error is coming (such as disk full) or interrupted by signal
158  }
159  UNREACHABLE();
160 }
#define UNREACHABLE()
Definition: Logger.h:266
size_t write(FILE *f, const size_t offset, const size_t size, const int8_t *buf)
Writes the specified number of bytes to the offset position in file f from buf.
Definition: File.cpp:150

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string heavyai::to_string ( const UUID &  uuid)
inline

Definition at line 110 of file uuid.h.

110  {
111  std::stringstream ss;
112  ss << uuid;
113  return ss.str();
114 }

Variable Documentation

constexpr UUID heavyai::empty_uuid {}

Definition at line 116 of file uuid.h.