OmniSciDB  6686921089
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
anonymous_namespace{TableArchiver.cpp} Namespace Reference

Functions

std::string abs_path (const File_Namespace::GlobalFileMgr *global_file_mgr)
 
std::string run (const std::string &cmd, const std::string &chdir="")
 
std::string simple_file_cat (const std::string &archive_path, const std::string &file_name, const std::string &compression)
 
std::string get_table_schema (const std::string &archive_path, const std::string &table, const std::string &compression)
 
void adjust_altered_table_files (const std::string &temp_data_dir, const std::unordered_map< int, int > &column_ids_map)
 
void rename_table_directories (const File_Namespace::GlobalFileMgr *global_file_mgr, const std::string &temp_data_dir, const std::vector< std::string > &target_paths, const std::string &name_prefix)
 

Variables

auto simple_file_closer = [](FILE* f) { std::fclose(f); }
 

Function Documentation

std::string anonymous_namespace{TableArchiver.cpp}::abs_path ( const File_Namespace::GlobalFileMgr global_file_mgr)
inline

Definition at line 69 of file TableArchiver.cpp.

References File_Namespace::GlobalFileMgr::getBasePath().

Referenced by TableArchiver::dumpTable(), rename_table_directories(), and TableArchiver::restoreTable().

69  {
70  return boost::filesystem::canonical(global_file_mgr->getBasePath()).string();
71 }
std::string getBasePath() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void anonymous_namespace{TableArchiver.cpp}::adjust_altered_table_files ( const std::string &  temp_data_dir,
const std::unordered_map< int, int > &  column_ids_map 
)

Definition at line 175 of file TableArchiver.cpp.

References CHECK, ThreadController_NS::SimpleThreadController< FutureReturnType >::checkThreadsStatus(), cpu_threads(), omnisci::file_size(), ThreadController_NS::SimpleThreadController< FutureReturnType >::finish(), omnisci::fopen(), MAPD_FILE_EXT, simple_file_closer, split(), ThreadController_NS::SimpleThreadController< FutureReturnType >::startThread(), and to_string().

Referenced by TableArchiver::restoreTable().

176  {
177  boost::filesystem::path base_path(temp_data_dir);
178  boost::filesystem::recursive_directory_iterator end_it;
180  for (boost::filesystem::recursive_directory_iterator fit(base_path); fit != end_it;
181  ++fit) {
182  if (boost::filesystem::is_regular_file(fit->status())) {
183  const std::string file_path = fit->path().string();
184  const std::string file_name = fit->path().filename().string();
185  std::vector<std::string> tokens;
186  boost::split(tokens, file_name, boost::is_any_of("."));
187  // ref. FileMgr::init for hint of data file name layout
188  if (tokens.size() > 2 && MAPD_FILE_EXT == "." + tokens[2]) {
189  thread_controller.startThread([file_name, file_path, tokens, &column_ids_map] {
190  const auto page_size = boost::lexical_cast<int64_t>(tokens[1]);
191  const auto file_size = boost::filesystem::file_size(file_path);
192  std::unique_ptr<FILE, decltype(simple_file_closer)> fp(
193  std::fopen(file_path.c_str(), "r+"), simple_file_closer);
194  if (!fp) {
195  throw std::runtime_error("Failed to open " + file_path +
196  " for update: " + std::strerror(errno));
197  }
198  // ref. FileInfo::openExistingFile for hint of chunk header layout
199  for (size_t page = 0; page < file_size / page_size; ++page) {
200  int ints[8];
201  if (0 != std::fseek(fp.get(), page * page_size, SEEK_SET)) {
202  throw std::runtime_error("Failed to seek to page# " + std::to_string(page) +
203  file_path + " for read: " + std::strerror(errno));
204  }
205  if (1 != fread(ints, sizeof ints, 1, fp.get())) {
206  throw std::runtime_error("Failed to read " + file_path + ": " +
207  std::strerror(errno));
208  }
209  if (ints[0] > 0) { // header size
210  auto cit = column_ids_map.find(ints[3]);
211  CHECK(cit != column_ids_map.end());
212  if (ints[3] != cit->second) {
213  ints[3] = cit->second;
214  if (0 != std::fseek(fp.get(), page * page_size, SEEK_SET)) {
215  throw std::runtime_error("Failed to seek to page# " +
216  std::to_string(page) + file_path +
217  " for write: " + std::strerror(errno));
218  }
219  if (1 != fwrite(ints, sizeof ints, 1, fp.get())) {
220  throw std::runtime_error("Failed to write " + file_path + ": " +
221  std::strerror(errno));
222  }
223  }
224  }
225  }
226  });
227  thread_controller.checkThreadsStatus();
228  }
229  }
230  }
231  thread_controller.finish();
232 }
::FILE * fopen(const char *filename, const char *mode)
Definition: omnisci_fs.cpp:72
#define MAPD_FILE_EXT
Definition: File.h:25
std::string to_string(char const *&&v)
std::vector< std::string > split(std::string_view str, std::string_view delim, std::optional< size_t > maxsplit)
split apart a string into a vector of substrings
#define CHECK(condition)
Definition: Logger.h:209
int cpu_threads()
Definition: thread_count.h:24
size_t file_size(const int fd)
Definition: omnisci_fs.cpp:31

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string anonymous_namespace{TableArchiver.cpp}::get_table_schema ( const std::string &  archive_path,
const std::string &  table,
const std::string &  compression 
)
inline

Definition at line 163 of file TableArchiver.cpp.

References simple_file_cat(), and table_schema_filename.

Referenced by TableArchiver::restoreTable().

165  {
166  const auto schema_str =
167  simple_file_cat(archive_path, table_schema_filename, compression);
168  std::regex regex("@T");
169  return std::regex_replace(schema_str, regex, table);
170 }
static constexpr char const * table_schema_filename
std::string simple_file_cat(const std::string &archive_path, const std::string &file_name, const std::string &compression)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void anonymous_namespace{TableArchiver.cpp}::rename_table_directories ( const File_Namespace::GlobalFileMgr global_file_mgr,
const std::string &  temp_data_dir,
const std::vector< std::string > &  target_paths,
const std::string &  name_prefix 
)

Definition at line 234 of file TableArchiver.cpp.

References abs_path().

Referenced by TableArchiver::restoreTable().

237  {
238  boost::filesystem::path base_path(temp_data_dir);
239  boost::filesystem::directory_iterator end_it;
240  int target_path_index = 0;
241  for (boost::filesystem::directory_iterator fit(base_path); fit != end_it; ++fit) {
242  if (!boost::filesystem::is_regular_file(fit->status())) {
243  const std::string file_path = fit->path().string();
244  const std::string file_name = fit->path().filename().string();
245  if (boost::istarts_with(file_name, name_prefix)) {
246  const std::string target_path =
247  abs_path(global_file_mgr) + "/" + target_paths[target_path_index++];
248  if (std::rename(file_path.c_str(), target_path.c_str())) {
249  throw std::runtime_error("Failed to rename file " + file_path + " to " +
250  target_path + ": " + std::strerror(errno));
251  }
252  }
253  }
254  }
255 }
std::string abs_path(const File_Namespace::GlobalFileMgr *global_file_mgr)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string anonymous_namespace{TableArchiver.cpp}::run ( const std::string &  cmd,
const std::string &  chdir = "" 
)
inline

Definition at line 73 of file TableArchiver.cpp.

References logger::ERROR, measure< TimeT >::execution(), LOG, to_lower(), to_string(), and VLOG.

73  {
74  VLOG(3) << "running cmd: " << cmd;
75  int rcode;
76  std::error_code ec;
77  std::string output, errors;
78  const auto time_ms = measure<>::execution([&]() {
79  using namespace boost::process;
80  ipstream stdout, stderr;
81  if (!chdir.empty()) {
82  rcode = system(cmd, std_out > stdout, std_err > stderr, ec, start_dir = chdir);
83  } else {
84  rcode = system(cmd, std_out > stdout, std_err > stderr, ec);
85  }
86  std::ostringstream ss_output, ss_errors;
87  stdout >> ss_output.rdbuf();
88  stderr >> ss_errors.rdbuf();
89  output = ss_output.str();
90  errors = ss_errors.str();
91  });
92  if (rcode || ec) {
93  LOG(ERROR) << "failed cmd: " << cmd;
94  LOG(ERROR) << "exit code: " << rcode;
95  LOG(ERROR) << "error code: " << ec.value() << " - " << ec.message();
96  LOG(ERROR) << "stdout: " << output;
97  LOG(ERROR) << "stderr: " << errors;
98 #if defined(__APPLE__)
99  // osx bsdtar options "--use-compress-program" and "--fast-read" together
100  // run into pipe write error after tar extracts the first occurrence of a
101  // file and closes the read end while the decompression program still writes
102  // to the pipe. bsdtar doesn't handle this situation well like gnu tar does.
103  if (1 == rcode && cmd.find("--fast-read") &&
104  (errors.find("cannot write decoded block") != std::string::npos ||
105  errors.find("Broken pipe") != std::string::npos)) {
106  // ignore this error, or lose speed advantage of "--fast-read" on osx.
107  LOG(ERROR) << "tar error ignored on osx for --fast-read";
108  } else
109 #endif
110  // circumvent tar warning on reading file that is "changed as we read it".
111  // this warning results from reading a table file under concurrent inserts
112  if (1 == rcode && errors.find("changed as we read") != std::string::npos) {
113  LOG(ERROR) << "tar error ignored under concurrent inserts";
114  } else {
115  int error_code;
116  std::string error_message;
117  if (ec) {
118  error_code = ec.value();
119  error_message = ec.message();
120  } else {
121  error_code = rcode;
122  // Show a more concise message for permission errors instead of the default
123  // verbose message. Error logs will still contain all details.
124  if (to_lower(errors).find("permission denied") != std::string::npos) {
125  error_message = "Insufficient file read/write permission.";
126  } else {
127  error_message = errors;
128  }
129  }
130  throw std::runtime_error(
131  "An error occurred while executing an internal command. Error code: " +
132  std::to_string(error_code) + ", message: " + error_message);
133  }
134  } else {
135  VLOG(3) << "finished cmd: " << cmd;
136  VLOG(3) << "time: " << time_ms << " ms";
137  VLOG(3) << "stdout: " << output;
138  }
139  return output;
140 }
std::string to_lower(const std::string &str)
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
#define LOG(tag)
Definition: Logger.h:203
std::string to_string(char const *&&v)
#define VLOG(n)
Definition: Logger.h:303

+ Here is the call graph for this function:

std::string anonymous_namespace{TableArchiver.cpp}::simple_file_cat ( const std::string &  archive_path,
const std::string &  file_name,
const std::string &  compression 
)
inline

Definition at line 142 of file TableArchiver.cpp.

References get_quoted_string(), ddl_utils::IMPORT, run, and ddl_utils::validate_allowed_file_path().

Referenced by get_table_schema(), and TableArchiver::restoreTable().

144  {
147 #if defined(__APPLE__)
148  constexpr static auto opt_occurrence = "--fast-read";
149 #else
150  constexpr static auto opt_occurrence = "--occurrence=1";
151 #endif
152  boost::filesystem::path temp_dir =
153  boost::filesystem::temp_directory_path() / boost::filesystem::unique_path();
154  boost::filesystem::create_directories(temp_dir);
155  run("tar " + compression + " -xvf " + get_quoted_string(archive_path) + " " +
156  opt_occurrence + " " + file_name,
157  temp_dir.string());
158  const auto output = run("cat " + (temp_dir / file_name).string());
159  boost::filesystem::remove_all(temp_dir);
160  return output;
161 }
std::string get_quoted_string(const std::string &filename, char quote, char escape)
Quote a string while escaping any existing quotes in the string.
void validate_allowed_file_path(const std::string &file_path, const DataTransferType data_transfer_type, const bool allow_wildcards)
Definition: DdlUtils.cpp:770
static bool run

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Variable Documentation

auto anonymous_namespace{TableArchiver.cpp}::simple_file_closer = [](FILE* f) { std::fclose(f); }
inline

Definition at line 67 of file TableArchiver.cpp.

Referenced by adjust_altered_table_files(), and TableArchiver::dumpTable().