OmniSciDB  72c90bc290
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
TableArchiver Class Reference

#include <TableArchiver.h>

+ Collaboration diagram for TableArchiver:

Public Member Functions

 TableArchiver (Catalog_Namespace::Catalog *cat)
 
void dumpTable (const TableDescriptor *td, const std::string &archive_path, const std::string &compression)
 
void restoreTable (const Catalog_Namespace::SessionInfo &session, const std::string &table_name, const std::string &archive_path, const std::string &compression, const TableArchiverS3Options &s3_options)
 

Private Member Functions

void restoreTable (const Catalog_Namespace::SessionInfo &session, const TableDescriptor *td, const std::string &archive_path, const std::string &compression)
 

Private Attributes

Catalog_Namespace::Catalogcat_
 

Detailed Description

Definition at line 32 of file TableArchiver.h.

Constructor & Destructor Documentation

TableArchiver::TableArchiver ( Catalog_Namespace::Catalog cat)
inline

Definition at line 34 of file TableArchiver.h.

34 : cat_(cat){};
Catalog_Namespace::Catalog * cat_
Definition: TableArchiver.h:52

Member Function Documentation

void TableArchiver::dumpTable ( const TableDescriptor td,
const std::string &  archive_path,
const std::string &  compression 
)

Definition at line 439 of file TableArchiver.cpp.

References anonymous_namespace{TableArchiver.cpp}::abs_path(), cat_, Catalog_Namespace::DBMetadata::dbId, Data_Namespace::DISK_LEVEL, Catalog_Namespace::Catalog::dumpSchema(), ddl_utils::EXPORT, nvtx_helpers::anonymous_namespace{nvtx_helpers.cpp}::filename(), heavyai::fopen(), g_cluster, get_quoted_string(), Catalog_Namespace::Catalog::getAllColumnMetadataForTable(), Catalog_Namespace::Catalog::getColumnDictDirectory(), Catalog_Namespace::Catalog::getCurrentDB(), Catalog_Namespace::Catalog::getDataMgr(), Data_Namespace::DataMgr::getGlobalFileMgr(), Catalog_Namespace::Catalog::getTableDataDirectories(), Catalog_Namespace::Catalog::getTableDictDirectories(), Catalog_Namespace::Catalog::getTableEpoch(), TableDescriptor::is_system_table, TableDescriptor::isForeignTable(), TableDescriptor::isView, join(), kDumpVersion, TableDescriptor::persistenceLevel, run, anonymous_namespace{TableArchiver.cpp}::simple_file_closer, table_dumpversion_filename, table_epoch_filename, table_oldinfo_filename, table_schema_filename, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), shared::transform(), and ddl_utils::validate_allowed_file_path().

Referenced by Parser::DumpTableStmt::execute().

441  {
442  if (td->is_system_table) {
443  throw std::runtime_error("Dumping a system table is not supported.");
444  }
445  if (td->isForeignTable()) {
446  throw std::runtime_error("Dumping a foreign table is not supported.");
447  }
450  if (g_cluster) {
451  throw std::runtime_error("DUMP/RESTORE is not supported yet on distributed setup.");
452  }
453  if (boost::filesystem::exists(archive_path)) {
454  throw std::runtime_error("Archive " + archive_path + " already exists.");
455  }
457  throw std::runtime_error("Dumping view or temporary table is not supported.");
458  }
459  // create a unique uuid for this table dump
460  std::string uuid = boost::uuids::to_string(boost::uuids::random_generator()());
461 
462  // collect paths of files to archive
463  const auto global_file_mgr = cat_->getDataMgr().getGlobalFileMgr();
464  std::vector<std::string> file_paths;
465  auto file_writer = [&file_paths, uuid](const std::string& file_name,
466  const std::string& file_type,
467  const std::string& file_data) {
468  std::unique_ptr<FILE, decltype(simple_file_closer)> fp(
469  std::fopen(file_name.c_str(), "w"), simple_file_closer);
470  if (!fp) {
471  throw std::runtime_error("Failed to create " + file_type + " file '" + file_name +
472  "': " + std::strerror(errno));
473  }
474  if (std::fwrite(file_data.data(), 1, file_data.size(), fp.get()) < file_data.size()) {
475  throw std::runtime_error("Failed to write " + file_type + " file '" + file_name +
476  "': " + std::strerror(errno));
477  }
478  file_paths.push_back(uuid / std::filesystem::path(file_name).filename());
479  };
480 
481  const auto file_mgr_dir = std::filesystem::path(abs_path(global_file_mgr));
482  const auto uuid_dir = file_mgr_dir / uuid;
483 
484  if (!std::filesystem::create_directory(uuid_dir)) {
485  throw std::runtime_error("Failed to create work directory '" + uuid_dir.string() +
486  "' while dumping table.");
487  }
488 
489  ScopeGuard cleanup_guard = [&] {
490  if (std::filesystem::exists(uuid_dir)) {
491  std::filesystem::remove_all(uuid_dir);
492  }
493  };
494 
495  const auto table_name = td->tableName;
496  {
497  // - gen dumpversion file
498  const auto dumpversion_str = std::to_string(kDumpVersion);
499  file_writer(
500  uuid_dir / table_dumpversion_filename, "table dumpversion", dumpversion_str);
501  // - gen schema file
502  const auto schema_str = cat_->dumpSchema(td);
503  file_writer(uuid_dir / table_schema_filename, "table schema", schema_str);
504  // - gen column-old-info file
505  const auto cds = cat_->getAllColumnMetadataForTable(td->tableId, true, true, true);
506  std::vector<std::string> column_oldinfo;
507  std::transform(cds.begin(),
508  cds.end(),
509  std::back_inserter(column_oldinfo),
510  [&](const auto cd) -> std::string {
511  return cd->columnName + ":" + std::to_string(cd->columnId) + ":" +
513  });
514  const auto column_oldinfo_str = boost::algorithm::join(column_oldinfo, " ");
515  file_writer(uuid_dir / table_oldinfo_filename, "table old info", column_oldinfo_str);
516  // - gen table epoch
517  const auto epoch = cat_->getTableEpoch(cat_->getCurrentDB().dbId, td->tableId);
518  file_writer(uuid_dir / table_epoch_filename, "table epoch", std::to_string(epoch));
519  // - collect table data file paths ...
520  const auto data_file_dirs = cat_->getTableDataDirectories(td);
521  file_paths.insert(file_paths.end(), data_file_dirs.begin(), data_file_dirs.end());
522  // - collect table dict file paths ...
523  const auto dict_file_dirs = cat_->getTableDictDirectories(td);
524  file_paths.insert(file_paths.end(), dict_file_dirs.begin(), dict_file_dirs.end());
525  // tar takes time. release cat lock to yield the cat to concurrent CREATE statements.
526  }
527  // run tar to archive the files ... this may take a while !!
528  run("tar " + compression + " --transform=s|" + uuid +
529  std::filesystem::path::preferred_separator + "|| -cvf " +
530  get_quoted_string(archive_path) + " " + boost::algorithm::join(file_paths, " "),
531  file_mgr_dir);
532 }
std::string getColumnDictDirectory(const ColumnDescriptor *cd, bool file_name_only=true) const
Definition: Catalog.cpp:5388
static constexpr char const * table_schema_filename
std::string tableName
std::string abs_path(const File_Namespace::GlobalFileMgr *global_file_mgr)
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:266
std::string join(T const &container, std::string const &delim)
static constexpr char const * table_oldinfo_filename
int32_t getTableEpoch(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:3579
std::vector< std::string > getTableDataDirectories(const TableDescriptor *td) const
Definition: Catalog.cpp:5374
bool isForeignTable() const
std::string to_string(char const *&&v)
std::string get_quoted_string(const std::string &filename, char quote, char escape)
Quote a string while escaping any existing quotes in the string.
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:265
::FILE * fopen(const char *filename, const char *mode)
Definition: heavyai_fs.cpp:74
OUTPUT transform(INPUT const &input, FUNC const &func)
Definition: misc.h:320
File_Namespace::GlobalFileMgr * getGlobalFileMgr() const
Definition: DataMgr.cpp:649
void validate_allowed_file_path(const std::string &file_path, const DataTransferType data_transfer_type, const bool allow_wildcards)
Definition: DdlUtils.cpp:822
std::vector< std::string > getTableDictDirectories(const TableDescriptor *td) const
Definition: Catalog.cpp:5409
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:2172
Data_Namespace::MemoryLevel persistenceLevel
static constexpr char const * table_epoch_filename
bool g_cluster
static constexpr int kDumpVersion
static bool run
static constexpr char const * table_dumpversion_filename
std::string dumpSchema(const TableDescriptor *td) const
Definition: Catalog.cpp:5440
Catalog_Namespace::Catalog * cat_
Definition: TableArchiver.h:52

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void TableArchiver::restoreTable ( const Catalog_Namespace::SessionInfo session,
const std::string &  table_name,
const std::string &  archive_path,
const std::string &  compression,
const TableArchiverS3Options s3_options 
)

Definition at line 821 of file TableArchiver.cpp.

References cat_, CHECK, Parser::create_stmt_for_query(), anonymous_namespace{TableArchiver.cpp}::get_table_schema(), Catalog_Namespace::Catalog::getMetadataForTable(), and shared::is_s3_uri().

Referenced by Parser::RestoreTableStmt::execute().

825  {
826  auto local_archive_path = archive_path;
827 #ifdef HAVE_AWS_S3
828  const auto restore_dir_path = get_restore_dir_path();
829  ScopeGuard archive_cleanup_guard = [&archive_path, &restore_dir_path] {
830  if (shared::is_s3_uri(archive_path) && std::filesystem::exists(restore_dir_path)) {
831  std::filesystem::remove_all(restore_dir_path);
832  }
833  };
834  if (shared::is_s3_uri(archive_path)) {
835  local_archive_path = download_s3_file(archive_path, s3_options, restore_dir_path);
836  }
837 #endif
838 
839  // replace table name and drop foreign dict references
840  const auto schema_str = get_table_schema(local_archive_path, table_name, compression);
841  std::unique_ptr<Parser::Stmt> stmt = Parser::create_stmt_for_query(schema_str, session);
842  const auto create_table_stmt = dynamic_cast<Parser::CreateTableStmt*>(stmt.get());
843  CHECK(create_table_stmt);
844  create_table_stmt->execute(session, false /*read-only*/);
845 
846  try {
847  restoreTable(
848  session, cat_->getMetadataForTable(table_name), local_archive_path, compression);
849  } catch (...) {
850  const auto schema_str = "DROP TABLE IF EXISTS " + table_name + ";";
851  std::unique_ptr<Parser::Stmt> stmt =
852  Parser::create_stmt_for_query(schema_str, session);
853  const auto drop_table_stmt = dynamic_cast<Parser::DropTableStmt*>(stmt.get());
854  CHECK(drop_table_stmt);
855  drop_table_stmt->execute(session, false /*read-only*/);
856 
857  throw;
858  }
859 }
std::string get_table_schema(const std::string &archive_path, const std::string &table, const std::string &compression)
bool is_s3_uri(const std::string &file_path)
#define CHECK(condition)
Definition: Logger.h:291
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
void restoreTable(const Catalog_Namespace::SessionInfo &session, const std::string &table_name, const std::string &archive_path, const std::string &compression, const TableArchiverS3Options &s3_options)
std::unique_ptr< Parser::Stmt > create_stmt_for_query(const std::string &queryStr, const Catalog_Namespace::SessionInfo &session_info)
Catalog_Namespace::Catalog * cat_
Definition: TableArchiver.h:52

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void TableArchiver::restoreTable ( const Catalog_Namespace::SessionInfo session,
const TableDescriptor td,
const std::string &  archive_path,
const std::string &  compression 
)
private

Definition at line 535 of file TableArchiver.cpp.

References anonymous_namespace{Utm.h}::a, anonymous_namespace{TableArchiver.cpp}::abs_path(), cat_, CHECK, Parser::create_stmt_for_query(), Catalog_Namespace::DBMetadata::dbId, Data_Namespace::DISK_LEVEL, anonymous_namespace{TableArchiver.cpp}::drop_render_group_columns(), measure< TimeT >::execution(), anonymous_namespace{TableArchiver.cpp}::find_render_group_columns(), g_cluster, g_test_rollback_dump_restore, get_quoted_string(), anonymous_namespace{TableArchiver.cpp}::get_table_schema(), Catalog_Namespace::Catalog::getAllColumnMetadataForTable(), Catalog_Namespace::Catalog::getColumnDictDirectory(), Catalog_Namespace::Catalog::getCurrentDB(), Catalog_Namespace::Catalog::getDataMgr(), Data_Namespace::DataMgr::getGlobalFileMgr(), lockmgr::TableLockMgrImpl< TableSchemaLockMgr >::getReadLockForTable(), Catalog_Namespace::Catalog::getTableDataDirectories(), Catalog_Namespace::Catalog::getTableDictDirectories(), lockmgr::TableLockMgrImpl< T >::getWriteLockForTable(), TableDescriptor::hasDeletedCol, ddl_utils::IMPORT, logger::INFO, TableDescriptor::isView, join(), kDumpVersion_remove_render_group_columns, LOG, TableDescriptor::nShards, TableDescriptor::persistenceLevel, anonymous_namespace{TableArchiver.cpp}::rename_table_directories(), run, Catalog_Namespace::Catalog::setTableEpoch(), anonymous_namespace{TableArchiver.cpp}::simple_file_cat(), split(), table_dumpversion_filename, table_epoch_filename, table_oldinfo_filename, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), shared::transform(), anonymous_namespace{TableArchiver.cpp}::update_or_drop_column_ids_in_table_files(), ddl_utils::validate_allowed_file_path(), and VLOG.

538  {
541  if (g_cluster) {
542  throw std::runtime_error("DUMP/RESTORE is not supported yet on distributed setup.");
543  }
544  if (!boost::filesystem::exists(archive_path)) {
545  throw std::runtime_error("Archive " + archive_path + " does not exist.");
546  }
548  throw std::runtime_error("Restoring view or temporary table is not supported.");
549  }
550  // Obtain table schema read lock to prevent modification of the schema during
551  // restoration
552  const auto table_read_lock =
554  // prevent concurrent inserts into table during restoration
555  const auto insert_data_lock =
557 
558  // untar takes time. no grab of cat lock to yield to concurrent CREATE stmts.
559  const auto global_file_mgr = cat_->getDataMgr().getGlobalFileMgr();
560 
561  // create a unique uuid for this table restore
562  std::string uuid = boost::uuids::to_string(boost::uuids::random_generator()());
563 
564  const auto uuid_dir = std::filesystem::path(abs_path(global_file_mgr)) / uuid;
565 
566  if (!std::filesystem::create_directory(uuid_dir)) {
567  throw std::runtime_error("Failed to create work directory '" + uuid_dir.string() +
568  "' while restoring table.");
569  }
570 
571  ScopeGuard cleanup_guard = [&] {
572  if (std::filesystem::exists(uuid_dir)) {
573  std::filesystem::remove_all(uuid_dir);
574  }
575  };
576 
577  // dirs where src files are untarred and dst files are backed up
578  constexpr static const auto temp_data_basename = "_data";
579  constexpr static const auto temp_back_basename = "_back";
580  const auto temp_data_dir = uuid_dir / temp_data_basename;
581  const auto temp_back_dir = uuid_dir / temp_back_basename;
582 
583  // extract & parse schema
584  const auto schema_str = get_table_schema(archive_path, td->tableName, compression);
585  std::unique_ptr<Parser::Stmt> stmt = Parser::create_stmt_for_query(schema_str, session);
586  const auto create_table_stmt = dynamic_cast<Parser::CreateTableStmt*>(stmt.get());
587  CHECK(create_table_stmt);
588 
589  // verify compatibility between source and destination schemas
590  TableDescriptor src_td;
591  std::list<ColumnDescriptor> src_columns;
592  std::vector<Parser::SharedDictionaryDef> shared_dict_defs;
593  create_table_stmt->executeDryRun(session, src_td, src_columns, shared_dict_defs);
594  // - sanity check table-level compatibility
595  if (src_td.hasDeletedCol != td->hasDeletedCol) {
596  // TODO: allow the case, in which src data enables vacuum while
597  // dst doesn't, by simply discarding src $deleted column data.
598  throw std::runtime_error("Incompatible table VACCUM option");
599  }
600  if (src_td.nShards != td->nShards) {
601  // TODO: allow different shard numbers if they have a "GCD",
602  // by splitting/merging src data files before drop into dst.
603  throw std::runtime_error("Unmatched number of table shards");
604  }
605  // - sanity check column-level compatibility (based on column names)
606  const auto dst_columns =
607  cat_->getAllColumnMetadataForTable(td->tableId, false, false, false);
608  if (dst_columns.size() != src_columns.size()) {
609  throw std::runtime_error("Unmatched number of table columns");
610  }
611  for (const auto& [src_cd, dst_cd] : boost::combine(src_columns, dst_columns)) {
612  if (src_cd.columnType.get_type_name() != dst_cd->columnType.get_type_name() ||
613  src_cd.columnType.get_compression_name() !=
614  dst_cd->columnType.get_compression_name()) {
615  throw std::runtime_error("Incompatible types on column " + src_cd.columnName);
616  }
617  }
618  // extract src table column ids (ALL columns incl. system/virtual/phy geo cols)
619  const auto all_src_oldinfo_str =
620  simple_file_cat(archive_path, table_oldinfo_filename, compression);
621  std::vector<std::string> src_oldinfo_strs;
622  boost::algorithm::split(src_oldinfo_strs,
623  all_src_oldinfo_str,
624  boost::is_any_of(" "),
625  boost::token_compress_on);
626 
627  // fetch dump version
628  int dump_version = -1;
629  try {
630  // attempt to read file, do not log if fail to read
631  auto const dump_version_str =
632  simple_file_cat(archive_path, table_dumpversion_filename, compression, false);
633  dump_version = std::stoi(dump_version_str);
634  } catch (std::runtime_error& e) {
635  // no dump version file found
636  dump_version = 0;
637  }
638  LOG(INFO) << "Dump Version: " << dump_version;
639 
640  // version-specific behavior
641  const bool do_drop_render_group_columns =
643 
644  // remove any render group columns from the source columns so that the list of
645  // source columns matches the already-created table, and the removed ones will
646  // not have an entry in column_ids_map, and hence will not have their data
647  // mapped later (effectively dropping them), and return their IDs for when
648  // they are actually dropped later
649  std::unordered_map<int, int> render_group_column_ids;
650  if (do_drop_render_group_columns) {
651  render_group_column_ids =
652  find_render_group_columns(src_columns, src_oldinfo_strs, archive_path);
653  }
654 
655  // compare with the destination columns
656  auto all_dst_columns =
657  cat_->getAllColumnMetadataForTable(td->tableId, true, true, true);
658  if (src_oldinfo_strs.size() != all_dst_columns.size()) {
659  throw std::runtime_error("Source table has a unmatched number of columns: " +
660  std::to_string(src_oldinfo_strs.size()) + " vs " +
661  std::to_string(all_dst_columns.size()));
662  }
663  // build a map of src column ids and dst column ids, just in case src table has been
664  // ALTERed before and chunk keys of src table needs to be adjusted accordingly.
665  // note: this map is used only for the case of migrating a table and not for restoring
666  // a table. When restoring a table, the two tables must have the same column ids.
667  //
668  // also build a map of src dict paths and dst dict paths for relocating src dicts
669  std::unordered_map<int, int> column_ids_map;
670  std::unordered_map<std::string, std::string> dict_paths_map;
671  // sort inputs of transform in lexical order of column names for correct mappings
672  std::list<std::vector<std::string>> src_oldinfo_tokens;
674  src_oldinfo_strs.begin(),
675  src_oldinfo_strs.end(),
676  std::back_inserter(src_oldinfo_tokens),
677  [](const auto& src_oldinfo_str) -> auto{
678  std::vector<std::string> tokens;
680  tokens, src_oldinfo_str, boost::is_any_of(":"), boost::token_compress_on);
681  return tokens;
682  });
683  src_oldinfo_tokens.sort(
684  [](const auto& lhs, const auto& rhs) { return lhs[0].compare(rhs[0]) < 0; });
685  all_dst_columns.sort(
686  [](auto a, auto b) { return a->columnName.compare(b->columnName) < 0; });
687  // transform inputs into the maps
688  std::transform(src_oldinfo_tokens.begin(),
689  src_oldinfo_tokens.end(),
690  all_dst_columns.begin(),
691  std::inserter(column_ids_map, column_ids_map.end()),
692  [&](const auto& tokens, const auto& cd) -> std::pair<int, int> {
693  VLOG(3) << boost::algorithm::join(tokens, ":") << " ==> "
694  << cd->columnName << ":" << cd->columnId;
695  dict_paths_map[tokens[2]] = cat_->getColumnDictDirectory(cd);
696  return {boost::lexical_cast<int>(tokens[1]), cd->columnId};
697  });
698  bool was_table_altered = false;
699  std::for_each(column_ids_map.begin(), column_ids_map.end(), [&](auto& it) {
700  was_table_altered = was_table_altered || it.first != it.second;
701  });
702  VLOG(3) << "was_table_altered = " << was_table_altered;
703 
704  // extract all data files to a temp dir. will swap with dst table dir after all set,
705  // otherwise will corrupt table in case any bad thing happens in the middle.
706  run("rm -rf " + temp_data_dir.string());
707  run("mkdir -p " + temp_data_dir.string());
708  run("tar " + compression + " -xvf " + get_quoted_string(archive_path), temp_data_dir);
709 
710  // drop the render group columns here
711  if (do_drop_render_group_columns) {
713  render_group_column_ids, archive_path, temp_data_dir, compression);
714  }
715 
716  // if table was ever altered after it was created, update column ids in chunk headers.
717  if (was_table_altered) {
718  const auto epoch = boost::lexical_cast<int32_t>(
719  simple_file_cat(archive_path, table_epoch_filename, compression));
720  const auto time_ms = measure<>::execution([&]() {
722  epoch, temp_data_dir, column_ids_map, false /* update */);
723  });
724  VLOG(3) << "update_column_ids_table_files: " << time_ms << " ms";
725  }
726 
727  // finally,,, swap table data/dict dirs!
728  const auto data_file_dirs = cat_->getTableDataDirectories(td);
729  const auto dict_file_dirs = cat_->getTableDictDirectories(td);
730  // move current target dirs, if exists, to backup dir
731  std::vector<std::string> both_file_dirs;
732  std::merge(data_file_dirs.begin(),
733  data_file_dirs.end(),
734  dict_file_dirs.begin(),
735  dict_file_dirs.end(),
736  std::back_inserter(both_file_dirs));
737  bool backup_completed = false;
738  try {
739  run("rm -rf " + temp_back_dir.string());
740  run("mkdir -p " + temp_back_dir.string());
741  for (const auto& dir : both_file_dirs) {
742  const auto dir_full_path = abs_path(global_file_mgr) + "/" + dir;
743  if (boost::filesystem::is_directory(dir_full_path)) {
744  run("mv " + dir_full_path + " " + temp_back_dir.string());
745  }
746  }
747  backup_completed = true;
748  // Move table directories from temp dir to main data directory.
749  rename_table_directories(global_file_mgr, temp_data_dir, data_file_dirs, "table_");
750  // Move dictionaries from temp dir to main dir.
751  for (const auto& dit : dict_paths_map) {
752  if (!dit.first.empty() && !dit.second.empty()) {
753  const auto src_dict_path = temp_data_dir.string() + "/" + dit.first;
754  const auto dst_dict_path = abs_path(global_file_mgr) + "/" + dit.second;
755  run("mv " + src_dict_path + " " + dst_dict_path);
756  }
757  }
758  // throw if sanity test forces a rollback
760  throw std::runtime_error("lol!");
761  }
762  } catch (...) {
763  // once backup is completed, whatever in abs_path(global_file_mgr) is the "src"
764  // dirs that are to be rolled back and discarded
765  if (backup_completed) {
766  run("rm -rf " + boost::algorithm::join(both_file_dirs, " "),
767  abs_path(global_file_mgr));
768  }
769  // complete rollback by recovering original "dst" table dirs from backup dir
770  boost::filesystem::path base_path(temp_back_dir);
771  boost::filesystem::directory_iterator end_it;
772  for (boost::filesystem::directory_iterator fit(base_path); fit != end_it; ++fit) {
773  run("mv " + fit->path().string() + " .", abs_path(global_file_mgr));
774  }
775  throw;
776  }
777  // set for reloading table from the restored/migrated files
778  const auto epoch = simple_file_cat(archive_path, table_epoch_filename, compression);
780  cat_->getCurrentDB().dbId, td->tableId, boost::lexical_cast<int>(epoch));
781 }
std::string get_table_schema(const std::string &archive_path, const std::string &table, const std::string &compression)
std::string getColumnDictDirectory(const ColumnDescriptor *cd, bool file_name_only=true) const
Definition: Catalog.cpp:5388
std::string tableName
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
std::string abs_path(const File_Namespace::GlobalFileMgr *global_file_mgr)
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:266
std::unordered_map< int, int > find_render_group_columns(const std::list< ColumnDescriptor > &src_columns, std::vector< std::string > &src_oldinfo_strs, const std::string &archive_path)
#define LOG(tag)
Definition: Logger.h:285
static WriteLock getWriteLockForTable(const Catalog_Namespace::Catalog &cat, const std::string &table_name)
Definition: LockMgrImpl.h:225
std::string join(T const &container, std::string const &delim)
static constexpr char const * table_oldinfo_filename
void drop_render_group_columns(const std::unordered_map< int, int > &render_group_column_ids, const std::string &archive_path, const std::string &temp_data_dir, const std::string &compression)
std::vector< std::string > getTableDataDirectories(const TableDescriptor *td) const
Definition: Catalog.cpp:5374
std::string to_string(char const *&&v)
std::vector< std::string > split(std::string_view str, std::string_view delim, std::optional< size_t > maxsplit)
split apart a string into a vector of substrings
constexpr double a
Definition: Utm.h:32
std::string get_quoted_string(const std::string &filename, char quote, char escape)
Quote a string while escaping any existing quotes in the string.
static constexpr int kDumpVersion_remove_render_group_columns
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:265
void rename_table_directories(const File_Namespace::GlobalFileMgr *global_file_mgr, const std::string &temp_data_dir, const std::vector< std::string > &target_paths, const std::string &name_prefix)
OUTPUT transform(INPUT const &input, FUNC const &func)
Definition: misc.h:320
File_Namespace::GlobalFileMgr * getGlobalFileMgr() const
Definition: DataMgr.cpp:649
void validate_allowed_file_path(const std::string &file_path, const DataTransferType data_transfer_type, const bool allow_wildcards)
Definition: DdlUtils.cpp:822
std::string simple_file_cat(const std::string &archive_path, const std::string &file_name, const std::string &compression, const bool log_failure=true)
void setTableEpoch(const int db_id, const int table_id, const int new_epoch)
Definition: Catalog.cpp:3647
std::vector< std::string > getTableDictDirectories(const TableDescriptor *td) const
Definition: Catalog.cpp:5409
static ReadLock getReadLockForTable(Catalog_Namespace::Catalog &cat, const std::string &table_name)
Definition: LockMgrImpl.h:238
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:2172
Data_Namespace::MemoryLevel persistenceLevel
static constexpr char const * table_epoch_filename
#define CHECK(condition)
Definition: Logger.h:291
bool g_cluster
void update_or_drop_column_ids_in_table_files(const int32_t table_epoch, const std::string &temp_data_dir, const std::unordered_map< int, int > &column_ids_map, const bool drop_not_update)
static bool run
static constexpr char const * table_dumpversion_filename
std::unique_ptr< Parser::Stmt > create_stmt_for_query(const std::string &queryStr, const Catalog_Namespace::SessionInfo &session_info)
#define VLOG(n)
Definition: Logger.h:388
bool g_test_rollback_dump_restore
Catalog_Namespace::Catalog * cat_
Definition: TableArchiver.h:52

+ Here is the call graph for this function:

Member Data Documentation

Catalog_Namespace::Catalog* TableArchiver::cat_
private

Definition at line 52 of file TableArchiver.h.

Referenced by dumpTable(), and restoreTable().


The documentation for this class was generated from the following files: