24 #include <unordered_map>
33 #include "MapDRelease.h"
37 namespace migrations {
56 "migration.lockfile");
62 throw std::runtime_error(
63 "another HeavyDB server instance is already using data directory: " + base_path);
68 if (!migration_enabled_) {
72 migration_enabled_ =
true;
93 const int database_id,
96 std::vector<int> tables_migrated = {};
97 std::unordered_map<int, std::vector<std::string>> tables_to_migrate;
98 sqlite.
query(
"BEGIN TRANSACTION");
101 "select name from sqlite_master WHERE type='table' AND "
102 "name='mapd_version_history'");
105 "CREATE TABLE mapd_version_history(version integer, migration_history text "
108 "CREATE TABLE mapd_date_in_days_column_migration_tmp(table_id integer primary "
112 "select * from mapd_version_history where migration_history = "
113 "'date_in_days_column'");
116 sqlite.
query(
"END TRANSACTION");
119 LOG(
INFO) <<
"Checking for date columns requiring metadata migration.";
121 "select name from sqlite_master where type='table' AND "
122 "name='mapd_date_in_days_column_migration_tmp'");
124 sqlite.
query(
"select table_id from mapd_date_in_days_column_migration_tmp");
126 for (
size_t i = 0; i < sqlite.
getNumRows(); i++) {
127 tables_migrated.push_back(sqlite.
getData<
int>(i, 0));
132 "CREATE TABLE mapd_date_in_days_column_migration_tmp(table_id integer "
137 "SELECT tables.tableid, tables.name, columns.name FROM mapd_tables tables, "
138 "mapd_columns columns where tables.tableid = columns.tableid AND "
139 "columns.coltype = ?1 AND columns.compression = ?2",
140 std::vector<std::string>{
144 for (
size_t i = 0; i < sqlite.
getNumRows(); i++) {
145 tables_to_migrate[sqlite.
getData<
int>(i, 0)] = {
146 sqlite.
getData<std::string>(i, 1), sqlite.
getData<std::string>(i, 2)};
149 }
catch (
const std::exception& e) {
150 LOG(
ERROR) <<
"Failed to complete migration on date in days column metadata: "
152 sqlite.
query(
"ROLLBACK");
155 sqlite.
query(
"END TRANSACTION");
157 for (
auto& id_names : tables_to_migrate) {
158 if (std::find(tables_migrated.begin(), tables_migrated.end(), id_names.first) ==
159 tables_migrated.end()) {
160 sqlite.
query(
"BEGIN TRANSACTION");
162 LOG(
INFO) <<
"Table: " << id_names.second[0]
163 <<
" may suffer from issues with DATE column: " << id_names.second[1]
164 <<
". Running an OPTIMIZE command to solve any issues with metadata.";
169 auto table_desc_itr = table_descriptors_by_id.find(id_names.first);
170 if (table_desc_itr == table_descriptors_by_id.end()) {
171 throw std::runtime_error(
"Table descriptor does not exist for table " +
172 id_names.second[0] +
" does not exist.");
174 auto td = table_desc_itr->second;
179 "INSERT INTO mapd_date_in_days_column_migration_tmp VALUES(?)",
181 }
catch (
const std::exception& e) {
182 LOG(
ERROR) <<
"Failed to complete metadata migration on date in days column: "
184 sqlite.
query(
"ROLLBACK");
187 sqlite.
query(
"COMMIT");
191 sqlite.
query(
"BEGIN TRANSACTION");
193 sqlite.
query(
"DROP TABLE mapd_date_in_days_column_migration_tmp");
195 "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
197 }
catch (
const std::exception& e) {
198 LOG(
ERROR) <<
"Failed to complete migraion on date in days column: " << e.what();
199 sqlite.
query(
"ROLLBACK");
202 sqlite.
query(
"END TRANSACTION");
203 LOG(
INFO) <<
"Successfully migrated all date in days column metadata.";
208 const std::filesystem::path& new_path) {
209 bool file_updated{
false};
210 if (std::filesystem::exists(old_path)) {
212 if (std::filesystem::is_symlink(old_path)) {
213 if (std::filesystem::read_symlink(old_path) != new_path.filename()) {
214 std::stringstream ss;
215 ss <<
"Rebrand migration: Encountered an unexpected symlink at path: " << old_path
216 <<
". Symlink does not reference file: " << new_path.filename();
217 throw std::runtime_error(ss.str());
219 if (!std::filesystem::exists(new_path)) {
220 std::stringstream ss;
221 ss <<
"Rebrand migration: Encountered symlink at legacy path: " << old_path
222 <<
" but no corresponding file at new path: " << new_path;
223 throw std::runtime_error(ss.str());
226 if (std::filesystem::exists(new_path)) {
227 std::stringstream ss;
228 ss <<
"Rebrand migration: Encountered existing non-symlink files at the legacy "
230 << old_path <<
" and new path: " << new_path;
231 throw std::runtime_error(ss.str());
233 std::filesystem::rename(old_path, new_path);
234 std::cout <<
"Rebrand migration: Renamed " << old_path <<
" to " << new_path
240 if (std::filesystem::exists(old_path)) {
241 if (!std::filesystem::is_symlink(old_path)) {
242 std::stringstream ss;
243 ss <<
"Rebrand migration: An unexpected error occurred. A symlink should have been "
246 throw std::runtime_error(ss.str());
248 if (std::filesystem::read_symlink(old_path) != new_path.filename()) {
249 std::stringstream ss;
250 ss <<
"Rebrand migration: Encountered an unexpected symlink at path: " << old_path
251 <<
". Symlink does not reference file: " << new_path.filename();
252 throw std::runtime_error(ss.str());
254 }
else if (std::filesystem::exists(new_path)) {
255 std::filesystem::create_symlink(new_path.filename(), old_path);
256 std::cout <<
"Rebrand migration: Added symlink from " << old_path <<
" to "
257 << new_path.filename() << std::endl;
264 const std::string& dir_name,
265 const std::string& old_file_name,
266 const std::string& new_file_name) {
267 auto old_path = std::filesystem::canonical(base_path);
268 auto new_path = std::filesystem::canonical(base_path);
269 if (!dir_name.empty()) {
270 old_path /= dir_name;
271 new_path /= dir_name;
273 if (old_file_name.empty()) {
274 throw std::runtime_error(
275 "Unexpected error in rename_and_symlink_file: old_file_name is empty");
277 old_path /= old_file_name;
279 if (new_file_name.empty()) {
280 throw std::runtime_error(
281 "Unexpected error in rename_and_symlink_file: new_file_name is empty");
283 new_path /= new_file_name;
290 bool migration_occurred{
false};
293 const std::map<std::string, std::string> old_to_new_dir_names {
303 const auto storage_base_path = std::filesystem::canonical(base_path);
306 for (
const auto& [old_dir_name, new_dir_name] : old_to_new_dir_names) {
307 auto old_path = storage_base_path / old_dir_name;
308 auto new_path = storage_base_path / new_dir_name;
310 migration_occurred =
true;
323 "omnisci_system_catalog",
325 if (license_updated || key_updated || sys_catalog_updated) {
326 migration_occurred =
true;
330 const std::array<std::filesystem::path, 9> files_to_delete{
331 storage_base_path /
"omnisci_disk_cache",
332 storage_base_path /
"omnisci_server_pid.lck",
333 storage_base_path /
"mapd_server_pid.lck",
341 for (
const auto& file_path : files_to_delete) {
342 if (std::filesystem::exists(file_path)) {
343 std::filesystem::remove_all(file_path);
344 std::cout <<
"Rebrand migration: Deleted file " << file_path << std::endl;
345 migration_occurred =
true;
348 if (migration_occurred) {
349 std::cout <<
"Rebrand migration completed" << std::endl;
const std::string kDataDirectoryName
T getData(const int row, const int col)
class for a per-database catalog. also includes metadata for the current database and the current use...
virtual void query_with_text_params(std::string const &query_only)
static void relaxMigrationLock()
const std::string kDefaultLogDirName
const std::string kSystemCatalogName
virtual void query(const std::string &queryString)
Driver for running cleanup processes on a table. TableOptimizer provides functions for various cleanu...
Constants for Builtin SQL Types supported by HEAVY.AI.
const std::string kDefaultExportDirName
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
bool rename_and_symlink_path(const std::filesystem::path &old_path, const std::filesystem::path &new_path)
const std::string kDefaultImportDirName
std::map< int, TableDescriptor * > TableDescriptorMapById
static bool migration_enabled_
static const int32_t MAPD_VERSION
const std::string kDefaultKeyFileName
const std::string kDefaultKeyStoreDirName
static void executeRebrandMigration(const std::string &base_path)
static void takeMigrationLock(const std::string &base_path)
bool rename_and_symlink_file(const std::filesystem::path &base_path, const std::string &dir_name, const std::string &old_file_name, const std::string &new_file_name)
const std::string kCatalogDirectoryName
const std::string kDefaultLicenseFileName
static void migrateDateInDaysMetadata(const Catalog_Namespace::TableDescriptorMapById &table_descriptors_by_id, const int database_id, Catalog_Namespace::Catalog *cat, SqliteConnector &sqlite)
static std::unique_ptr< heavyai::DistributedSharedMutex > migration_mutex_
const std::string kLockfilesDirectoryName
virtual size_t getNumRows() const
void recomputeMetadata() const
Recomputes per-chunk metadata for each fragment in the table. Updates and deletes can cause chunk met...
static const ExecutorId UNITARY_EXECUTOR_ID