OmniSciDB  340b00dbf6
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
MigrationMgr.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2020 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
18 
19 #include <algorithm>
20 #include <exception>
21 #include <string>
22 #include <unordered_map>
23 #include <vector>
24 
25 #include "Logger/Logger.h"
26 #include "QueryEngine/Execute.h"
28 #include "Shared/sqltypes.h"
29 
30 #include "MapDRelease.h"
31 
32 namespace migrations {
33 
35  const Catalog_Namespace::TableDescriptorMapById& table_descriptors_by_id,
36  const int database_id,
38  SqliteConnector& sqlite) {
39  std::vector<int> tables_migrated = {};
40  std::unordered_map<int, std::vector<std::string>> tables_to_migrate;
41  sqlite.query("BEGIN TRANSACTION");
42  try {
43  sqlite.query(
44  "select name from sqlite_master WHERE type='table' AND "
45  "name='mapd_version_history'");
46  if (sqlite.getNumRows() == 0) {
47  sqlite.query(
48  "CREATE TABLE mapd_version_history(version integer, migration_history text "
49  "unique)");
50  sqlite.query(
51  "CREATE TABLE mapd_date_in_days_column_migration_tmp(table_id integer primary "
52  "key)");
53  } else {
54  sqlite.query(
55  "select * from mapd_version_history where migration_history = "
56  "'date_in_days_column'");
57  if (sqlite.getNumRows() != 0) {
58  // no need for further execution
59  sqlite.query("END TRANSACTION");
60  return;
61  }
62  LOG(INFO) << "Checking for date columns requiring metadata migration.";
63  sqlite.query(
64  "select name from sqlite_master where type='table' AND "
65  "name='mapd_date_in_days_column_migration_tmp'");
66  if (sqlite.getNumRows() != 0) {
67  sqlite.query("select table_id from mapd_date_in_days_column_migration_tmp");
68  if (sqlite.getNumRows() != 0) {
69  for (size_t i = 0; i < sqlite.getNumRows(); i++) {
70  tables_migrated.push_back(sqlite.getData<int>(i, 0));
71  }
72  }
73  } else {
74  sqlite.query(
75  "CREATE TABLE mapd_date_in_days_column_migration_tmp(table_id integer "
76  "primary key)");
77  }
78  }
80  "SELECT tables.tableid, tables.name, columns.name FROM mapd_tables tables, "
81  "mapd_columns columns where tables.tableid = columns.tableid AND "
82  "columns.coltype = ?1 AND columns.compression = ?2",
83  std::vector<std::string>{
84  std::to_string(static_cast<int>(SQLTypes::kDATE)),
86  if (sqlite.getNumRows() != 0) {
87  for (size_t i = 0; i < sqlite.getNumRows(); i++) {
88  tables_to_migrate[sqlite.getData<int>(i, 0)] = {
89  sqlite.getData<std::string>(i, 1), sqlite.getData<std::string>(i, 2)};
90  }
91  }
92  } catch (const std::exception& e) {
93  LOG(ERROR) << "Failed to complete migration on date in days column metadata: "
94  << e.what();
95  sqlite.query("ROLLBACK");
96  throw;
97  }
98  sqlite.query("END TRANSACTION");
99 
100  for (auto& id_names : tables_to_migrate) {
101  if (std::find(tables_migrated.begin(), tables_migrated.end(), id_names.first) ==
102  tables_migrated.end()) {
103  sqlite.query("BEGIN TRANSACTION");
104  try {
105  LOG(INFO) << "Table: " << id_names.second[0]
106  << " may suffer from issues with DATE column: " << id_names.second[1]
107  << ". Running an OPTIMIZE command to solve any issues with metadata.";
108 
109  // TODO(adb): Could have the TableOptimizer get the Executor and avoid including
110  // Execute.h
112  auto table_desc_itr = table_descriptors_by_id.find(id_names.first);
113  if (table_desc_itr == table_descriptors_by_id.end()) {
114  throw std::runtime_error("Table descriptor does not exist for table " +
115  id_names.second[0] + " does not exist.");
116  }
117  auto td = table_desc_itr->second;
118  TableOptimizer optimizer(td, executor.get(), *cat);
119  optimizer.recomputeMetadata();
120 
121  sqlite.query_with_text_params(
122  "INSERT INTO mapd_date_in_days_column_migration_tmp VALUES(?)",
123  std::vector<std::string>{std::to_string(id_names.first)});
124  } catch (const std::exception& e) {
125  LOG(ERROR) << "Failed to complete metadata migration on date in days column: "
126  << e.what();
127  sqlite.query("ROLLBACK");
128  throw;
129  }
130  sqlite.query("COMMIT");
131  }
132  }
133 
134  sqlite.query("BEGIN TRANSACTION");
135  try {
136  sqlite.query("DROP TABLE mapd_date_in_days_column_migration_tmp");
137  sqlite.query_with_text_params(
138  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
139  std::vector<std::string>{std::to_string(MAPD_VERSION), "date_in_days_column"});
140  } catch (const std::exception& e) {
141  LOG(ERROR) << "Failed to complete migraion on date in days column: " << e.what();
142  sqlite.query("ROLLBACK");
143  throw;
144  }
145  sqlite.query("END TRANSACTION");
146  LOG(INFO) << "Successfully migrated all date in days column metadata.";
147 }
148 
149 } // namespace migrations
std::string cat(Ts &&...args)
T getData(const int row, const int col)
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:97
virtual void query_with_text_params(std::string const &query_only)
#define LOG(tag)
Definition: Logger.h:188
virtual void query(const std::string &queryString)
Driver for running cleanup processes on a table. TableOptimizer provides functions for various cleanu...
Constants for Builtin SQL Types supported by OmniSci.
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters system_parameters=SystemParameters())
Definition: Execute.cpp:150
std::string to_string(char const *&&v)
std::map< int, TableDescriptor * > TableDescriptorMapById
Definition: Types.h:34
static void migrateDateInDaysMetadata(const Catalog_Namespace::TableDescriptorMapById &table_descriptors_by_id, const int database_id, const Catalog_Namespace::Catalog *cat, SqliteConnector &sqlite)
static const int32_t MAPD_VERSION
Definition: release.h:33
Definition: sqltypes.h:55
virtual size_t getNumRows() const
void recomputeMetadata() const
Recomputes per-chunk metadata for each fragment in the table. Updates and deletes can cause chunk met...
static const ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:337