OmniSciDB  340b00dbf6
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
File.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
23 #include "Shared/File.h"
24 
25 #include <algorithm>
26 #include <atomic>
27 #include <cerrno>
28 #include <chrono>
29 #include <cstdio>
30 #include <cstring>
31 #include <iostream>
32 #include <stdexcept>
33 #include <string>
34 
35 #include "Logger/Logger.h"
36 #include "OSDependent/omnisci_fs.h"
37 
38 namespace File_Namespace {
39 
40 FILE* create(const std::string& basePath,
41  const int fileId,
42  const size_t pageSize,
43  const size_t numPages) {
44  std::string path(basePath + "/" + std::to_string(fileId) + "." +
45  std::to_string(pageSize) +
46  std::string(MAPD_FILE_EXT)); // MAPD_FILE_EXT has preceding "."
47  if (numPages < 1 || pageSize < 1) {
48  LOG(FATAL) << "Error trying to create file '" << path
49  << "', Number of pages and page size must be positive integers. numPages "
50  << numPages << " pageSize " << pageSize;
51  }
52  FILE* f = omnisci::fopen(path.c_str(), "w+b");
53  if (f == nullptr) {
54  LOG(FATAL) << "Error trying to create file '" << path
55  << "', the error was: " << std::strerror(errno);
56  ;
57  }
58  fseek(f, static_cast<long>((pageSize * numPages) - 1), SEEK_SET);
59  fputc(EOF, f);
60  fseek(f, 0, SEEK_SET); // rewind
61  if (fileSize(f) != pageSize * numPages) {
62  LOG(FATAL) << "Error trying to create file '" << path << "', file size "
63  << fileSize(f) << " does not equal pageSize * numPages "
64  << pageSize * numPages;
65  }
66 
67  return f;
68 }
69 
70 FILE* create(const std::string& fullPath, const size_t requestedFileSize) {
71  FILE* f = omnisci::fopen(fullPath.c_str(), "w+b");
72  if (f == nullptr) {
73  LOG(FATAL) << "Error trying to create file '" << fullPath
74  << "', the error was: " << std::strerror(errno);
75  ;
76  }
77  fseek(f, static_cast<long>(requestedFileSize - 1), SEEK_SET);
78  fputc(EOF, f);
79  fseek(f, 0, SEEK_SET); // rewind
80  if (fileSize(f) != requestedFileSize) {
81  LOG(FATAL) << "Error trying to create file '" << fullPath << "', file size "
82  << fileSize(f) << " does not equal requestedFileSize "
83  << requestedFileSize;
84  }
85  return f;
86 }
87 
88 FILE* open(int fileId) {
89  std::string s(std::to_string(fileId) + std::string(MAPD_FILE_EXT));
90  FILE* f = omnisci::fopen(s.c_str(), "r+b"); // opens existing file for updates
91  if (f == nullptr) {
92  LOG(FATAL) << "Error trying to open file '" << s
93  << "', the error was: " << std::strerror(errno);
94  }
95  return f;
96 }
97 
98 FILE* open(const std::string& path) {
99  FILE* f = omnisci::fopen(path.c_str(), "r+b"); // opens existing file for updates
100  if (f == nullptr) {
101  LOG(FATAL) << "Error trying to open file '" << path
102  << "', the errno was: " << std::strerror(errno);
103  }
104  return f;
105 }
106 
107 void close(FILE* f) {
108  CHECK(f);
109  CHECK_EQ(fflush(f), 0);
110  CHECK_EQ(fclose(f), 0);
111 }
112 
113 bool removeFile(const std::string basePath, const std::string filename) {
114  const std::string filePath = basePath + filename;
115  return remove(filePath.c_str()) == 0;
116 }
117 
118 size_t read(FILE* f, const size_t offset, const size_t size, int8_t* buf) {
119  // read "size" bytes from the offset location in the file into the buffer
120  CHECK_EQ(fseek(f, static_cast<long>(offset), SEEK_SET), 0);
121  size_t bytesRead = fread(buf, sizeof(int8_t), size, f);
122  CHECK_EQ(bytesRead, sizeof(int8_t) * size);
123  return bytesRead;
124 }
125 
126 size_t write(FILE* f, const size_t offset, const size_t size, int8_t* buf) {
127  // write size bytes from the buffer to the offset location in the file
128  if (fseek(f, static_cast<long>(offset), SEEK_SET) != 0) {
129  LOG(FATAL)
130  << "Error trying to write to file (during positioning seek) the error was: "
131  << std::strerror(errno);
132  }
133  size_t bytesWritten = fwrite(buf, sizeof(int8_t), size, f);
134  if (bytesWritten != sizeof(int8_t) * size) {
135  LOG(FATAL) << "Error trying to write to file (during fwrite) the error was: "
136  << std::strerror(errno);
137  }
138  return bytesWritten;
139 }
140 
141 size_t append(FILE* f, const size_t size, int8_t* buf) {
142  return write(f, fileSize(f), size, buf);
143 }
144 
145 size_t readPage(FILE* f, const size_t pageSize, const size_t pageNum, int8_t* buf) {
146  return read(f, pageNum * pageSize, pageSize, buf);
147 }
148 
149 size_t readPartialPage(FILE* f,
150  const size_t pageSize,
151  const size_t offset,
152  const size_t readSize,
153  const size_t pageNum,
154  int8_t* buf) {
155  return read(f, pageNum * pageSize + offset, readSize, buf);
156 }
157 
158 size_t writePage(FILE* f, const size_t pageSize, const size_t pageNum, int8_t* buf) {
159  return write(f, pageNum * pageSize, pageSize, buf);
160 }
161 
162 size_t writePartialPage(FILE* f,
163  const size_t pageSize,
164  const size_t offset,
165  const size_t writeSize,
166  const size_t pageNum,
167  int8_t* buf) {
168  return write(f, pageNum * pageSize + offset, writeSize, buf);
169 }
170 
171 size_t appendPage(FILE* f, const size_t pageSize, int8_t* buf) {
172  return write(f, fileSize(f), pageSize, buf);
173 }
174 
176 size_t fileSize(FILE* f) {
177  fseek(f, 0, SEEK_END);
178  size_t size = (size_t)ftell(f);
179  fseek(f, 0, SEEK_SET);
180  return size;
181 }
182 
183 // this is a helper function to rename existing directories
184 // allowing for an async process to actually remove the physical directries
185 // and subfolders and files later
186 // it is required due to the large amount of time it can take to delete
187 // physical files from large disks
188 void renameForDelete(const std::string directoryName) {
189  boost::system::error_code ec;
190  boost::filesystem::path directoryPath(directoryName);
191  using namespace std::chrono;
192  milliseconds ms = duration_cast<milliseconds>(system_clock::now().time_since_epoch());
193 
194  if (boost::filesystem::exists(directoryPath) &&
195  boost::filesystem::is_directory(directoryPath)) {
196  boost::filesystem::path newDirectoryPath(directoryName + "_" +
197  std::to_string(ms.count()) + "_DELETE_ME");
198  boost::filesystem::rename(directoryPath, newDirectoryPath, ec);
199 
200  if (ec.value() == boost::system::errc::success) {
201  std::thread th([newDirectoryPath]() {
202  boost::system::error_code ec;
203  boost::filesystem::remove_all(newDirectoryPath, ec);
204  if (ec.value() != boost::system::errc::success) {
205  LOG(ERROR) << "Failed to remove directory " << newDirectoryPath << " error was "
206  << ec;
207  }
208  });
209  // let it run free so we can return
210  // if it fails the file_delete_thread in DBHandler will clean up
211  th.detach();
212 
213  return;
214  }
215 
216  LOG(FATAL) << "Failed to rename file " << directoryName << " to "
217  << directoryName + "_" + std::to_string(ms.count()) + "_DELETE_ME Error: "
218  << ec;
219  }
220 }
221 
222 } // namespace File_Namespace
223 
224 // Still temporary location but avoids the link errors in the new distributed branch.
225 // See the comment file_delete.h
226 
227 #include <atomic>
228 #include <boost/algorithm/string/predicate.hpp>
229 #include <boost/filesystem.hpp>
230 #include <chrono>
231 #include <thread>
232 
233 void file_delete(std::atomic<bool>& program_is_running,
234  const unsigned int wait_interval_seconds,
235  const std::string base_path) {
236  const auto wait_duration = std::chrono::seconds(wait_interval_seconds);
237  const boost::filesystem::path path(base_path);
238  while (program_is_running) {
239  using vec = std::vector<boost::filesystem::path>; // store paths,
240  vec v;
241  boost::system::error_code ec;
242 
243  // copy vector from iterator as was getting weird random errors if
244  // removed direct from iterator
245  copy(boost::filesystem::directory_iterator(path),
246  boost::filesystem::directory_iterator(),
247  back_inserter(v));
248  for (vec::const_iterator it(v.begin()); it != v.end(); ++it) {
249  std::string object_name(it->string());
250 
251  if (boost::algorithm::ends_with(object_name, "DELETE_ME")) {
252  LOG(INFO) << " removing object " << object_name;
253  boost::filesystem::remove_all(*it, ec);
254  if (ec.value() != boost::system::errc::success) {
255  LOG(ERROR) << "Failed to remove object " << object_name << " error was " << ec;
256  }
257  }
258  }
259 
260  std::this_thread::sleep_for(wait_duration);
261  }
262 }
size_t appendPage(FILE *f, const size_t pageSize, int8_t *buf)
Appends a page from buf to the file.
Definition: File.cpp:171
#define CHECK_EQ(x, y)
Definition: Logger.h:205
std::string filename(char const *path)
Definition: Logger.cpp:62
::FILE * fopen(const char *filename, const char *mode)
Definition: omnisci_fs.cpp:72
#define LOG(tag)
Definition: Logger.h:188
size_t writePartialPage(FILE *f, const size_t pageSize, const size_t offset, const size_t writeSize, const size_t pageNum, int8_t *buf)
Definition: File.cpp:162
#define MAPD_FILE_EXT
Definition: File.h:25
size_t writePage(FILE *f, const size_t pageSize, const size_t pageNum, int8_t *buf)
Writes a page from buf to the file.
Definition: File.cpp:158
std::string to_string(char const *&&v)
FILE * create(const std::string &basePath, const int fileId, const size_t pageSize, const size_t numPages)
Definition: File.cpp:40
size_t read(FILE *f, const size_t offset, const size_t size, int8_t *buf)
Reads the specified number of bytes from the offset position in file f into buf.
Definition: File.cpp:118
size_t append(FILE *f, const size_t size, int8_t *buf)
Appends the specified number of bytes to the end of the file f from buf.
Definition: File.cpp:141
size_t readPage(FILE *f, const size_t pageSize, const size_t pageNum, int8_t *buf)
Reads the specified page from the file f into buf.
Definition: File.cpp:145
size_t fileSize(FILE *f)
Returns the size of the specified file.
Definition: File.cpp:176
size_t readPartialPage(FILE *f, const size_t pageSize, const size_t offset, const size_t readSize, const size_t pageNum, int8_t *buf)
Definition: File.cpp:149
FILE * open(int fileId)
Opens/creates the file with the given id; returns NULL on error.
Definition: File.cpp:88
size_t write(FILE *f, const size_t offset, const size_t size, int8_t *buf)
Writes the specified number of bytes to the offset position in file f from buf.
Definition: File.cpp:126
#define CHECK(condition)
Definition: Logger.h:197
void close(FILE *f)
Closes the file pointed to by the FILE pointer.
Definition: File.cpp:107
bool removeFile(const std::string basePath, const std::string filename)
Deletes the file pointed to by the FILE pointer.
Definition: File.cpp:113
void file_delete(std::atomic< bool > &program_is_running, const unsigned int wait_interval_seconds, const std::string base_path)
Definition: File.cpp:233
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_&lt;EPOCH&gt;_&lt;oldname&gt;.
Definition: File.cpp:188
A selection of helper methods for File I/O.