OmniSciDB  8a228a1076
File.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
23 #include "Shared/File.h"
24 
25 #include <algorithm>
26 #include <atomic>
27 #include <cerrno>
28 #include <chrono>
29 #include <cstdio>
30 #include <cstring>
31 #include <iostream>
32 #include <stdexcept>
33 #include <string>
34 
35 #include "Logger/Logger.h"
36 
37 namespace File_Namespace {
38 
39 FILE* create(const std::string& basePath,
40  const int fileId,
41  const size_t pageSize,
42  const size_t numPages) {
43  std::string path(basePath + "/" + std::to_string(fileId) + "." +
44  std::to_string(pageSize) +
45  std::string(MAPD_FILE_EXT)); // MAPD_FILE_EXT has preceding "."
46  if (numPages < 1 || pageSize < 1) {
47  LOG(FATAL) << "Error trying to create file '" << path
48  << "', Number of pages and page size must be positive integers. numPages "
49  << numPages << " pageSize " << pageSize;
50  }
51  FILE* f = fopen(path.c_str(), "w+b");
52  if (f == nullptr) {
53  LOG(FATAL) << "Error trying to create file '" << path
54  << "', the error was: " << std::strerror(errno);
55  ;
56  }
57  fseek(f, static_cast<long>((pageSize * numPages) - 1), SEEK_SET);
58  fputc(EOF, f);
59  fseek(f, 0, SEEK_SET); // rewind
60  if (fileSize(f) != pageSize * numPages) {
61  LOG(FATAL) << "Error trying to create file '" << path << "', file size "
62  << fileSize(f) << " does not equal pageSize * numPages "
63  << pageSize * numPages;
64  }
65 
66  return f;
67 }
68 
69 FILE* create(const std::string& fullPath, const size_t requestedFileSize) {
70  FILE* f = fopen(fullPath.c_str(), "w+b");
71  if (f == nullptr) {
72  LOG(FATAL) << "Error trying to create file '" << fullPath
73  << "', the error was: " << std::strerror(errno);
74  ;
75  }
76  fseek(f, static_cast<long>(requestedFileSize - 1), SEEK_SET);
77  fputc(EOF, f);
78  fseek(f, 0, SEEK_SET); // rewind
79  if (fileSize(f) != requestedFileSize) {
80  LOG(FATAL) << "Error trying to create file '" << fullPath << "', file size "
81  << fileSize(f) << " does not equal requestedFileSize "
82  << requestedFileSize;
83  }
84  return f;
85 }
86 
87 FILE* open(int fileId) {
88  std::string s(std::to_string(fileId) + std::string(MAPD_FILE_EXT));
89  FILE* f = fopen(s.c_str(), "r+b"); // opens existing file for updates
90  if (f == nullptr) {
91  LOG(FATAL) << "Error trying to open file '" << s
92  << "', the error was: " << std::strerror(errno);
93  }
94  return f;
95 }
96 
97 FILE* open(const std::string& path) {
98  FILE* f = fopen(path.c_str(), "r+b"); // opens existing file for updates
99  if (f == nullptr) {
100  LOG(FATAL) << "Error trying to open file '" << path
101  << "', the errno was: " << std::strerror(errno);
102  }
103  return f;
104 }
105 
106 void close(FILE* f) {
107  CHECK(f);
108  CHECK_EQ(fflush(f), 0);
109  CHECK_EQ(fclose(f), 0);
110 }
111 
112 bool removeFile(const std::string basePath, const std::string filename) {
113  const std::string filePath = basePath + filename;
114  return remove(filePath.c_str()) == 0;
115 }
116 
117 size_t read(FILE* f, const size_t offset, const size_t size, int8_t* buf) {
118  // read "size" bytes from the offset location in the file into the buffer
119  CHECK_EQ(fseek(f, static_cast<long>(offset), SEEK_SET), 0);
120  size_t bytesRead = fread(buf, sizeof(int8_t), size, f);
121  CHECK_EQ(bytesRead, sizeof(int8_t) * size);
122  return bytesRead;
123 }
124 
125 size_t write(FILE* f, const size_t offset, const size_t size, int8_t* buf) {
126  // write size bytes from the buffer to the offset location in the file
127  if (fseek(f, static_cast<long>(offset), SEEK_SET) != 0) {
128  LOG(FATAL)
129  << "Error trying to write to file (during positioning seek) the error was: "
130  << std::strerror(errno);
131  }
132  size_t bytesWritten = fwrite(buf, sizeof(int8_t), size, f);
133  if (bytesWritten != sizeof(int8_t) * size) {
134  LOG(FATAL) << "Error trying to write to file (during fwrite) the error was: "
135  << std::strerror(errno);
136  }
137  return bytesWritten;
138 }
139 
140 size_t append(FILE* f, const size_t size, int8_t* buf) {
141  return write(f, fileSize(f), size, buf);
142 }
143 
144 size_t readPage(FILE* f, const size_t pageSize, const size_t pageNum, int8_t* buf) {
145  return read(f, pageNum * pageSize, pageSize, buf);
146 }
147 
148 size_t readPartialPage(FILE* f,
149  const size_t pageSize,
150  const size_t offset,
151  const size_t readSize,
152  const size_t pageNum,
153  int8_t* buf) {
154  return read(f, pageNum * pageSize + offset, readSize, buf);
155 }
156 
157 size_t writePage(FILE* f, const size_t pageSize, const size_t pageNum, int8_t* buf) {
158  return write(f, pageNum * pageSize, pageSize, buf);
159 }
160 
161 size_t writePartialPage(FILE* f,
162  const size_t pageSize,
163  const size_t offset,
164  const size_t writeSize,
165  const size_t pageNum,
166  int8_t* buf) {
167  return write(f, pageNum * pageSize + offset, writeSize, buf);
168 }
169 
170 size_t appendPage(FILE* f, const size_t pageSize, int8_t* buf) {
171  return write(f, fileSize(f), pageSize, buf);
172 }
173 
175 size_t fileSize(FILE* f) {
176  fseek(f, 0, SEEK_END);
177  size_t size = (size_t)ftell(f);
178  fseek(f, 0, SEEK_SET);
179  return size;
180 }
181 
182 // this is a helper function to rename existing directories
183 // allowing for an async process to actually remove the physical directries
184 // and subfolders and files later
185 // it is required due to the large amount of time it can take to delete
186 // physical files from large disks
187 void renameForDelete(const std::string directoryName) {
189  boost::filesystem::path directoryPath(directoryName);
190  using namespace std::chrono;
191  milliseconds ms = duration_cast<milliseconds>(system_clock::now().time_since_epoch());
192 
193  if (boost::filesystem::exists(directoryPath) &&
194  boost::filesystem::is_directory(directoryPath)) {
195  boost::filesystem::path newDirectoryPath(directoryName + "_" +
196  std::to_string(ms.count()) + "_DELETE_ME");
197  boost::filesystem::rename(directoryPath, newDirectoryPath, ec);
198 
199  if (ec.value() == boost::system::errc::success) {
200  std::thread th([newDirectoryPath]() {
202  boost::filesystem::remove_all(newDirectoryPath, ec);
203  if (ec.value() != boost::system::errc::success) {
204  LOG(ERROR) << "Failed to remove directory " << newDirectoryPath << " error was "
205  << ec;
206  }
207  });
208  // let it run free so we can return
209  // if it fails the file_delete_thread in DBHandler will clean up
210  th.detach();
211 
212  return;
213  }
214 
215  LOG(FATAL) << "Failed to rename file " << directoryName << " to "
216  << directoryName + "_" + std::to_string(ms.count()) + "_DELETE_ME Error: "
217  << ec;
218  }
219 }
220 
221 } // namespace File_Namespace
222 
223 // Still temporary location but avoids the link errors in the new distributed branch.
224 // See the comment file_delete.h
225 
226 #include <atomic>
227 #include <boost/algorithm/string/predicate.hpp>
228 #include <boost/filesystem.hpp>
229 #include <chrono>
230 #include <thread>
231 
232 void file_delete(std::atomic<bool>& program_is_running,
233  const unsigned int wait_interval_seconds,
234  const std::string base_path) {
235  const auto wait_duration = std::chrono::seconds(wait_interval_seconds);
236  const boost::filesystem::path path(base_path);
237  while (program_is_running) {
238  using vec = std::vector<boost::filesystem::path>; // store paths,
239  vec v;
241 
242  // copy vector from iterator as was getting weird random errors if
243  // removed direct from iterator
244  copy(boost::filesystem::directory_iterator(path),
245  boost::filesystem::directory_iterator(),
246  back_inserter(v));
247  for (vec::const_iterator it(v.begin()); it != v.end(); ++it) {
248  std::string object_name(it->string());
249 
250  if (boost::algorithm::ends_with(object_name, "DELETE_ME")) {
251  LOG(INFO) << " removing object " << object_name;
252  boost::filesystem::remove_all(*it, ec);
253  if (ec.value() != boost::system::errc::success) {
254  LOG(ERROR) << "Failed to remove object " << object_name << " error was " << ec;
255  }
256  }
257  }
258 
259  std::this_thread::sleep_for(wait_duration);
260  }
261 }
size_t appendPage(FILE *f, const size_t pageSize, int8_t *buf)
Appends a page from buf to the file.
Definition: File.cpp:170
#define CHECK_EQ(x, y)
Definition: Logger.h:205
std::string filename(char const *path)
Definition: Logger.cpp:62
#define LOG(tag)
Definition: Logger.h:188
size_t writePartialPage(FILE *f, const size_t pageSize, const size_t offset, const size_t writeSize, const size_t pageNum, int8_t *buf)
Definition: File.cpp:161
#define MAPD_FILE_EXT
Definition: File.h:25
size_t writePage(FILE *f, const size_t pageSize, const size_t pageNum, int8_t *buf)
Writes a page from buf to the file.
Definition: File.cpp:157
std::string to_string(char const *&&v)
FILE * create(const std::string &basePath, const int fileId, const size_t pageSize, const size_t numPages)
Definition: File.cpp:39
size_t read(FILE *f, const size_t offset, const size_t size, int8_t *buf)
Reads the specified number of bytes from the offset position in file f into buf.
Definition: File.cpp:117
const int8_t const int64_t const uint64_t const int32_t const int64_t int64_t uint32_t const int64_t int32_t * error_code
size_t append(FILE *f, const size_t size, int8_t *buf)
Appends the specified number of bytes to the end of the file f from buf.
Definition: File.cpp:140
size_t readPage(FILE *f, const size_t pageSize, const size_t pageNum, int8_t *buf)
Reads the specified page from the file f into buf.
Definition: File.cpp:144
size_t fileSize(FILE *f)
Returns the size of the specified file.
Definition: File.cpp:175
size_t readPartialPage(FILE *f, const size_t pageSize, const size_t offset, const size_t readSize, const size_t pageNum, int8_t *buf)
Definition: File.cpp:148
FILE * open(int fileId)
Opens/creates the file with the given id; returns NULL on error.
Definition: File.cpp:87
size_t write(FILE *f, const size_t offset, const size_t size, int8_t *buf)
Writes the specified number of bytes to the offset position in file f from buf.
Definition: File.cpp:125
#define CHECK(condition)
Definition: Logger.h:197
void close(FILE *f)
Closes the file pointed to by the FILE pointer.
Definition: File.cpp:106
bool removeFile(const std::string basePath, const std::string filename)
Deletes the file pointed to by the FILE pointer.
Definition: File.cpp:112
void file_delete(std::atomic< bool > &program_is_running, const unsigned int wait_interval_seconds, const std::string base_path)
Definition: File.cpp:232
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_<EPOCH>_<oldname>.
Definition: File.cpp:187
A selection of helper methods for File I/O.