OmniSciDB  c07336695a
File.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
23 #include "File.h"
24 #include <unistd.h>
25 #include <boost/filesystem.hpp>
26 #include <cstdio>
27 #include <cstring>
28 #include <iostream>
29 #include <stdexcept>
30 #include <string>
31 #include "Logger.h"
32 
33 namespace File_Namespace {
34 
35 FILE* create(const std::string& basePath,
36  const int fileId,
37  const size_t pageSize,
38  const size_t numPages) {
39  std::string path(basePath + "/" + std::to_string(fileId) + "." +
40  std::to_string(pageSize) +
41  std::string(MAPD_FILE_EXT)); // MAPD_FILE_EXT has preceding "."
42  if (numPages < 1 || pageSize < 1) {
43  LOG(FATAL) << "Error trying to create file '" << path
44  << "', Number of pages and page size must be positive integers. numPages "
45  << numPages << " pageSize " << pageSize;
46  }
47  FILE* f = fopen(path.c_str(), "w+b");
48  if (f == nullptr) {
49  LOG(FATAL) << "Error trying to create file '" << path
50  << "', the error was: " << std::strerror(errno);
51  ;
52  }
53  fseek(f, (pageSize * numPages) - 1, SEEK_SET);
54  fputc(EOF, f);
55  fseek(f, 0, SEEK_SET); // rewind
56  if (fileSize(f) != pageSize * numPages) {
57  LOG(FATAL) << "Error trying to create file '" << path << "', file size "
58  << fileSize(f) << " does not equal pageSize * numPages "
59  << pageSize * numPages;
60  }
61 
62  return f;
63 }
64 
65 FILE* create(const std::string& fullPath, const size_t requestedFileSize) {
66  FILE* f = fopen(fullPath.c_str(), "w+b");
67  if (f == nullptr) {
68  LOG(FATAL) << "Error trying to create file '" << fullPath
69  << "', the error was: " << std::strerror(errno);
70  ;
71  }
72  fseek(f, requestedFileSize - 1, SEEK_SET);
73  fputc(EOF, f);
74  fseek(f, 0, SEEK_SET); // rewind
75  if (fileSize(f) != requestedFileSize) {
76  LOG(FATAL) << "Error trying to create file '" << fullPath << "', file size "
77  << fileSize(f) << " does not equal requestedFileSize "
78  << requestedFileSize;
79  }
80  return f;
81 }
82 
83 FILE* open(int fileId) {
84  std::string s(std::to_string(fileId) + std::string(MAPD_FILE_EXT));
85  FILE* f = fopen(s.c_str(), "r+b"); // opens existing file for updates
86  if (f == nullptr) {
87  LOG(FATAL) << "Error trying to open file '" << s
88  << "', the error was: " << std::strerror(errno);
89  }
90  return f;
91 }
92 
93 FILE* open(const std::string& path) {
94  FILE* f = fopen(path.c_str(), "r+b"); // opens existing file for updates
95  if (f == nullptr) {
96  LOG(FATAL) << "Error trying to open file '" << path
97  << "', the errno was: " << std::strerror(errno);
98  }
99  return f;
100 }
101 
102 void close(FILE* f) {
103  CHECK(f);
104  CHECK_EQ(fflush(f), 0);
105  CHECK_EQ(fclose(f), 0);
106 }
107 
108 bool removeFile(const std::string basePath, const std::string filename) {
109  const std::string filePath = basePath + filename;
110  return remove(filePath.c_str()) == 0;
111 }
112 
113 size_t read(FILE* f, const size_t offset, const size_t size, int8_t* buf) {
114  // read "size" bytes from the offset location in the file into the buffer
115  CHECK_EQ(fseek(f, offset, SEEK_SET), 0);
116  size_t bytesRead = fread(buf, sizeof(int8_t), size, f);
117  CHECK_EQ(bytesRead, sizeof(int8_t) * size);
118  return bytesRead;
119 }
120 
121 size_t write(FILE* f, const size_t offset, const size_t size, int8_t* buf) {
122  // write size bytes from the buffer to the offset location in the file
123  if (fseek(f, offset, SEEK_SET) != 0) {
124  LOG(FATAL)
125  << "Error trying to write to file (during positioning seek) the error was: "
126  << std::strerror(errno);
127  }
128  size_t bytesWritten = fwrite(buf, sizeof(int8_t), size, f);
129  if (bytesWritten != sizeof(int8_t) * size) {
130  LOG(FATAL) << "Error trying to write to file (during fwrite) the error was: "
131  << std::strerror(errno);
132  }
133  return bytesWritten;
134 }
135 
136 size_t append(FILE* f, const size_t size, int8_t* buf) {
137  return write(f, fileSize(f), size, buf);
138 }
139 
140 size_t readPage(FILE* f, const size_t pageSize, const size_t pageNum, int8_t* buf) {
141  return read(f, pageNum * pageSize, pageSize, buf);
142 }
143 
144 size_t readPartialPage(FILE* f,
145  const size_t pageSize,
146  const size_t offset,
147  const size_t readSize,
148  const size_t pageNum,
149  int8_t* buf) {
150  return read(f, pageNum * pageSize + offset, readSize, buf);
151 }
152 
153 size_t writePage(FILE* f, const size_t pageSize, const size_t pageNum, int8_t* buf) {
154  return write(f, pageNum * pageSize, pageSize, buf);
155 }
156 
157 size_t writePartialPage(FILE* f,
158  const size_t pageSize,
159  const size_t offset,
160  const size_t writeSize,
161  const size_t pageNum,
162  int8_t* buf) {
163  return write(f, pageNum * pageSize + offset, writeSize, buf);
164 }
165 
166 size_t appendPage(FILE* f, const size_t pageSize, int8_t* buf) {
167  return write(f, fileSize(f), pageSize, buf);
168 }
169 
171 size_t fileSize(FILE* f) {
172  fseek(f, 0, SEEK_END);
173  size_t size = (size_t)ftell(f);
174  fseek(f, 0, SEEK_SET);
175  return size;
176 }
177 
178 // this is a helper function to rename existing directories
179 // allowing for an async process to actually remove the physical directries
180 // and subfolders and files later
181 // it is required due to the large amount of time it can take to delete
182 // physical files from large disks
183 void renameForDelete(const std::string directoryName) {
185  boost::filesystem::path directoryPath(directoryName);
186  using namespace std::chrono;
187  milliseconds ms = duration_cast<milliseconds>(system_clock::now().time_since_epoch());
188 
189  if (boost::filesystem::exists(directoryPath) &&
190  boost::filesystem::is_directory(directoryPath)) {
191  boost::filesystem::path newDirectoryPath(directoryName + "_" +
192  std::to_string(ms.count()) + "_DELETE_ME");
193  boost::filesystem::rename(directoryPath, newDirectoryPath, ec);
194 
195  if (ec.value() == boost::system::errc::success) {
196  std::thread th([newDirectoryPath]() {
198  boost::filesystem::remove_all(newDirectoryPath, ec);
199  if (ec.value() != boost::system::errc::success) {
200  LOG(ERROR) << "Failed to remove directory " << newDirectoryPath << " error was "
201  << ec;
202  }
203  });
204  // let it run free so we can return
205  // if it fails the file_delete_thread in MapDHandler will clean up
206  th.detach();
207 
208  return;
209  }
210 
211  LOG(FATAL) << "Failed to rename file " << directoryName << " to "
212  << directoryName + "_" + std::to_string(ms.count()) + "_DELETE_ME Error: "
213  << ec;
214  }
215 }
216 
217 } // namespace File_Namespace
218 
219 // Still temporary location but avoids the link errors in the new distributed branch.
220 // See the comment file_delete.h
221 
222 #include <atomic>
223 #include <boost/algorithm/string/predicate.hpp>
224 #include <boost/filesystem.hpp>
225 #include <chrono>
226 #include <thread>
227 
228 void file_delete(std::atomic<bool>& program_is_running,
229  const unsigned int wait_interval_seconds,
230  const std::string base_path) {
231  const auto wait_duration = std::chrono::seconds(wait_interval_seconds);
232  const boost::filesystem::path path(base_path);
233  while (program_is_running) {
234  using vec = std::vector<boost::filesystem::path>; // store paths,
235  vec v;
237 
238  // copy vector from iterator as was getting weird random errors if
239  // removed direct from iterator
240  copy(boost::filesystem::directory_iterator(path),
241  boost::filesystem::directory_iterator(),
242  back_inserter(v));
243  for (vec::const_iterator it(v.begin()); it != v.end(); ++it) {
244  std::string object_name(it->string());
245 
246  if (boost::algorithm::ends_with(object_name, "DELETE_ME")) {
247  LOG(INFO) << " removing object " << object_name;
248  boost::filesystem::remove_all(*it, ec);
249  if (ec.value() != boost::system::errc::success) {
250  LOG(ERROR) << "Failed to remove object " << object_name << " error was " << ec;
251  }
252  }
253  }
254 
255  std::this_thread::sleep_for(wait_duration);
256  }
257 }
size_t appendPage(FILE *f, const size_t pageSize, int8_t *buf)
Appends a page from buf to the file.
Definition: File.cpp:166
#define CHECK_EQ(x, y)
Definition: Logger.h:195
#define LOG(tag)
Definition: Logger.h:182
size_t writePartialPage(FILE *f, const size_t pageSize, const size_t offset, const size_t writeSize, const size_t pageNum, int8_t *buf)
Definition: File.cpp:157
#define MAPD_FILE_EXT
Definition: File.h:26
size_t writePage(FILE *f, const size_t pageSize, const size_t pageNum, int8_t *buf)
Writes a page from buf to the file.
Definition: File.cpp:153
std::string to_string(char const *&&v)
FILE * create(const std::string &basePath, const int fileId, const size_t pageSize, const size_t numPages)
Definition: File.cpp:35
size_t read(FILE *f, const size_t offset, const size_t size, int8_t *buf)
Reads the specified number of bytes from the offset position in file f into buf.
Definition: File.cpp:113
T v(const TargetValue &r)
const int8_t const int64_t const uint64_t const int32_t const int64_t int64_t uint32_t const int64_t int32_t * error_code
size_t append(FILE *f, const size_t size, int8_t *buf)
Appends the specified number of bytes to the end of the file f from buf.
Definition: File.cpp:136
size_t readPage(FILE *f, const size_t pageSize, const size_t pageNum, int8_t *buf)
Reads the specified page from the file f into buf.
Definition: File.cpp:140
size_t fileSize(FILE *f)
Returns the size of the specified file.
Definition: File.cpp:171
size_t readPartialPage(FILE *f, const size_t pageSize, const size_t offset, const size_t readSize, const size_t pageNum, int8_t *buf)
Definition: File.cpp:144
FILE * open(int fileId)
Opens/creates the file with the given id; returns NULL on error.
Definition: File.cpp:83
size_t write(FILE *f, const size_t offset, const size_t size, int8_t *buf)
Writes the specified number of bytes to the offset position in file f from buf.
Definition: File.cpp:121
#define CHECK(condition)
Definition: Logger.h:187
void close(FILE *f)
Closes the file pointed to by the FILE pointer.
Definition: File.cpp:102
bool removeFile(const std::string basePath, const std::string filename)
Deletes the file pointed to by the FILE pointer.
Definition: File.cpp:108
void file_delete(std::atomic< bool > &program_is_running, const unsigned int wait_interval_seconds, const std::string base_path)
Definition: File.cpp:228
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_<EPOCH>_<oldname>.
Definition: File.cpp:183
A selection of helper methods for File I/O.