OmniSciDB  eee9fa949c
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
File.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
23 #include "File.h"
24 #include <boost/filesystem.hpp>
25 #include <cstdio>
26 #include <cstring>
27 #include <iostream>
28 #include <stdexcept>
29 #include <string>
30 #include "Logger.h"
31 
32 namespace File_Namespace {
33 
34 FILE* create(const std::string& basePath,
35  const int fileId,
36  const size_t pageSize,
37  const size_t numPages) {
38  std::string path(basePath + "/" + std::to_string(fileId) + "." +
39  std::to_string(pageSize) +
40  std::string(MAPD_FILE_EXT)); // MAPD_FILE_EXT has preceding "."
41  if (numPages < 1 || pageSize < 1) {
42  LOG(FATAL) << "Error trying to create file '" << path
43  << "', Number of pages and page size must be positive integers. numPages "
44  << numPages << " pageSize " << pageSize;
45  }
46  FILE* f = fopen(path.c_str(), "w+b");
47  if (f == nullptr) {
48  LOG(FATAL) << "Error trying to create file '" << path
49  << "', the error was: " << std::strerror(errno);
50  ;
51  }
52  fseek(f, (pageSize * numPages) - 1, SEEK_SET);
53  fputc(EOF, f);
54  fseek(f, 0, SEEK_SET); // rewind
55  if (fileSize(f) != pageSize * numPages) {
56  LOG(FATAL) << "Error trying to create file '" << path << "', file size "
57  << fileSize(f) << " does not equal pageSize * numPages "
58  << pageSize * numPages;
59  }
60 
61  return f;
62 }
63 
64 FILE* create(const std::string& fullPath, const size_t requestedFileSize) {
65  FILE* f = fopen(fullPath.c_str(), "w+b");
66  if (f == nullptr) {
67  LOG(FATAL) << "Error trying to create file '" << fullPath
68  << "', the error was: " << std::strerror(errno);
69  ;
70  }
71  fseek(f, requestedFileSize - 1, SEEK_SET);
72  fputc(EOF, f);
73  fseek(f, 0, SEEK_SET); // rewind
74  if (fileSize(f) != requestedFileSize) {
75  LOG(FATAL) << "Error trying to create file '" << fullPath << "', file size "
76  << fileSize(f) << " does not equal requestedFileSize "
77  << requestedFileSize;
78  }
79  return f;
80 }
81 
82 FILE* open(int fileId) {
83  std::string s(std::to_string(fileId) + std::string(MAPD_FILE_EXT));
84  FILE* f = fopen(s.c_str(), "r+b"); // opens existing file for updates
85  if (f == nullptr) {
86  LOG(FATAL) << "Error trying to open file '" << s
87  << "', the error was: " << std::strerror(errno);
88  }
89  return f;
90 }
91 
92 FILE* open(const std::string& path) {
93  FILE* f = fopen(path.c_str(), "r+b"); // opens existing file for updates
94  if (f == nullptr) {
95  LOG(FATAL) << "Error trying to open file '" << path
96  << "', the errno was: " << std::strerror(errno);
97  }
98  return f;
99 }
100 
101 void close(FILE* f) {
102  CHECK(f);
103  CHECK_EQ(fflush(f), 0);
104  CHECK_EQ(fclose(f), 0);
105 }
106 
107 bool removeFile(const std::string basePath, const std::string filename) {
108  const std::string filePath = basePath + filename;
109  return remove(filePath.c_str()) == 0;
110 }
111 
112 size_t read(FILE* f, const size_t offset, const size_t size, int8_t* buf) {
113  // read "size" bytes from the offset location in the file into the buffer
114  CHECK_EQ(fseek(f, offset, SEEK_SET), 0);
115  size_t bytesRead = fread(buf, sizeof(int8_t), size, f);
116  CHECK_EQ(bytesRead, sizeof(int8_t) * size);
117  return bytesRead;
118 }
119 
120 size_t write(FILE* f, const size_t offset, const size_t size, int8_t* buf) {
121  // write size bytes from the buffer to the offset location in the file
122  if (fseek(f, offset, SEEK_SET) != 0) {
123  LOG(FATAL)
124  << "Error trying to write to file (during positioning seek) the error was: "
125  << std::strerror(errno);
126  }
127  size_t bytesWritten = fwrite(buf, sizeof(int8_t), size, f);
128  if (bytesWritten != sizeof(int8_t) * size) {
129  LOG(FATAL) << "Error trying to write to file (during fwrite) the error was: "
130  << std::strerror(errno);
131  }
132  return bytesWritten;
133 }
134 
135 size_t append(FILE* f, const size_t size, int8_t* buf) {
136  return write(f, fileSize(f), size, buf);
137 }
138 
139 size_t readPage(FILE* f, const size_t pageSize, const size_t pageNum, int8_t* buf) {
140  return read(f, pageNum * pageSize, pageSize, buf);
141 }
142 
143 size_t readPartialPage(FILE* f,
144  const size_t pageSize,
145  const size_t offset,
146  const size_t readSize,
147  const size_t pageNum,
148  int8_t* buf) {
149  return read(f, pageNum * pageSize + offset, readSize, buf);
150 }
151 
152 size_t writePage(FILE* f, const size_t pageSize, const size_t pageNum, int8_t* buf) {
153  return write(f, pageNum * pageSize, pageSize, buf);
154 }
155 
156 size_t writePartialPage(FILE* f,
157  const size_t pageSize,
158  const size_t offset,
159  const size_t writeSize,
160  const size_t pageNum,
161  int8_t* buf) {
162  return write(f, pageNum * pageSize + offset, writeSize, buf);
163 }
164 
165 size_t appendPage(FILE* f, const size_t pageSize, int8_t* buf) {
166  return write(f, fileSize(f), pageSize, buf);
167 }
168 
170 size_t fileSize(FILE* f) {
171  fseek(f, 0, SEEK_END);
172  size_t size = (size_t)ftell(f);
173  fseek(f, 0, SEEK_SET);
174  return size;
175 }
176 
177 // this is a helper function to rename existing directories
178 // allowing for an async process to actually remove the physical directries
179 // and subfolders and files later
180 // it is required due to the large amount of time it can take to delete
181 // physical files from large disks
182 void renameForDelete(const std::string directoryName) {
184  boost::filesystem::path directoryPath(directoryName);
185  using namespace std::chrono;
186  milliseconds ms = duration_cast<milliseconds>(system_clock::now().time_since_epoch());
187 
188  if (boost::filesystem::exists(directoryPath) &&
189  boost::filesystem::is_directory(directoryPath)) {
190  boost::filesystem::path newDirectoryPath(directoryName + "_" +
191  std::to_string(ms.count()) + "_DELETE_ME");
192  boost::filesystem::rename(directoryPath, newDirectoryPath, ec);
193 
194  if (ec.value() == boost::system::errc::success) {
195  std::thread th([newDirectoryPath]() {
197  boost::filesystem::remove_all(newDirectoryPath, ec);
198  if (ec.value() != boost::system::errc::success) {
199  LOG(ERROR) << "Failed to remove directory " << newDirectoryPath << " error was "
200  << ec;
201  }
202  });
203  // let it run free so we can return
204  // if it fails the file_delete_thread in MapDHandler will clean up
205  th.detach();
206 
207  return;
208  }
209 
210  LOG(FATAL) << "Failed to rename file " << directoryName << " to "
211  << directoryName + "_" + std::to_string(ms.count()) + "_DELETE_ME Error: "
212  << ec;
213  }
214 }
215 
216 } // namespace File_Namespace
217 
218 // Still temporary location but avoids the link errors in the new distributed branch.
219 // See the comment file_delete.h
220 
221 #include <atomic>
222 #include <boost/algorithm/string/predicate.hpp>
223 #include <boost/filesystem.hpp>
224 #include <chrono>
225 #include <thread>
226 
227 void file_delete(std::atomic<bool>& program_is_running,
228  const unsigned int wait_interval_seconds,
229  const std::string base_path) {
230  const auto wait_duration = std::chrono::seconds(wait_interval_seconds);
231  const boost::filesystem::path path(base_path);
232  while (program_is_running) {
233  using vec = std::vector<boost::filesystem::path>; // store paths,
234  vec v;
236 
237  // copy vector from iterator as was getting weird random errors if
238  // removed direct from iterator
239  copy(boost::filesystem::directory_iterator(path),
240  boost::filesystem::directory_iterator(),
241  back_inserter(v));
242  for (vec::const_iterator it(v.begin()); it != v.end(); ++it) {
243  std::string object_name(it->string());
244 
245  if (boost::algorithm::ends_with(object_name, "DELETE_ME")) {
246  LOG(INFO) << " removing object " << object_name;
247  boost::filesystem::remove_all(*it, ec);
248  if (ec.value() != boost::system::errc::success) {
249  LOG(ERROR) << "Failed to remove object " << object_name << " error was " << ec;
250  }
251  }
252  }
253 
254  std::this_thread::sleep_for(wait_duration);
255  }
256 }
size_t appendPage(FILE *f, const size_t pageSize, int8_t *buf)
Appends a page from buf to the file.
Definition: File.cpp:165
#define CHECK_EQ(x, y)
Definition: Logger.h:205
std::string filename(char const *path)
Definition: Logger.cpp:62
#define LOG(tag)
Definition: Logger.h:188
size_t writePartialPage(FILE *f, const size_t pageSize, const size_t offset, const size_t writeSize, const size_t pageNum, int8_t *buf)
Definition: File.cpp:156
#define MAPD_FILE_EXT
Definition: File.h:26
size_t writePage(FILE *f, const size_t pageSize, const size_t pageNum, int8_t *buf)
Writes a page from buf to the file.
Definition: File.cpp:152
std::string to_string(char const *&&v)
FILE * create(const std::string &basePath, const int fileId, const size_t pageSize, const size_t numPages)
Definition: File.cpp:34
size_t read(FILE *f, const size_t offset, const size_t size, int8_t *buf)
Reads the specified number of bytes from the offset position in file f into buf.
Definition: File.cpp:112
CHECK(cgen_state)
const int8_t const int64_t const uint64_t const int32_t const int64_t int64_t uint32_t const int64_t int32_t * error_code
size_t append(FILE *f, const size_t size, int8_t *buf)
Appends the specified number of bytes to the end of the file f from buf.
Definition: File.cpp:135
size_t readPage(FILE *f, const size_t pageSize, const size_t pageNum, int8_t *buf)
Reads the specified page from the file f into buf.
Definition: File.cpp:139
size_t fileSize(FILE *f)
Returns the size of the specified file.
Definition: File.cpp:170
size_t readPartialPage(FILE *f, const size_t pageSize, const size_t offset, const size_t readSize, const size_t pageNum, int8_t *buf)
Definition: File.cpp:143
FILE * open(int fileId)
Opens/creates the file with the given id; returns NULL on error.
Definition: File.cpp:82
size_t write(FILE *f, const size_t offset, const size_t size, int8_t *buf)
Writes the specified number of bytes to the offset position in file f from buf.
Definition: File.cpp:120
void close(FILE *f)
Closes the file pointed to by the FILE pointer.
Definition: File.cpp:101
bool removeFile(const std::string basePath, const std::string filename)
Deletes the file pointed to by the FILE pointer.
Definition: File.cpp:107
void file_delete(std::atomic< bool > &program_is_running, const unsigned int wait_interval_seconds, const std::string base_path)
Definition: File.cpp:227
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_&lt;EPOCH&gt;_&lt;oldname&gt;.
Definition: File.cpp:182
A selection of helper methods for File I/O.