OmniSciDB  04ee39c94c
FileMgr.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
23 #include "FileMgr.h"
24 #include <boost/filesystem.hpp>
25 #include <boost/lexical_cast.hpp>
26 #include <boost/system/error_code.hpp>
27 #include <string>
28 #include "GlobalFileMgr.h"
29 #include "Shared/File.h"
30 #include "Shared/measure.h"
31 
32 #include <fcntl.h>
33 #include <unistd.h>
34 #include <algorithm>
35 #include <future>
36 #include <thread>
37 #include <utility>
38 #include <vector>
39 
40 #define EPOCH_FILENAME "epoch"
41 #define DB_META_FILENAME "dbmeta"
42 
43 using namespace std;
44 
45 namespace File_Namespace {
46 
47 bool headerCompare(const HeaderInfo& firstElem, const HeaderInfo& secondElem) {
48  // HeaderInfo.first is a pair of Chunk key with a vector containing
49  // pageId and version
50  if (firstElem.chunkKey != secondElem.chunkKey) {
51  return firstElem.chunkKey < secondElem.chunkKey;
52  }
53  if (firstElem.pageId != secondElem.pageId) {
54  return firstElem.pageId < secondElem.pageId;
55  }
56  return firstElem.versionEpoch < secondElem.versionEpoch;
57 
58  /*
59  if (firstElem.first.first != secondElem.first.first)
60  return firstElem.first.first < secondElem.first.first;
61  return firstElem.first.second < secondElem.first.second;
62  */
63 }
64 
65 FileMgr::FileMgr(const int deviceId,
66  GlobalFileMgr* gfm,
67  const std::pair<const int, const int> fileMgrKey,
68  const size_t num_reader_threads,
69  const int epoch,
70  const size_t defaultPageSize)
71  : AbstractBufferMgr(deviceId)
72  , gfm_(gfm)
73  , fileMgrKey_(fileMgrKey)
74  , defaultPageSize_(defaultPageSize)
75  , nextFileId_(0)
76  , epoch_(epoch) {
77  init(num_reader_threads);
78 }
79 
80 // used only to initialize enough to drop
81 FileMgr::FileMgr(const int deviceId,
82  GlobalFileMgr* gfm,
83  const std::pair<const int, const int> fileMgrKey,
84  const bool initOnly)
85  : AbstractBufferMgr(deviceId)
86  , gfm_(gfm)
87  , fileMgrKey_(fileMgrKey)
88  , defaultPageSize_(0)
89  , nextFileId_(0)
90  , epoch_(0) {
91  const std::string fileMgrDirPrefix("table");
92  const std::string FileMgrDirDelim("_");
93  fileMgrBasePath_ = (gfm_->getBasePath() + fileMgrDirPrefix + FileMgrDirDelim +
94  std::to_string(fileMgrKey_.first) + // db_id
95  FileMgrDirDelim + std::to_string(fileMgrKey_.second)); // tb_id
96  epochFile_ = nullptr;
97  files_.clear();
98 }
99 
100 FileMgr::FileMgr(GlobalFileMgr* gfm, const size_t defaultPageSize, std::string basePath)
101  : AbstractBufferMgr(0)
102  , gfm_(gfm)
103  , fileMgrKey_(0, 0)
104  , fileMgrBasePath_(basePath)
105  , defaultPageSize_(defaultPageSize)
106  , nextFileId_(0)
107  , epoch_(-1) {
108  init(basePath);
109 }
110 
112  // checkpoint();
113  // free memory used by FileInfo objects
114  for (auto chunkIt = chunkIndex_.begin(); chunkIt != chunkIndex_.end(); ++chunkIt) {
115  delete chunkIt->second;
116  }
117  for (auto file_info : files_) {
118  delete file_info;
119  }
120 }
121 
122 void FileMgr::init(const size_t num_reader_threads) {
123  // if epoch = -1 this means open from epoch file
124  const std::string fileMgrDirPrefix("table");
125  const std::string FileMgrDirDelim("_");
126  fileMgrBasePath_ = (gfm_->getBasePath() + fileMgrDirPrefix + FileMgrDirDelim +
127  std::to_string(fileMgrKey_.first) + // db_id
128  FileMgrDirDelim + std::to_string(fileMgrKey_.second)); // tb_id
129  boost::filesystem::path path(fileMgrBasePath_);
130  if (boost::filesystem::exists(path)) {
131  if (!boost::filesystem::is_directory(path)) {
132  LOG(FATAL) << "Specified path '" << fileMgrBasePath_
133  << "' for table data is not a directory.";
134  }
135  if (epoch_ != -1) { // if opening at previous epoch
136  int epochCopy = epoch_;
138  epoch_ = epochCopy;
139  } else {
141  }
142 
143  auto clock_begin = timer_start();
144 
145  boost::filesystem::directory_iterator
146  endItr; // default construction yields past-the-end
147  int maxFileId = -1;
148  int fileCount = 0;
149  int threadCount = std::thread::hardware_concurrency();
150  std::vector<HeaderInfo> headerVec;
151  std::vector<std::future<std::vector<HeaderInfo>>> file_futures;
152  for (boost::filesystem::directory_iterator fileIt(path); fileIt != endItr; ++fileIt) {
153  if (boost::filesystem::is_regular_file(fileIt->status())) {
154  // note that boost::filesystem leaves preceding dot on
155  // extension - hence MAPD_FILE_EXT is ".mapd"
156  std::string extension(fileIt->path().extension().string());
157 
158  if (extension == MAPD_FILE_EXT) {
159  std::string fileStem(fileIt->path().stem().string());
160  // remove trailing dot if any
161  if (fileStem.size() > 0 && fileStem.back() == '.') {
162  fileStem = fileStem.substr(0, fileStem.size() - 1);
163  }
164  size_t dotPos = fileStem.find_last_of("."); // should only be one
165  if (dotPos == std::string::npos) {
166  LOG(FATAL) << "File `" << fileIt->path()
167  << "` does not carry page size information in the filename.";
168  }
169  int fileId = boost::lexical_cast<int>(fileStem.substr(0, dotPos));
170  if (fileId > maxFileId) {
171  maxFileId = fileId;
172  }
173  size_t pageSize =
174  boost::lexical_cast<size_t>(fileStem.substr(dotPos + 1, fileStem.size()));
175  std::string filePath(fileIt->path().string());
176  size_t fileSize = boost::filesystem::file_size(filePath);
177  assert(fileSize % pageSize == 0); // should be no partial pages
178  size_t numPages = fileSize / pageSize;
179 
180  VLOG(4) << "File id: " << fileId << " Page size: " << pageSize
181  << " Num pages: " << numPages;
182 
183  file_futures.emplace_back(std::async(
184  std::launch::async, [filePath, fileId, pageSize, numPages, this] {
185  std::vector<HeaderInfo> tempHeaderVec;
186  openExistingFile(filePath, fileId, pageSize, numPages, tempHeaderVec);
187  return tempHeaderVec;
188  }));
189  fileCount++;
190  if (fileCount % threadCount == 0) {
191  processFileFutures(file_futures, headerVec);
192  }
193  }
194  }
195  }
196 
197  if (file_futures.size() > 0) {
198  processFileFutures(file_futures, headerVec);
199  }
200  int64_t queue_time_ms = timer_stop(clock_begin);
201 
202  LOG(INFO) << "Completed Reading table's file metadata, Elapsed time : "
203  << queue_time_ms << "ms Epoch: " << epoch_ << " files read: " << fileCount
204  << " table location: '" << fileMgrBasePath_ << "'";
205 
206  /* Sort headerVec so that all HeaderInfos
207  * from a chunk will be grouped together
208  * and in order of increasing PageId
209  * - Version Epoch */
210 
211  std::sort(headerVec.begin(), headerVec.end(), headerCompare);
212 
213  /* Goal of next section is to find sequences in the
214  * sorted headerVec of the same ChunkId, which we
215  * can then initiate a FileBuffer with */
216 
217  VLOG(4) << "Number of Headers in Vector: " << headerVec.size();
218  if (headerVec.size() > 0) {
219  ChunkKey lastChunkKey = headerVec.begin()->chunkKey;
220  auto startIt = headerVec.begin();
221 
222  for (auto headerIt = headerVec.begin() + 1; headerIt != headerVec.end();
223  ++headerIt) {
224  // for (auto chunkIt = headerIt->chunkKey.begin(); chunkIt !=
225  // headerIt->chunkKey.end(); ++chunkIt) {
226  // std::cout << *chunkIt << " ";
227  //}
228 
229  if (headerIt->chunkKey != lastChunkKey) {
230  chunkIndex_[lastChunkKey] =
231  new FileBuffer(this, /*pageSize,*/ lastChunkKey, startIt, headerIt);
232  /*
233  if (startIt->versionEpoch != -1) {
234  cout << "not skipping bc version != -1" << endl;
235  // -1 means that chunk was deleted
236  // lets not read it in
237  chunkIndex_[lastChunkKey] = new FileBuffer
238  (this,/lastChunkKey,startIt,headerIt);
239 
240  }
241  else {
242  cout << "Skipping bc version == -1" << endl;
243  }
244  */
245  lastChunkKey = headerIt->chunkKey;
246  startIt = headerIt;
247  }
248  }
249  // now need to insert last Chunk
250  // size_t pageSize = files_[startIt->page.fileId]->pageSize;
251  // cout << "Inserting last chunk" << endl;
252  // if (startIt->versionEpoch != -1) {
253  chunkIndex_[lastChunkKey] =
254  new FileBuffer(this, /*pageSize,*/ lastChunkKey, startIt, headerVec.end());
255  //}
256  }
257  nextFileId_ = maxFileId + 1;
258  // std::cout << "next file id: " << nextFileId_ << std::endl;
259  } else {
260  if (!boost::filesystem::create_directory(path)) {
261  LOG(FATAL) << "Could not create data directory: " << path;
262  }
264  }
265 
266  /* define number of reader threads to be used */
267  size_t num_hardware_based_threads =
268  std::thread::hardware_concurrency(); // # of threads is based on # of cores on the
269  // host
270  if (num_reader_threads == 0) { // # of threads has not been defined by user
271  num_reader_threads_ = num_hardware_based_threads;
272  } else {
273  if (num_reader_threads > num_hardware_based_threads) {
274  num_reader_threads_ = num_hardware_based_threads;
275  } else {
276  num_reader_threads_ = num_reader_threads;
277  }
278  }
279 }
280 
282  std::vector<std::future<std::vector<HeaderInfo>>>& file_futures,
283  std::vector<HeaderInfo>& headerVec) {
284  for (auto& file_future : file_futures) {
285  file_future.wait();
286  }
287  // concatenate the vectors after thread completes
288  for (auto& file_future : file_futures) {
289  auto tempHeaderVec = file_future.get();
290  headerVec.insert(headerVec.end(), tempHeaderVec.begin(), tempHeaderVec.end());
291  }
292  file_futures.clear();
293 }
294 
295 void FileMgr::init(const std::string dataPathToConvertFrom) {
296  int converted_data_epoch = 0;
297  boost::filesystem::path path(dataPathToConvertFrom);
298  if (boost::filesystem::exists(path)) {
299  if (!boost::filesystem::is_directory(path)) {
300  LOG(FATAL) << "Specified path `" << path << "` is not a directory.";
301  }
302 
303  if (epoch_ != -1) { // if opening at previous epoch
304  int epochCopy = epoch_;
306  epoch_ = epochCopy;
307  } else {
309  }
310 
311  boost::filesystem::directory_iterator
312  endItr; // default construction yields past-the-end
313  int maxFileId = -1;
314  int fileCount = 0;
315  int threadCount = std::thread::hardware_concurrency();
316  std::vector<HeaderInfo> headerVec;
317  std::vector<std::future<std::vector<HeaderInfo>>> file_futures;
318  for (boost::filesystem::directory_iterator fileIt(path); fileIt != endItr; ++fileIt) {
319  if (boost::filesystem::is_regular_file(fileIt->status())) {
320  // note that boost::filesystem leaves preceding dot on
321  // extension - hence MAPD_FILE_EXT is ".mapd"
322  std::string extension(fileIt->path().extension().string());
323 
324  if (extension == MAPD_FILE_EXT) {
325  std::string fileStem(fileIt->path().stem().string());
326  // remove trailing dot if any
327  if (fileStem.size() > 0 && fileStem.back() == '.') {
328  fileStem = fileStem.substr(0, fileStem.size() - 1);
329  }
330  size_t dotPos = fileStem.find_last_of("."); // should only be one
331  if (dotPos == std::string::npos) {
332  LOG(FATAL) << "File `" + fileIt->path().string() +
333  "` does not carry page size information in the filename.";
334  }
335  int fileId = boost::lexical_cast<int>(fileStem.substr(0, dotPos));
336  if (fileId > maxFileId) {
337  maxFileId = fileId;
338  }
339  size_t pageSize =
340  boost::lexical_cast<size_t>(fileStem.substr(dotPos + 1, fileStem.size()));
341  std::string filePath(fileIt->path().string());
342  size_t fileSize = boost::filesystem::file_size(filePath);
343  assert(fileSize % pageSize == 0); // should be no partial pages
344  size_t numPages = fileSize / pageSize;
345 
346  file_futures.emplace_back(std::async(
347  std::launch::async, [filePath, fileId, pageSize, numPages, this] {
348  std::vector<HeaderInfo> tempHeaderVec;
349  openExistingFile(filePath, fileId, pageSize, numPages, tempHeaderVec);
350  return tempHeaderVec;
351  }));
352  fileCount++;
353  if (fileCount % threadCount) {
354  processFileFutures(file_futures, headerVec);
355  }
356  }
357  }
358  }
359 
360  if (file_futures.size() > 0) {
361  processFileFutures(file_futures, headerVec);
362  }
363 
364  /* Sort headerVec so that all HeaderInfos
365  * from a chunk will be grouped together
366  * and in order of increasing PageId
367  * - Version Epoch */
368 
369  std::sort(headerVec.begin(), headerVec.end(), headerCompare);
370 
371  /* Goal of next section is to find sequences in the
372  * sorted headerVec of the same ChunkId, which we
373  * can then initiate a FileBuffer with */
374 
375  if (headerVec.size() > 0) {
376  ChunkKey lastChunkKey = headerVec.begin()->chunkKey;
377  auto startIt = headerVec.begin();
378 
379  for (auto headerIt = headerVec.begin() + 1; headerIt != headerVec.end();
380  ++headerIt) {
381  if (headerIt->chunkKey != lastChunkKey) {
382  FileMgr* c_fm_ = gfm_->getFileMgr(lastChunkKey);
383  FileBuffer* srcBuf = new FileBuffer(this, lastChunkKey, startIt, headerIt);
384  chunkIndex_[lastChunkKey] = srcBuf;
385  FileBuffer* destBuf = new FileBuffer(c_fm_, srcBuf->pageSize(), lastChunkKey);
386  c_fm_->chunkIndex_[lastChunkKey] = destBuf;
387  destBuf->syncEncoder(srcBuf);
388  destBuf->setSize(srcBuf->size());
389  destBuf->setDirty(); // this needs to be set to force writing out metadata
390  // files from "checkpoint()" call
391 
392  size_t totalNumPages = srcBuf->getMultiPage().size();
393  for (size_t pageNum = 0; pageNum < totalNumPages; pageNum++) {
394  Page srcPage = srcBuf->getMultiPage()[pageNum].current();
395  Page destPage = c_fm_->requestFreePage(
396  srcBuf->pageSize(),
397  false); // may modify and use api "FileBuffer::addNewMultiPage" instead
398  MultiPage multiPage(srcBuf->pageSize());
399  multiPage.epochs.push_back(converted_data_epoch);
400  multiPage.pageVersions.push_back(destPage);
401  destBuf->multiPages_.push_back(multiPage);
402  size_t reservedHeaderSize = srcBuf->reservedHeaderSize();
403  copyPage(
404  srcPage, c_fm_, destPage, reservedHeaderSize, srcBuf->pageDataSize(), 0);
405  destBuf->writeHeader(destPage, pageNum, converted_data_epoch, false);
406  }
407  lastChunkKey = headerIt->chunkKey;
408  startIt = headerIt;
409  }
410  }
411 
412  // now need to insert last Chunk
413  FileMgr* c_fm_ = gfm_->getFileMgr(lastChunkKey);
414  FileBuffer* srcBuf = new FileBuffer(this, lastChunkKey, startIt, headerVec.end());
415  chunkIndex_[lastChunkKey] = srcBuf;
416  FileBuffer* destBuf = new FileBuffer(c_fm_, srcBuf->pageSize(), lastChunkKey);
417  c_fm_->chunkIndex_[lastChunkKey] = destBuf;
418  destBuf->syncEncoder(srcBuf);
419  destBuf->setSize(srcBuf->size());
420  destBuf->setDirty(); // this needs to be set to write out metadata file from the
421  // "checkpoint()" call
422 
423  size_t totalNumPages = srcBuf->getMultiPage().size();
424  for (size_t pageNum = 0; pageNum < totalNumPages; pageNum++) {
425  Page srcPage = srcBuf->getMultiPage()[pageNum].current();
426  Page destPage = c_fm_->requestFreePage(
427  srcBuf->pageSize(),
428  false); // may modify and use api "FileBuffer::addNewMultiPage" instead
429  MultiPage multiPage(srcBuf->pageSize());
430  multiPage.epochs.push_back(converted_data_epoch);
431  multiPage.pageVersions.push_back(destPage);
432  destBuf->multiPages_.push_back(multiPage);
433  size_t reservedHeaderSize = srcBuf->reservedHeaderSize();
434  copyPage(srcPage, c_fm_, destPage, reservedHeaderSize, srcBuf->pageDataSize(), 0);
435  destBuf->writeHeader(destPage, pageNum, converted_data_epoch, false);
436  }
437  }
438  nextFileId_ = maxFileId + 1;
439  } else {
440  if (!boost::filesystem::create_directory(path)) {
441  LOG(FATAL) << "Specified path does not exist: " << path;
442  }
443  }
444 }
445 
447  for (auto file_info : files_) {
448  if (file_info->f) {
449  close(file_info->f);
450  file_info->f = nullptr;
451  }
452  }
453 
454  if (epochFile_) {
455  close(epochFile_);
456  epochFile_ = nullptr;
457  }
458 
459  /* rename for later deletion the directory containing table related data */
461 }
462 
463 void FileMgr::copyPage(Page& srcPage,
464  FileMgr* destFileMgr,
465  Page& destPage,
466  const size_t reservedHeaderSize,
467  const size_t numBytes,
468  const size_t offset) {
469  CHECK(offset + numBytes <= defaultPageSize_);
470  FileInfo* srcFileInfo = getFileInfoForFileId(srcPage.fileId);
471  FileInfo* destFileInfo = destFileMgr->getFileInfoForFileId(destPage.fileId);
472  int8_t* buffer = new int8_t[numBytes];
473 
474  size_t bytesRead = srcFileInfo->read(
475  srcPage.pageNum * defaultPageSize_ + offset + reservedHeaderSize, numBytes, buffer);
476  CHECK(bytesRead == numBytes);
477  size_t bytesWritten = destFileInfo->write(
478  destPage.pageNum * defaultPageSize_ + offset + reservedHeaderSize,
479  numBytes,
480  buffer);
481  CHECK(bytesWritten == numBytes);
482  delete[] buffer;
483 }
484 
485 void FileMgr::createEpochFile(const std::string& epochFileName) {
486  std::string epochFilePath(fileMgrBasePath_ + "/" + epochFileName);
487  if (boost::filesystem::exists(epochFilePath)) {
488  LOG(FATAL) << "Epoch file `" << epochFilePath << "` already exists";
489  }
490  epochFile_ = create(epochFilePath, sizeof(int));
491  // Write out current epoch to file - which if this
492  // function is being called should be 0
493  write(epochFile_, 0, sizeof(int), (int8_t*)&epoch_);
494  epoch_++;
495 }
496 
497 void FileMgr::openEpochFile(const std::string& epochFileName) {
498  std::string epochFilePath(fileMgrBasePath_ + "/" + epochFileName);
499  if (!boost::filesystem::exists(epochFilePath)) {
500  LOG(FATAL) << "Epoch file `" << epochFilePath << "` does not exist";
501  }
502  if (!boost::filesystem::is_regular_file(epochFilePath)) {
503  LOG(FATAL) << "Epoch file `" << epochFilePath << "` is not a regular file";
504  }
505  if (boost::filesystem::file_size(epochFilePath) < 4) {
506  LOG(FATAL) << "Epoch file `" << epochFilePath
507  << "` is not sized properly (current size: "
508  << boost::filesystem::file_size(epochFilePath) << ", expected size: 4)";
509  }
510  epochFile_ = open(epochFilePath);
511  read(epochFile_, 0, sizeof(int), (int8_t*)&epoch_);
512  // std::cout << "Epoch after open file: " << epoch_ << std::endl;
513  epoch_++; // we are in new epoch from last checkpoint
514 }
515 
517  write(epochFile_, 0, sizeof(int), (int8_t*)&epoch_);
518  int status = fflush(epochFile_);
519  // int status = fcntl(fileno(epochFile_),51);
520  if (status != 0) {
521  LOG(FATAL) << "Could not flush epoch file to disk";
522  }
523 #ifdef __APPLE__
524  status = fcntl(fileno(epochFile_), 51);
525 #else
526  status = fsync(fileno(epochFile_));
527 #endif
528  if (status != 0) {
529  LOG(FATAL) << "Could not sync epoch file to disk";
530  }
531  ++epoch_;
532 }
533 
534 void FileMgr::createDBMetaFile(const std::string& DBMetaFileName) {
535  std::string DBMetaFilePath(fileMgrBasePath_ + "/" + DBMetaFileName);
536  if (boost::filesystem::exists(DBMetaFilePath)) {
537  LOG(FATAL) << "DB metadata file `" << DBMetaFilePath << "` already exists.";
538  }
539  DBMetaFile_ = create(DBMetaFilePath, sizeof(int));
540  int db_ver = getDBVersion();
541  write(DBMetaFile_, 0, sizeof(int), (int8_t*)&db_ver);
542  // LOG(INFO) << "DB metadata file has been created.";
543 }
544 
545 bool FileMgr::openDBMetaFile(const std::string& DBMetaFileName) {
546  std::string DBMetaFilePath(fileMgrBasePath_ + "/" + DBMetaFileName);
547 
548  if (!boost::filesystem::exists(DBMetaFilePath)) {
549  // LOG(INFO) << "DB metadata file does not exist, one will be created.";
550  return false;
551  }
552  if (!boost::filesystem::is_regular_file(DBMetaFilePath)) {
553  // LOG(INFO) << "DB metadata file is not a regular file, one will be created.";
554  return false;
555  }
556  if (boost::filesystem::file_size(DBMetaFilePath) < 4) {
557  // LOG(INFO) << "DB metadata file is not sized properly, one will be created.";
558  return false;
559  }
560  DBMetaFile_ = open(DBMetaFilePath);
561  read(DBMetaFile_, 0, sizeof(int), (int8_t*)&db_version_);
562 
563  return true;
564 }
565 
567  int db_ver = getDBVersion();
568  write(DBMetaFile_, 0, sizeof(int), (int8_t*)&db_ver);
569  int status = fflush(DBMetaFile_);
570  if (status != 0) {
571  LOG(FATAL) << "Could not sync DB metadata file to disk";
572  }
573 }
574 
576  VLOG(2) << "Checkpointing epoch: " << epoch_;
577  mapd_unique_lock<mapd_shared_mutex> chunkIndexWriteLock(chunkIndexMutex_);
578  for (auto chunkIt = chunkIndex_.begin(); chunkIt != chunkIndex_.end(); ++chunkIt) {
579  /*
580  for (auto vecIt = chunkIt->first.begin(); vecIt != chunkIt->first.end(); ++vecIt) {
581  std::cout << *vecIt << ",";
582  }
583  cout << "Is dirty: " << chunkIt->second->isDirty_ << endl;
584  */
585  if (chunkIt->second->isDirty_) {
586  chunkIt->second->writeMetadata(epoch_);
587  chunkIt->second->clearDirtyBits();
588  }
589  }
590  chunkIndexWriteLock.unlock();
591 
592  mapd_shared_lock<mapd_shared_mutex> read_lock(files_rw_mutex_);
593  for (auto fileIt = files_.begin(); fileIt != files_.end(); ++fileIt) {
594  int status = (*fileIt)->syncToDisk();
595  if (status != 0) {
596  LOG(FATAL) << "Could not sync file to disk";
597  }
598  }
599 
601 
602  mapd_unique_lock<mapd_shared_mutex> freePagesWriteLock(mutex_free_page);
603  for (auto& free_page : free_pages) {
604  free_page.first->freePageDeferred(free_page.second);
605  }
606  free_pages.clear();
607 }
608 
610  const size_t pageSize,
611  const size_t numBytes) {
612  size_t actualPageSize = pageSize;
613  if (actualPageSize == 0) {
614  actualPageSize = defaultPageSize_;
615  }
617  // we will do this lazily and not allocate space for the Chunk (i.e.
618  // FileBuffer yet)
619  mapd_unique_lock<mapd_shared_mutex> chunkIndexWriteLock(chunkIndexMutex_);
620 
621  if (chunkIndex_.find(key) != chunkIndex_.end()) {
622  LOG(FATAL) << "Chunk already exists for key: " << showChunk(key);
623  }
624  chunkIndex_[key] = new FileBuffer(this, actualPageSize, key, numBytes);
625  chunkIndexWriteLock.unlock();
626  return (chunkIndex_[key]);
627 }
628 
630  mapd_shared_lock<mapd_shared_mutex> chunkIndexReadLock(chunkIndexMutex_);
631  return chunkIndex_.find(key) != chunkIndex_.end();
632 }
633 
634 void FileMgr::deleteBuffer(const ChunkKey& key, const bool purge) {
635  mapd_unique_lock<mapd_shared_mutex> chunkIndexWriteLock(chunkIndexMutex_);
636  auto chunkIt = chunkIndex_.find(key);
637  // ensure the Chunk exists
638  if (chunkIt == chunkIndex_.end()) {
639  LOG(FATAL) << "Chunk does not exist for key: " << showChunk(key);
640  }
641  chunkIndexWriteLock.unlock();
642  // chunkIt->second->writeMetadata(-1); // writes -1 as epoch - signifies deleted
643  if (purge) {
644  chunkIt->second->freePages();
645  }
646  //@todo need a way to represent delete in non purge case
647  delete chunkIt->second;
648  chunkIndex_.erase(chunkIt);
649 }
650 
651 void FileMgr::deleteBuffersWithPrefix(const ChunkKey& keyPrefix, const bool purge) {
652  mapd_unique_lock<mapd_shared_mutex> chunkIndexWriteLock(chunkIndexMutex_);
653  auto chunkIt = chunkIndex_.lower_bound(keyPrefix);
654  if (chunkIt == chunkIndex_.end()) {
655  return; // should we throw?
656  }
657  while (chunkIt != chunkIndex_.end() &&
658  std::search(chunkIt->first.begin(),
659  chunkIt->first.begin() + keyPrefix.size(),
660  keyPrefix.begin(),
661  keyPrefix.end()) != chunkIt->first.begin() + keyPrefix.size()) {
662  /*
663  cout << "Freeing pages for chunk ";
664  for (auto vecIt = chunkIt->first.begin(); vecIt != chunkIt->first.end(); ++vecIt) {
665  std::cout << *vecIt << ",";
666  }
667  cout << endl;
668  */
669  if (purge) {
670  chunkIt->second->freePages();
671  }
672  //@todo need a way to represent delete in non purge case
673  delete chunkIt->second;
674  chunkIndex_.erase(chunkIt++);
675  }
676 }
677 
678 AbstractBuffer* FileMgr::getBuffer(const ChunkKey& key, const size_t numBytes) {
679  mapd_shared_lock<mapd_shared_mutex> chunkIndexReadLock(chunkIndexMutex_);
680  auto chunkIt = chunkIndex_.find(key);
681  if (chunkIt == chunkIndex_.end()) {
682  LOG(FATAL) << "Chunk does not exist for key: " << showChunk(key);
683  }
684  return chunkIt->second;
685 }
686 
688  AbstractBuffer* destBuffer,
689  const size_t numBytes) {
690  // reads chunk specified by ChunkKey into AbstractBuffer provided by
691  // destBuffer
692  if (destBuffer->isDirty()) {
693  LOG(FATAL)
694  << "Aborting attempt to fetch a chunk marked dirty. Chunk inconsistency for key: "
695  << showChunk(key);
696  }
697  mapd_shared_lock<mapd_shared_mutex> chunkIndexReadLock(chunkIndexMutex_);
698  auto chunkIt = chunkIndex_.find(key);
699  if (chunkIt == chunkIndex_.end()) {
700  LOG(FATAL) << "Chunk does not exist for key: " << showChunk(key);
701  }
702  chunkIndexReadLock.unlock();
703 
704  AbstractBuffer* chunk = chunkIt->second;
705  // ChunkSize is either specified in function call with numBytes or we
706  // just look at pageSize * numPages in FileBuffer
707  size_t chunkSize = numBytes == 0 ? chunk->size() : numBytes;
708  if (numBytes > 0 && numBytes > chunk->size()) {
709  LOG(FATAL) << "Chunk retrieved for key `" << showChunk(key) << "` is smaller ("
710  << chunk->size() << ") than number of bytes requested (" << numBytes
711  << ")";
712  }
713  destBuffer->reserve(chunkSize);
714  // std::cout << "After reserve chunksize: " << chunkSize << std::endl;
715  if (chunk->isUpdated()) {
716  chunk->read(destBuffer->getMemoryPtr(),
717  chunkSize,
718  0,
719  destBuffer->getType(),
720  destBuffer->getDeviceId());
721  } else {
722  chunk->read(destBuffer->getMemoryPtr() + destBuffer->size(),
723  chunkSize - destBuffer->size(),
724  destBuffer->size(),
725  destBuffer->getType(),
726  destBuffer->getDeviceId());
727  }
728  destBuffer->setSize(chunkSize);
729  destBuffer->syncEncoder(chunk);
730 }
731 
733  AbstractBuffer* srcBuffer,
734  const size_t numBytes) {
735  // obtain a pointer to the Chunk
736  mapd_unique_lock<mapd_shared_mutex> chunkIndexWriteLock(chunkIndexMutex_);
737  auto chunkIt = chunkIndex_.find(key);
738  AbstractBuffer* chunk;
739  if (chunkIt == chunkIndex_.end()) {
740  chunk = createBuffer(key, defaultPageSize_);
741  } else {
742  chunk = chunkIt->second;
743  }
744  chunkIndexWriteLock.unlock();
745  size_t oldChunkSize = chunk->size();
746  // write the buffer's data to the Chunk
747  // size_t newChunkSize = numBytes == 0 ? srcBuffer->size() : numBytes;
748  size_t newChunkSize = numBytes == 0 ? srcBuffer->size() : numBytes;
749  if (chunk->isDirty()) {
750  // multiple appends are allowed,
751  // but only single update is allowed
752  if (srcBuffer->isUpdated() && chunk->isUpdated()) {
753  LOG(FATAL) << "Aborting attempt to write a chunk marked dirty. Chunk inconsistency "
754  "for key: "
755  << showChunk(key);
756  }
757  }
758  if (srcBuffer->isUpdated()) {
759  // chunk size is not changed when fixed rows are updated or are marked as deleted.
760  // but when rows are vacuumed or varlen rows are updated (new rows are appended),
761  // chunk size will change. For vacuum, checkpoint should sync size from cpu to disk.
762  // For varlen update, it takes another route via fragmenter using disk-level buffer.
763  if (0 == numBytes && !chunk->isDirty()) {
764  chunk->setSize(newChunkSize);
765  }
766  //@todo use dirty flags to only flush pages of chunk that need to
767  // be flushed
768  chunk->write((int8_t*)srcBuffer->getMemoryPtr(),
769  newChunkSize,
770  0,
771  srcBuffer->getType(),
772  srcBuffer->getDeviceId());
773  } else if (srcBuffer->isAppended()) {
774  assert(oldChunkSize < newChunkSize);
775  chunk->append((int8_t*)srcBuffer->getMemoryPtr() + oldChunkSize,
776  newChunkSize - oldChunkSize,
777  srcBuffer->getType(),
778  srcBuffer->getDeviceId());
779  }
780  // chunk->clearDirtyBits(); // Hack: because write and append will set dirty bits
781  //@todo commenting out line above will make sure this metadata is set
782  // but will trigger error on fetch chunk
783  srcBuffer->clearDirtyBits();
784  chunk->syncEncoder(srcBuffer);
785  return chunk;
786 }
787 
788 AbstractBuffer* FileMgr::alloc(const size_t numBytes = 0) {
789  LOG(FATAL) << "Operation not supported";
790  return nullptr; // satisfy return-type warning
791 }
792 
794  LOG(FATAL) << "Operation not supported";
795 }
796 
797 // AbstractBuffer* FileMgr::putBuffer(AbstractBuffer *d) {
798 // throw std::runtime_error("Operation not supported");
799 //}
800 
801 Page FileMgr::requestFreePage(size_t pageSize, const bool isMetadata) {
802  std::lock_guard<std::mutex> lock(getPageMutex_);
803 
804  auto candidateFiles = fileIndex_.equal_range(pageSize);
805  int pageNum = -1;
806  for (auto fileIt = candidateFiles.first; fileIt != candidateFiles.second; ++fileIt) {
807  FileInfo* fileInfo = files_[fileIt->second];
808  pageNum = fileInfo->getFreePage();
809  if (pageNum != -1) {
810  return (Page(fileInfo->fileId, pageNum));
811  }
812  }
813  // if here then we need to add a file
814  FileInfo* fileInfo;
815  if (isMetadata) {
816  fileInfo = createFile(pageSize, MAX_FILE_N_METADATA_PAGES);
817  } else {
818  fileInfo = createFile(pageSize, MAX_FILE_N_PAGES);
819  }
820  pageNum = fileInfo->getFreePage();
821  assert(pageNum != -1);
822  return (Page(fileInfo->fileId, pageNum));
823 }
824 
825 void FileMgr::requestFreePages(size_t numPagesRequested,
826  size_t pageSize,
827  std::vector<Page>& pages,
828  const bool isMetadata) {
829  // not used currently
830  // @todo add method to FileInfo to get more than one page
831  std::lock_guard<std::mutex> lock(getPageMutex_);
832  auto candidateFiles = fileIndex_.equal_range(pageSize);
833  size_t numPagesNeeded = numPagesRequested;
834  for (auto fileIt = candidateFiles.first; fileIt != candidateFiles.second; ++fileIt) {
835  FileInfo* fileInfo = files_[fileIt->second];
836  int pageNum;
837  do {
838  pageNum = fileInfo->getFreePage();
839  if (pageNum != -1) {
840  pages.emplace_back(fileInfo->fileId, pageNum);
841  numPagesNeeded--;
842  }
843  } while (pageNum != -1 && numPagesNeeded > 0);
844  if (numPagesNeeded == 0) {
845  break;
846  }
847  }
848  while (numPagesNeeded > 0) {
849  FileInfo* fileInfo;
850  if (isMetadata) {
851  fileInfo = createFile(pageSize, MAX_FILE_N_METADATA_PAGES);
852  } else {
853  fileInfo = createFile(pageSize, MAX_FILE_N_PAGES);
854  }
855  int pageNum;
856  do {
857  pageNum = fileInfo->getFreePage();
858  if (pageNum != -1) {
859  pages.emplace_back(fileInfo->fileId, pageNum);
860  numPagesNeeded--;
861  }
862  } while (pageNum != -1 && numPagesNeeded > 0);
863  if (numPagesNeeded == 0) {
864  break;
865  }
866  }
867  assert(pages.size() == numPagesRequested);
868 }
869 
870 FileInfo* FileMgr::openExistingFile(const std::string& path,
871  const int fileId,
872  const size_t pageSize,
873  const size_t numPages,
874  std::vector<HeaderInfo>& headerVec) {
875  FILE* f = open(path);
876  FileInfo* fInfo = new FileInfo(
877  this, fileId, f, pageSize, numPages, false); // false means don't init file
878 
879  fInfo->openExistingFile(headerVec, epoch_);
880  mapd_unique_lock<mapd_shared_mutex> write_lock(files_rw_mutex_);
881  if (fileId >= static_cast<int>(files_.size())) {
882  files_.resize(fileId + 1);
883  }
884  files_[fileId] = fInfo;
885  fileIndex_.insert(std::pair<size_t, int>(pageSize, fileId));
886  return fInfo;
887 }
888 
889 FileInfo* FileMgr::createFile(const size_t pageSize, const size_t numPages) {
890  // check arguments
891  if (pageSize == 0 || numPages == 0) {
892  LOG(FATAL) << "File creation failed: pageSize and numPages must be greater than 0.";
893  }
894 
895  // create the new file
896  FILE* f = create(fileMgrBasePath_,
897  nextFileId_,
898  pageSize,
899  numPages); // TM: not sure if I like naming scheme here - should be in
900  // separate namespace?
901  CHECK(f);
902 
903  // instantiate a new FileInfo for the newly created file
904  int fileId = nextFileId_++;
905  FileInfo* fInfo =
906  new FileInfo(this, fileId, f, pageSize, numPages, true); // true means init file
907  assert(fInfo);
908 
909  mapd_unique_lock<mapd_shared_mutex> write_lock(files_rw_mutex_);
910  // update file manager data structures
911  files_.push_back(fInfo);
912  fileIndex_.insert(std::pair<size_t, int>(pageSize, fileId));
913 
914  assert(files_.back() == fInfo); // postcondition
915  return fInfo;
916 }
917 
918 FILE* FileMgr::getFileForFileId(const int fileId) {
919  assert(fileId >= 0);
920  return files_[fileId]->f;
921 }
922 /*
923 void FileMgr::getAllChunkMetaInfo(std::vector<std::pair<ChunkKey, int64_t> > &metadata) {
924  metadata.reserve(chunkIndex_.size());
925  for (auto chunkIt = chunkIndex_.begin(); chunkIt != chunkIndex_.end(); ++chunkIt) {
926  metadata.push_back(std::make_pair(chunkIt->first,
927 chunkIt->second->encoder->numElems));
928  }
929 }
930 */
932  std::vector<std::pair<ChunkKey, ChunkMetadata>>& chunkMetadataVec) {
933  mapd_unique_lock<mapd_shared_mutex> chunkIndexWriteLock(chunkIndexMutex_);
934  chunkMetadataVec.reserve(chunkIndex_.size());
935  for (auto chunkIt = chunkIndex_.begin(); chunkIt != chunkIndex_.end(); ++chunkIt) {
936  if (chunkIt->second->hasEncoder) {
937  ChunkMetadata chunkMetadata;
938  chunkIt->second->encoder->getMetadata(chunkMetadata);
939  chunkMetadataVec.emplace_back(chunkIt->first, chunkMetadata);
940  }
941  }
942 }
943 
945  std::vector<std::pair<ChunkKey, ChunkMetadata>>& chunkMetadataVec,
946  const ChunkKey& keyPrefix) {
947  mapd_unique_lock<mapd_shared_mutex> chunkIndexWriteLock(
948  chunkIndexMutex_); // is this guarding the right structure? it look slike we oly
949  // read here for chunk
950  auto chunkIt = chunkIndex_.lower_bound(keyPrefix);
951  if (chunkIt == chunkIndex_.end()) {
952  return; // throw?
953  }
954  while (chunkIt != chunkIndex_.end() &&
955  std::search(chunkIt->first.begin(),
956  chunkIt->first.begin() + keyPrefix.size(),
957  keyPrefix.begin(),
958  keyPrefix.end()) != chunkIt->first.begin() + keyPrefix.size()) {
959  /*
960  for (auto vecIt = chunkIt->first.begin(); vecIt != chunkIt->first.end(); ++vecIt) {
961  std::cout << *vecIt << ",";
962  }
963  cout << endl;
964  */
965  if (chunkIt->second->hasEncoder) {
966  ChunkMetadata chunkMetadata;
967  chunkIt->second->encoder->getMetadata(chunkMetadata);
968  chunkMetadataVec.emplace_back(chunkIt->first, chunkMetadata);
969  }
970  chunkIt++;
971  }
972 }
973 
975  return gfm_->getDBVersion();
976 }
977 
978 bool FileMgr::getDBConvert() const {
979  return gfm_->getDBConvert();
980 }
981 
984  if (db_version_ > getDBVersion()) {
985  LOG(FATAL) << "DB forward compatibility is not supported. Version of OmniSci "
986  "software used is older than the version of DB being read: "
987  << db_version_;
988  }
989  } else {
991  }
992 }
993 
995  epoch_ = epoch;
997 }
998 
999 void FileMgr::free_page(std::pair<FileInfo*, int>&& page) {
1000  std::unique_lock<mapd_shared_mutex> lock(mutex_free_page);
1001  free_pages.push_back(page);
1002 }
1003 
1004 } // namespace File_Namespace
std::string getBasePath() const
virtual size_t pageDataSize() const
Returns the size in bytes of the data portion of each page in the FileBuffer.
Definition: FileBuffer.h:130
AbstractBuffer * alloc(const size_t numBytes) override
Definition: FileMgr.cpp:788
FileInfo * getFileInfoForFileId(const int fileId)
Definition: FileMgr.h:157
void createEpochFile(const std::string &epochFileName)
Definition: FileMgr.cpp:485
Page requestFreePage(size_t pagesize, const bool isMetadata)
Definition: FileMgr.cpp:801
size_t size() const override
Definition: FileBuffer.h:139
void openEpochFile(const std::string &epochFileName)
Definition: FileMgr.cpp:497
virtual void write(int8_t *src, const size_t numBytes, const size_t offset=0, const MemoryLevel srcBufferType=CPU_LEVEL, const int srcDeviceId=-1)=0
void writeHeader(Page &page, const int pageId, const int epoch, const bool writeMetadata=false)
Write header writes header at top of page in format.
Definition: FileBuffer.cpp:368
A logical page (Page) belongs to a file on disk.
Definition: Page.h:46
#define LOG(tag)
Definition: Logger.h:182
virtual size_t size() const =0
FileMgr * getFileMgr(const int db_id, const int tb_id)
Stores Pair of ChunkKey and Page id and version, in a pair with a Page struct itself (File id and Pag...
Definition: Page.h:121
virtual int8_t * getMemoryPtr()=0
void free(AbstractBuffer *buffer) override
Definition: FileMgr.cpp:793
virtual MemoryLevel getType() const =0
FILE * getFileForFileId(const int fileId)
Returns FILE pointer associated with requested fileId.
Definition: FileMgr.cpp:918
void copyPage(Page &srcPage, FileMgr *destFileMgr, Page &destPage, const size_t reservedHeaderSize, const size_t numBytes, const size_t offset)
Definition: FileMgr.cpp:463
#define MAPD_FILE_EXT
Definition: File.h:26
int getDBVersion() const
Index for looking up chunks.
Definition: FileMgr.cpp:974
std::vector< std::pair< FileInfo *, int > > free_pages
Definition: FileMgr.h:259
TypeR::rep timer_stop(Type clock_begin)
Definition: measure.h:46
void deleteBuffer(const ChunkKey &key, const bool purge=true) override
Deletes the chunk with the specified key.
Definition: FileMgr.cpp:634
std::vector< MultiPage > multiPages_
Definition: FileBuffer.h:171
GlobalFileMgr * gfm_
Definition: FileMgr.h:239
std::mutex getPageMutex_
pointer to DB level metadata
Definition: FileMgr.h:254
bool getDBConvert() const
Definition: FileMgr.cpp:978
std::string showChunk(const ChunkKey &key)
Definition: types.h:37
Represents/provides access to contiguous data stored in the file system.
Definition: FileBuffer.h:55
void createDBMetaFile(const std::string &DBMetaFileName)
Definition: FileMgr.cpp:534
void checkpoint() override
Fsyncs data files, writes out epoch and fsyncs that.
Definition: FileMgr.cpp:575
std::string fileMgrBasePath_
Definition: FileMgr.h:241
bool headerCompare(const HeaderInfo &firstElem, const HeaderInfo &secondElem)
Definition: FileMgr.cpp:47
std::string to_string(char const *&&v)
FILE * create(const std::string &basePath, const int fileId, const size_t pageSize, const size_t numPages)
Definition: File.cpp:35
void fetchBuffer(const ChunkKey &key, AbstractBuffer *destBuffer, const size_t numBytes) override
Definition: FileMgr.cpp:687
AbstractBuffer * createBuffer(const ChunkKey &key, size_t pageSize=0, const size_t numBytes=0) override
Creates a chunk with the specified key and page size.
Definition: FileMgr.cpp:609
virtual void reserve(size_t numBytes)=0
FILE * epochFile_
the current epoch (time of last checkpoint)
Definition: FileMgr.h:249
int epoch()
Returns current value of epoch - should be one greater than recorded at last checkpoint.
Definition: FileMgr.h:206
size_t write(const size_t offset, const size_t size, int8_t *buf)
Definition: FileInfo.cpp:59
void free_page(std::pair< FileInfo *, int > &&page)
Definition: FileMgr.cpp:999
FileMgr(const int deviceId, GlobalFileMgr *gfm, const std::pair< const int, const int > fileMgrKey, const size_t num_reader_threads=0, const int epoch=-1, const size_t defaultPageSize=2097152)
Constructor.
Definition: FileMgr.cpp:65
AbstractBuffer * getBuffer(const ChunkKey &key, const size_t numBytes=0) override
Returns the a pointer to the chunk with the specified key.
Definition: FileMgr.cpp:678
std::vector< FileInfo * > files_
Definition: FileMgr.h:243
size_t read(FILE *f, const size_t offset, const size_t size, int8_t *buf)
Reads the specified number of bytes from the offset position in file f into buf.
Definition: File.cpp:113
void processFileFutures(std::vector< std::future< std::vector< HeaderInfo >>> &file_futures, std::vector< HeaderInfo > &headerVec)
Definition: FileMgr.cpp:281
size_t pageSize() const override
Returns the size in bytes of each page in the FileBuffer.
Definition: FileBuffer.h:127
virtual void append(int8_t *src, const size_t numBytes, const MemoryLevel srcBufferType=CPU_LEVEL, const int deviceId=-1)=0
void init(const size_t num_reader_threads)
Definition: FileMgr.cpp:122
int epoch_
the index of the next file id
Definition: FileMgr.h:248
#define MAX_FILE_N_METADATA_PAGES
Definition: File.h:28
std::string getFileMgrBasePath() const
Definition: FileMgr.h:232
ChunkKeyToChunkMap chunkIndex_
Definition: FileMgr.h:227
PageSizeFileMMap fileIndex_
A vector of files accessible via a file identifier.
Definition: FileMgr.h:244
#define DB_META_FILENAME
Definition: FileMgr.cpp:41
An AbstractBuffer is a unit of data management for a data manager.
size_t num_reader_threads_
Maps page sizes to FileInfo objects.
Definition: FileMgr.h:245
bool isBufferOnDevice(const ChunkKey &key) override
Definition: FileMgr.cpp:629
void writeAndSyncEpochToDisk()
Definition: FileMgr.cpp:516
size_t pageNum
unique identifier of the owning file
Definition: Page.h:48
virtual std::vector< MultiPage > getMultiPage() const
Returns vector of MultiPages in the FileBuffer.
Definition: FileBuffer.h:137
void deleteBuffersWithPrefix(const ChunkKey &keyPrefix, const bool purge=true) override
Definition: FileMgr.cpp:651
size_t fileSize(FILE *f)
Returns the size of the specified file.
Definition: File.cpp:171
AbstractBuffer * putBuffer(const ChunkKey &key, AbstractBuffer *d, const size_t numBytes=0) override
Puts the contents of d into the Chunk with the given key.
Definition: FileMgr.cpp:732
bool openDBMetaFile(const std::string &DBMetaFileName)
Definition: FileMgr.cpp:545
#define MAX_FILE_N_PAGES
Definition: File.h:27
size_t defaultPageSize_
number of threads used when loading data
Definition: FileMgr.h:246
std::deque< int > epochs
Definition: Page.h:72
virtual void read(int8_t *const dst, const size_t numBytes, const size_t offset=0, const MemoryLevel dstBufferType=CPU_LEVEL, const int dstDeviceId=-1)=0
size_t read(const size_t offset, const size_t size, int8_t *buf)
Definition: FileInfo.cpp:64
~FileMgr() override
Destructor.
Definition: FileMgr.cpp:111
virtual int getDeviceId() const
void setSize(const size_t size)
void getChunkMetadataVec(std::vector< std::pair< ChunkKey, ChunkMetadata >> &chunkMetadataVec) override
Definition: FileMgr.cpp:931
void openExistingFile(std::vector< HeaderInfo > &headerVec, const int fileMgrEpoch)
Definition: FileInfo.cpp:69
FILE * open(int fileId)
Opens/creates the file with the given id; returns NULL on error.
Definition: File.cpp:83
FileInfo * createFile(const size_t pageSize, const size_t numPages)
Adds a file to the file manager repository.
Definition: FileMgr.cpp:889
size_t write(FILE *f, const size_t offset, const size_t size, int8_t *buf)
Writes the specified number of bytes to the offset position in file f from buf.
Definition: File.cpp:121
#define CHECK(condition)
Definition: Logger.h:187
virtual bool isAppended() const
std::vector< int > ChunkKey
Definition: types.h:35
void getChunkMetadataVecForKeyPrefix(std::vector< std::pair< ChunkKey, ChunkMetadata >> &chunkMetadataVec, const ChunkKey &keyPrefix) override
Definition: FileMgr.cpp:944
virtual bool isDirty() const
void close(FILE *f)
Closes the file pointed to by the FILE pointer.
Definition: File.cpp:102
mapd_shared_mutex mutex_free_page
Definition: FileMgr.h:258
void requestFreePages(size_t npages, size_t pagesize, std::vector< Page > &pages, const bool isMetadata)
Obtains free pages – creates new files if necessary – of the requested size.
Definition: FileMgr.cpp:825
void setEpoch(int epoch)
Definition: FileMgr.cpp:994
virtual size_t reservedHeaderSize() const
Definition: FileBuffer.h:134
void syncEncoder(const AbstractBuffer *srcBuffer)
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_<EPOCH>_<oldname>.
Definition: File.cpp:183
void writeAndSyncDBMetaToDisk()
Definition: FileMgr.cpp:566
#define EPOCH_FILENAME
Definition: FileMgr.cpp:40
mapd_shared_mutex chunkIndexMutex_
Definition: FileMgr.h:255
mapd_shared_mutex files_rw_mutex_
Definition: FileMgr.h:256
The MultiPage stores versions of the same logical page in a deque.
Definition: Page.h:69
A selection of helper methods for File I/O.
virtual bool isUpdated() const
std::pair< const int, const int > fileMgrKey_
Global FileMgr.
Definition: FileMgr.h:240
#define VLOG(n)
Definition: Logger.h:277
Type timer_start()
Definition: measure.h:40
FileInfo * openExistingFile(const std::string &path, const int fileId, const size_t pageSize, const size_t numPages, std::vector< HeaderInfo > &headerVec)
Definition: FileMgr.cpp:870