OmniSciDB  340b00dbf6
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
FileMgr.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
24 
25 #include <fcntl.h>
26 #include <algorithm>
27 #include <future>
28 #include <string>
29 #include <thread>
30 #include <utility>
31 #include <vector>
32 
33 #include <boost/filesystem.hpp>
34 #include <boost/lexical_cast.hpp>
35 #include <boost/system/error_code.hpp>
36 
38 #include "Shared/File.h"
39 #include "Shared/checked_alloc.h"
40 #include "Shared/measure.h"
41 
42 #define EPOCH_FILENAME "epoch"
43 #define DB_META_FILENAME "dbmeta"
44 
45 using namespace std;
46 
47 namespace File_Namespace {
48 
49 bool headerCompare(const HeaderInfo& firstElem, const HeaderInfo& secondElem) {
50  // HeaderInfo.first is a pair of Chunk key with a vector containing
51  // pageId and version
52  if (firstElem.chunkKey != secondElem.chunkKey) {
53  return firstElem.chunkKey < secondElem.chunkKey;
54  }
55  if (firstElem.pageId != secondElem.pageId) {
56  return firstElem.pageId < secondElem.pageId;
57  }
58  return firstElem.versionEpoch < secondElem.versionEpoch;
59 
60  /*
61  if (firstElem.first.first != secondElem.first.first)
62  return firstElem.first.first < secondElem.first.first;
63  return firstElem.first.second < secondElem.first.second;
64  */
65 }
66 
67 FileMgr::FileMgr(const int deviceId,
68  GlobalFileMgr* gfm,
69  const std::pair<const int, const int> fileMgrKey,
70  const size_t num_reader_threads,
71  const int epoch,
72  const size_t defaultPageSize)
73  : AbstractBufferMgr(deviceId)
74  , gfm_(gfm)
75  , fileMgrKey_(fileMgrKey)
76  , defaultPageSize_(defaultPageSize)
77  , nextFileId_(0)
78  , epoch_(epoch) {
79  init(num_reader_threads);
80 }
81 
82 // used only to initialize enough to drop
83 FileMgr::FileMgr(const int deviceId,
84  GlobalFileMgr* gfm,
85  const std::pair<const int, const int> fileMgrKey,
86  const bool initOnly)
87  : AbstractBufferMgr(deviceId)
88  , gfm_(gfm)
89  , fileMgrKey_(fileMgrKey)
90  , defaultPageSize_(0)
91  , nextFileId_(0)
92  , epoch_(0) {
93  const std::string fileMgrDirPrefix("table");
94  const std::string FileMgrDirDelim("_");
95  fileMgrBasePath_ = (gfm_->getBasePath() + fileMgrDirPrefix + FileMgrDirDelim +
96  std::to_string(fileMgrKey_.first) + // db_id
97  FileMgrDirDelim + std::to_string(fileMgrKey_.second)); // tb_id
98  epochFile_ = nullptr;
99  files_.clear();
100 }
101 
102 FileMgr::FileMgr(GlobalFileMgr* gfm, const size_t defaultPageSize, std::string basePath)
103  : AbstractBufferMgr(0)
104  , gfm_(gfm)
105  , fileMgrKey_(0, 0)
106  , fileMgrBasePath_(basePath)
107  , defaultPageSize_(defaultPageSize)
108  , nextFileId_(0)
109  , epoch_(-1) {
110  init(basePath);
111 }
112 
113 // For testing purposes only
114 FileMgr::FileMgr(const int epoch) : AbstractBufferMgr(-1), epoch_(epoch) {}
115 
117  // checkpoint();
118  // free memory used by FileInfo objects
119  for (auto chunkIt = chunkIndex_.begin(); chunkIt != chunkIndex_.end(); ++chunkIt) {
120  delete chunkIt->second;
121  }
122  for (auto file_info : files_) {
123  delete file_info;
124  }
125 
126  if (epochFile_) {
127  close(epochFile_);
128  epochFile_ = nullptr;
129  }
130 
131  if (DBMetaFile_) {
133  DBMetaFile_ = nullptr;
134  }
135 }
136 
137 void FileMgr::init(const size_t num_reader_threads) {
138  // if epoch = -1 this means open from epoch file
139  const std::string fileMgrDirPrefix("table");
140  const std::string FileMgrDirDelim("_");
141  fileMgrBasePath_ = (gfm_->getBasePath() + fileMgrDirPrefix + FileMgrDirDelim +
142  std::to_string(fileMgrKey_.first) + // db_id
143  FileMgrDirDelim + std::to_string(fileMgrKey_.second)); // tb_id
144  boost::filesystem::path path(fileMgrBasePath_);
145  if (boost::filesystem::exists(path)) {
146  if (!boost::filesystem::is_directory(path)) {
147  LOG(FATAL) << "Specified path '" << fileMgrBasePath_
148  << "' for table data is not a directory.";
149  }
150  if (epoch_ != -1) { // if opening at previous epoch
151  int epochCopy = epoch_;
153  epoch_ = epochCopy;
154  } else {
156  }
157 
158  auto clock_begin = timer_start();
159 
160  boost::filesystem::directory_iterator
161  endItr; // default construction yields past-the-end
162  int maxFileId = -1;
163  int fileCount = 0;
164  int threadCount = std::thread::hardware_concurrency();
165  std::vector<HeaderInfo> headerVec;
166  std::vector<std::future<std::vector<HeaderInfo>>> file_futures;
167  for (boost::filesystem::directory_iterator fileIt(path); fileIt != endItr; ++fileIt) {
168  if (boost::filesystem::is_regular_file(fileIt->status())) {
169  // note that boost::filesystem leaves preceding dot on
170  // extension - hence MAPD_FILE_EXT is ".mapd"
171  std::string extension(fileIt->path().extension().string());
172 
173  if (extension == MAPD_FILE_EXT) {
174  std::string fileStem(fileIt->path().stem().string());
175  // remove trailing dot if any
176  if (fileStem.size() > 0 && fileStem.back() == '.') {
177  fileStem = fileStem.substr(0, fileStem.size() - 1);
178  }
179  size_t dotPos = fileStem.find_last_of("."); // should only be one
180  if (dotPos == std::string::npos) {
181  LOG(FATAL) << "File `" << fileIt->path()
182  << "` does not carry page size information in the filename.";
183  }
184  int fileId = boost::lexical_cast<int>(fileStem.substr(0, dotPos));
185  if (fileId > maxFileId) {
186  maxFileId = fileId;
187  }
188  size_t pageSize =
189  boost::lexical_cast<size_t>(fileStem.substr(dotPos + 1, fileStem.size()));
190  std::string filePath(fileIt->path().string());
191  size_t fileSize = boost::filesystem::file_size(filePath);
192  CHECK_EQ(fileSize % pageSize, size_t(0)); // should be no partial pages
193  size_t numPages = fileSize / pageSize;
194 
195  VLOG(4) << "File id: " << fileId << " Page size: " << pageSize
196  << " Num pages: " << numPages;
197 
198  file_futures.emplace_back(std::async(
199  std::launch::async, [filePath, fileId, pageSize, numPages, this] {
200  std::vector<HeaderInfo> tempHeaderVec;
201  openExistingFile(filePath, fileId, pageSize, numPages, tempHeaderVec);
202  return tempHeaderVec;
203  }));
204  fileCount++;
205  if (fileCount % threadCount == 0) {
206  processFileFutures(file_futures, headerVec);
207  }
208  }
209  }
210  }
211 
212  if (file_futures.size() > 0) {
213  processFileFutures(file_futures, headerVec);
214  }
215  int64_t queue_time_ms = timer_stop(clock_begin);
216 
217  LOG(INFO) << "Completed Reading table's file metadata, Elapsed time : "
218  << queue_time_ms << "ms Epoch: " << epoch_ << " files read: " << fileCount
219  << " table location: '" << fileMgrBasePath_ << "'";
220 
221  /* Sort headerVec so that all HeaderInfos
222  * from a chunk will be grouped together
223  * and in order of increasing PageId
224  * - Version Epoch */
225 
226  std::sort(headerVec.begin(), headerVec.end(), headerCompare);
227 
228  /* Goal of next section is to find sequences in the
229  * sorted headerVec of the same ChunkId, which we
230  * can then initiate a FileBuffer with */
231 
232  VLOG(4) << "Number of Headers in Vector: " << headerVec.size();
233  if (headerVec.size() > 0) {
234  ChunkKey lastChunkKey = headerVec.begin()->chunkKey;
235  auto startIt = headerVec.begin();
236 
237  for (auto headerIt = headerVec.begin() + 1; headerIt != headerVec.end();
238  ++headerIt) {
239  // for (auto chunkIt = headerIt->chunkKey.begin(); chunkIt !=
240  // headerIt->chunkKey.end(); ++chunkIt) {
241  // std::cout << *chunkIt << " ";
242  //}
243 
244  if (headerIt->chunkKey != lastChunkKey) {
245  chunkIndex_[lastChunkKey] =
246  new FileBuffer(this, /*pageSize,*/ lastChunkKey, startIt, headerIt);
247  /*
248  if (startIt->versionEpoch != -1) {
249  cout << "not skipping bc version != -1" << endl;
250  // -1 means that chunk was deleted
251  // lets not read it in
252  chunkIndex_[lastChunkKey] = new FileBuffer
253  (this,/lastChunkKey,startIt,headerIt);
254 
255  }
256  else {
257  cout << "Skipping bc version == -1" << endl;
258  }
259  */
260  lastChunkKey = headerIt->chunkKey;
261  startIt = headerIt;
262  }
263  }
264  // now need to insert last Chunk
265  // size_t pageSize = files_[startIt->page.fileId]->pageSize;
266  // cout << "Inserting last chunk" << endl;
267  // if (startIt->versionEpoch != -1) {
268  chunkIndex_[lastChunkKey] =
269  new FileBuffer(this, /*pageSize,*/ lastChunkKey, startIt, headerVec.end());
270  //}
271  }
272  nextFileId_ = maxFileId + 1;
273  // std::cout << "next file id: " << nextFileId_ << std::endl;
274  } else {
275  if (!boost::filesystem::create_directory(path)) {
276  LOG(FATAL) << "Could not create data directory: " << path;
277  }
279  }
280 
281  /* define number of reader threads to be used */
282  size_t num_hardware_based_threads =
283  std::thread::hardware_concurrency(); // # of threads is based on # of cores on the
284  // host
285  if (num_reader_threads == 0) { // # of threads has not been defined by user
286  num_reader_threads_ = num_hardware_based_threads;
287  } else {
288  if (num_reader_threads > num_hardware_based_threads) {
289  num_reader_threads_ = num_hardware_based_threads;
290  } else {
291  num_reader_threads_ = num_reader_threads;
292  }
293  }
294 }
295 
297  std::vector<std::future<std::vector<HeaderInfo>>>& file_futures,
298  std::vector<HeaderInfo>& headerVec) {
299  for (auto& file_future : file_futures) {
300  file_future.wait();
301  }
302  // concatenate the vectors after thread completes
303  for (auto& file_future : file_futures) {
304  auto tempHeaderVec = file_future.get();
305  headerVec.insert(headerVec.end(), tempHeaderVec.begin(), tempHeaderVec.end());
306  }
307  file_futures.clear();
308 }
309 
310 void FileMgr::init(const std::string dataPathToConvertFrom) {
311  int converted_data_epoch = 0;
312  boost::filesystem::path path(dataPathToConvertFrom);
313  if (boost::filesystem::exists(path)) {
314  if (!boost::filesystem::is_directory(path)) {
315  LOG(FATAL) << "Specified path `" << path << "` is not a directory.";
316  }
317 
318  if (epoch_ != -1) { // if opening at previous epoch
319  int epochCopy = epoch_;
321  epoch_ = epochCopy;
322  } else {
324  }
325 
326  boost::filesystem::directory_iterator
327  endItr; // default construction yields past-the-end
328  int maxFileId = -1;
329  int fileCount = 0;
330  int threadCount = std::thread::hardware_concurrency();
331  std::vector<HeaderInfo> headerVec;
332  std::vector<std::future<std::vector<HeaderInfo>>> file_futures;
333  for (boost::filesystem::directory_iterator fileIt(path); fileIt != endItr; ++fileIt) {
334  if (boost::filesystem::is_regular_file(fileIt->status())) {
335  // note that boost::filesystem leaves preceding dot on
336  // extension - hence MAPD_FILE_EXT is ".mapd"
337  std::string extension(fileIt->path().extension().string());
338 
339  if (extension == MAPD_FILE_EXT) {
340  std::string fileStem(fileIt->path().stem().string());
341  // remove trailing dot if any
342  if (fileStem.size() > 0 && fileStem.back() == '.') {
343  fileStem = fileStem.substr(0, fileStem.size() - 1);
344  }
345  size_t dotPos = fileStem.find_last_of("."); // should only be one
346  if (dotPos == std::string::npos) {
347  LOG(FATAL) << "File `" + fileIt->path().string() +
348  "` does not carry page size information in the filename.";
349  }
350  int fileId = boost::lexical_cast<int>(fileStem.substr(0, dotPos));
351  if (fileId > maxFileId) {
352  maxFileId = fileId;
353  }
354  size_t pageSize =
355  boost::lexical_cast<size_t>(fileStem.substr(dotPos + 1, fileStem.size()));
356  std::string filePath(fileIt->path().string());
357  size_t fileSize = boost::filesystem::file_size(filePath);
358  CHECK(fileSize % pageSize == 0); // should be no partial pages
359  size_t numPages = fileSize / pageSize;
360 
361  file_futures.emplace_back(std::async(
362  std::launch::async, [filePath, fileId, pageSize, numPages, this] {
363  std::vector<HeaderInfo> tempHeaderVec;
364  openExistingFile(filePath, fileId, pageSize, numPages, tempHeaderVec);
365  return tempHeaderVec;
366  }));
367  fileCount++;
368  if (fileCount % threadCount) {
369  processFileFutures(file_futures, headerVec);
370  }
371  }
372  }
373  }
374 
375  if (file_futures.size() > 0) {
376  processFileFutures(file_futures, headerVec);
377  }
378 
379  /* Sort headerVec so that all HeaderInfos
380  * from a chunk will be grouped together
381  * and in order of increasing PageId
382  * - Version Epoch */
383 
384  std::sort(headerVec.begin(), headerVec.end(), headerCompare);
385 
386  /* Goal of next section is to find sequences in the
387  * sorted headerVec of the same ChunkId, which we
388  * can then initiate a FileBuffer with */
389 
390  if (headerVec.size() > 0) {
391  ChunkKey lastChunkKey = headerVec.begin()->chunkKey;
392  auto startIt = headerVec.begin();
393 
394  for (auto headerIt = headerVec.begin() + 1; headerIt != headerVec.end();
395  ++headerIt) {
396  if (headerIt->chunkKey != lastChunkKey) {
397  FileMgr* c_fm_ =
398  dynamic_cast<File_Namespace::FileMgr*>(gfm_->getFileMgr(lastChunkKey));
399  CHECK(c_fm_);
400  FileBuffer* srcBuf = new FileBuffer(this, lastChunkKey, startIt, headerIt);
401  chunkIndex_[lastChunkKey] = srcBuf;
402  FileBuffer* destBuf = new FileBuffer(c_fm_, srcBuf->pageSize(), lastChunkKey);
403  c_fm_->chunkIndex_[lastChunkKey] = destBuf;
404  destBuf->syncEncoder(srcBuf);
405  destBuf->setSize(srcBuf->size());
406  destBuf->setDirty(); // this needs to be set to force writing out metadata
407  // files from "checkpoint()" call
408 
409  size_t totalNumPages = srcBuf->getMultiPage().size();
410  for (size_t pageNum = 0; pageNum < totalNumPages; pageNum++) {
411  Page srcPage = srcBuf->getMultiPage()[pageNum].current();
412  Page destPage = c_fm_->requestFreePage(
413  srcBuf->pageSize(),
414  false); // may modify and use api "FileBuffer::addNewMultiPage" instead
415  MultiPage multiPage(srcBuf->pageSize());
416  multiPage.epochs.push_back(converted_data_epoch);
417  multiPage.pageVersions.push_back(destPage);
418  destBuf->multiPages_.push_back(multiPage);
419  size_t reservedHeaderSize = srcBuf->reservedHeaderSize();
420  copyPage(
421  srcPage, c_fm_, destPage, reservedHeaderSize, srcBuf->pageDataSize(), 0);
422  destBuf->writeHeader(destPage, pageNum, converted_data_epoch, false);
423  }
424  lastChunkKey = headerIt->chunkKey;
425  startIt = headerIt;
426  }
427  }
428 
429  // now need to insert last Chunk
430  FileMgr* c_fm_ =
431  dynamic_cast<File_Namespace::FileMgr*>(gfm_->getFileMgr(lastChunkKey));
432  FileBuffer* srcBuf = new FileBuffer(this, lastChunkKey, startIt, headerVec.end());
433  chunkIndex_[lastChunkKey] = srcBuf;
434  FileBuffer* destBuf = new FileBuffer(c_fm_, srcBuf->pageSize(), lastChunkKey);
435  c_fm_->chunkIndex_[lastChunkKey] = destBuf;
436  destBuf->syncEncoder(srcBuf);
437  destBuf->setSize(srcBuf->size());
438  destBuf->setDirty(); // this needs to be set to write out metadata file from the
439  // "checkpoint()" call
440 
441  size_t totalNumPages = srcBuf->getMultiPage().size();
442  for (size_t pageNum = 0; pageNum < totalNumPages; pageNum++) {
443  Page srcPage = srcBuf->getMultiPage()[pageNum].current();
444  Page destPage = c_fm_->requestFreePage(
445  srcBuf->pageSize(),
446  false); // may modify and use api "FileBuffer::addNewMultiPage" instead
447  MultiPage multiPage(srcBuf->pageSize());
448  multiPage.epochs.push_back(converted_data_epoch);
449  multiPage.pageVersions.push_back(destPage);
450  destBuf->multiPages_.push_back(multiPage);
451  size_t reservedHeaderSize = srcBuf->reservedHeaderSize();
452  copyPage(srcPage, c_fm_, destPage, reservedHeaderSize, srcBuf->pageDataSize(), 0);
453  destBuf->writeHeader(destPage, pageNum, converted_data_epoch, false);
454  }
455  }
456  nextFileId_ = maxFileId + 1;
457  } else {
458  if (!boost::filesystem::create_directory(path)) {
459  LOG(FATAL) << "Specified path does not exist: " << path;
460  }
461  }
462 }
463 
465  for (auto file_info : files_) {
466  if (file_info->f) {
467  close(file_info->f);
468  file_info->f = nullptr;
469  }
470  }
471 
472  if (epochFile_) {
473  close(epochFile_);
474  epochFile_ = nullptr;
475  }
476 
477  /* rename for later deletion the directory containing table related data */
479 }
480 
481 void FileMgr::copyPage(Page& srcPage,
482  FileMgr* destFileMgr,
483  Page& destPage,
484  const size_t reservedHeaderSize,
485  const size_t numBytes,
486  const size_t offset) {
487  CHECK(offset + numBytes <= defaultPageSize_);
488  FileInfo* srcFileInfo = getFileInfoForFileId(srcPage.fileId);
489  FileInfo* destFileInfo = destFileMgr->getFileInfoForFileId(destPage.fileId);
490  int8_t* buffer = reinterpret_cast<int8_t*>(checked_malloc(numBytes));
491 
492  size_t bytesRead = srcFileInfo->read(
493  srcPage.pageNum * defaultPageSize_ + offset + reservedHeaderSize, numBytes, buffer);
494  CHECK(bytesRead == numBytes);
495  size_t bytesWritten = destFileInfo->write(
496  destPage.pageNum * defaultPageSize_ + offset + reservedHeaderSize,
497  numBytes,
498  buffer);
499  CHECK(bytesWritten == numBytes);
500  ::free(buffer);
501 }
502 
503 void FileMgr::createEpochFile(const std::string& epochFileName) {
504  std::string epochFilePath(fileMgrBasePath_ + "/" + epochFileName);
505  if (boost::filesystem::exists(epochFilePath)) {
506  LOG(FATAL) << "Epoch file `" << epochFilePath << "` already exists";
507  }
508  epochFile_ = create(epochFilePath, sizeof(int));
509  // Write out current epoch to file - which if this
510  // function is being called should be 0
512 }
513 
514 void FileMgr::openEpochFile(const std::string& epochFileName) {
515  std::string epochFilePath(fileMgrBasePath_ + "/" + epochFileName);
516  if (!boost::filesystem::exists(epochFilePath)) {
517  LOG(FATAL) << "Epoch file `" << epochFilePath << "` does not exist";
518  }
519  if (!boost::filesystem::is_regular_file(epochFilePath)) {
520  LOG(FATAL) << "Epoch file `" << epochFilePath << "` is not a regular file";
521  }
522  if (boost::filesystem::file_size(epochFilePath) < 4) {
523  LOG(FATAL) << "Epoch file `" << epochFilePath
524  << "` is not sized properly (current size: "
525  << boost::filesystem::file_size(epochFilePath) << ", expected size: 4)";
526  }
527  epochFile_ = open(epochFilePath);
528  read(epochFile_, 0, sizeof(int), (int8_t*)&epoch_);
529  // std::cout << "Epoch after open file: " << epoch_ << std::endl;
530  epoch_++; // we are in new epoch from last checkpoint
531 }
532 
534  write(epochFile_, 0, sizeof(int), (int8_t*)&epoch_);
535  int status = fflush(epochFile_);
536  // int status = fcntl(fileno(epochFile_),51);
537  if (status != 0) {
538  LOG(FATAL) << "Could not flush epoch file to disk";
539  }
540 #ifdef __APPLE__
541  status = fcntl(fileno(epochFile_), 51);
542 #else
543  status = fsync(fileno(epochFile_));
544 #endif
545  if (status != 0) {
546  LOG(FATAL) << "Could not sync epoch file to disk";
547  }
548  ++epoch_;
549 }
550 
551 void FileMgr::createDBMetaFile(const std::string& DBMetaFileName) {
552  std::string DBMetaFilePath(fileMgrBasePath_ + "/" + DBMetaFileName);
553  if (boost::filesystem::exists(DBMetaFilePath)) {
554  LOG(FATAL) << "DB metadata file `" << DBMetaFilePath << "` already exists.";
555  }
556  DBMetaFile_ = create(DBMetaFilePath, sizeof(int));
557  int db_ver = getDBVersion();
558  write(DBMetaFile_, 0, sizeof(int), (int8_t*)&db_ver);
559  // LOG(INFO) << "DB metadata file has been created.";
560 }
561 
562 bool FileMgr::openDBMetaFile(const std::string& DBMetaFileName) {
563  std::string DBMetaFilePath(fileMgrBasePath_ + "/" + DBMetaFileName);
564 
565  if (!boost::filesystem::exists(DBMetaFilePath)) {
566  // LOG(INFO) << "DB metadata file does not exist, one will be created.";
567  return false;
568  }
569  if (!boost::filesystem::is_regular_file(DBMetaFilePath)) {
570  // LOG(INFO) << "DB metadata file is not a regular file, one will be created.";
571  return false;
572  }
573  if (boost::filesystem::file_size(DBMetaFilePath) < 4) {
574  // LOG(INFO) << "DB metadata file is not sized properly, one will be created.";
575  return false;
576  }
577  DBMetaFile_ = open(DBMetaFilePath);
578  read(DBMetaFile_, 0, sizeof(int), (int8_t*)&db_version_);
579 
580  return true;
581 }
582 
584  int db_ver = getDBVersion();
585  write(DBMetaFile_, 0, sizeof(int), (int8_t*)&db_ver);
586  int status = fflush(DBMetaFile_);
587  if (status != 0) {
588  LOG(FATAL) << "Could not sync DB metadata file to disk";
589  }
590 }
591 
593  VLOG(2) << "Checkpointing epoch: " << epoch_;
594  mapd_unique_lock<mapd_shared_mutex> chunkIndexWriteLock(chunkIndexMutex_);
595  for (auto chunkIt = chunkIndex_.begin(); chunkIt != chunkIndex_.end(); ++chunkIt) {
596  if (chunkIt->second->isDirty()) {
597  chunkIt->second->writeMetadata(epoch_);
598  chunkIt->second->clearDirtyBits();
599  }
600  }
601  chunkIndexWriteLock.unlock();
602 
603  mapd_shared_lock<mapd_shared_mutex> read_lock(files_rw_mutex_);
604  for (auto fileIt = files_.begin(); fileIt != files_.end(); ++fileIt) {
605  int status = (*fileIt)->syncToDisk();
606  if (status != 0) {
607  LOG(FATAL) << "Could not sync file to disk";
608  }
609  }
610 
612 
613  mapd_unique_lock<mapd_shared_mutex> freePagesWriteLock(mutex_free_page);
614  for (auto& free_page : free_pages) {
615  free_page.first->freePageDeferred(free_page.second);
616  }
617  free_pages.clear();
618 }
619 
621  const size_t pageSize,
622  const size_t numBytes) {
623  mapd_unique_lock<mapd_shared_mutex> chunkIndexWriteLock(chunkIndexMutex_);
624  return createBufferUnlocked(key, pageSize, numBytes);
625 }
626 
627 // The underlying implementation of createBuffer needs to be lockless since
628 // some of the codepaths that call it will have already obtained a write lock
629 // and should not release it until they are complete.
631  const size_t pageSize,
632  const size_t numBytes) {
633  size_t actualPageSize = pageSize;
634  if (actualPageSize == 0) {
635  actualPageSize = defaultPageSize_;
636  }
638  // we will do this lazily and not allocate space for the Chunk (i.e.
639  // FileBuffer yet)
640 
641  if (chunkIndex_.find(key) != chunkIndex_.end()) {
642  LOG(FATAL) << "Chunk already exists for key: " << show_chunk(key);
643  }
644  chunkIndex_[key] = new FileBuffer(this, actualPageSize, key, numBytes);
645  return (chunkIndex_[key]);
646 }
647 
649  mapd_shared_lock<mapd_shared_mutex> chunkIndexReadLock(chunkIndexMutex_);
650  return chunkIndex_.find(key) != chunkIndex_.end();
651 }
652 
653 void FileMgr::deleteBuffer(const ChunkKey& key, const bool purge) {
654  mapd_unique_lock<mapd_shared_mutex> chunkIndexWriteLock(chunkIndexMutex_);
655  auto chunkIt = chunkIndex_.find(key);
656  // ensure the Chunk exists
657  if (chunkIt == chunkIndex_.end()) {
658  LOG(FATAL) << "Chunk does not exist for key: " << show_chunk(key);
659  }
660  chunkIndexWriteLock.unlock();
661  // chunkIt->second->writeMetadata(-1); // writes -1 as epoch - signifies deleted
662  if (purge) {
663  chunkIt->second->freePages();
664  }
665  //@todo need a way to represent delete in non purge case
666  delete chunkIt->second;
667  chunkIndex_.erase(chunkIt);
668 }
669 
670 void FileMgr::deleteBuffersWithPrefix(const ChunkKey& keyPrefix, const bool purge) {
671  mapd_unique_lock<mapd_shared_mutex> chunkIndexWriteLock(chunkIndexMutex_);
672  auto chunkIt = chunkIndex_.lower_bound(keyPrefix);
673  if (chunkIt == chunkIndex_.end()) {
674  return; // should we throw?
675  }
676  while (chunkIt != chunkIndex_.end() &&
677  std::search(chunkIt->first.begin(),
678  chunkIt->first.begin() + keyPrefix.size(),
679  keyPrefix.begin(),
680  keyPrefix.end()) != chunkIt->first.begin() + keyPrefix.size()) {
681  /*
682  cout << "Freeing pages for chunk ";
683  for (auto vecIt = chunkIt->first.begin(); vecIt != chunkIt->first.end(); ++vecIt) {
684  std::cout << *vecIt << ",";
685  }
686  cout << endl;
687  */
688  if (purge) {
689  chunkIt->second->freePages();
690  }
691  //@todo need a way to represent delete in non purge case
692  delete chunkIt->second;
693  chunkIndex_.erase(chunkIt++);
694  }
695 }
696 
697 FileBuffer* FileMgr::getBuffer(const ChunkKey& key, const size_t numBytes) {
698  mapd_shared_lock<mapd_shared_mutex> chunkIndexReadLock(chunkIndexMutex_);
699  auto chunkIt = chunkIndex_.find(key);
700  CHECK(chunkIt != chunkIndex_.end())
701  << "Chunk does not exist for key: " << show_chunk(key);
702  return chunkIt->second;
703 }
704 
706  AbstractBuffer* destBuffer,
707  const size_t numBytes) {
708  // reads chunk specified by ChunkKey into AbstractBuffer provided by
709  // destBuffer
710  if (destBuffer->isDirty()) {
711  LOG(FATAL)
712  << "Aborting attempt to fetch a chunk marked dirty. Chunk inconsistency for key: "
713  << show_chunk(key);
714  }
715  mapd_shared_lock<mapd_shared_mutex> chunkIndexReadLock(chunkIndexMutex_);
716  auto chunkIt = chunkIndex_.find(key);
717  if (chunkIt == chunkIndex_.end()) {
718  LOG(FATAL) << "Chunk does not exist for key: " << show_chunk(key);
719  }
720  chunkIndexReadLock.unlock();
721 
722  AbstractBuffer* chunk = chunkIt->second;
723  // chunk's size is either specified in function call with numBytes or we
724  // just look at pageSize * numPages in FileBuffer
725  if (numBytes > 0 && numBytes > chunk->size()) {
726  LOG(FATAL) << "Chunk retrieved for key `" << show_chunk(key) << "` is smaller ("
727  << chunk->size() << ") than number of bytes requested (" << numBytes
728  << ")";
729  }
730 
731  chunk->copyTo(destBuffer, numBytes);
732 }
733 
735  AbstractBuffer* srcBuffer,
736  const size_t numBytes) {
737  // obtain a pointer to the Chunk
738  mapd_unique_lock<mapd_shared_mutex> chunkIndexWriteLock(chunkIndexMutex_);
739  auto chunkIt = chunkIndex_.find(key);
740  FileBuffer* chunk;
741  if (chunkIt == chunkIndex_.end()) {
743  } else {
744  chunk = chunkIt->second;
745  }
746  chunkIndexWriteLock.unlock();
747  size_t oldChunkSize = chunk->size();
748  // write the buffer's data to the Chunk
749  // size_t newChunkSize = numBytes == 0 ? srcBuffer->size() : numBytes;
750  size_t newChunkSize = numBytes == 0 ? srcBuffer->size() : numBytes;
751  if (chunk->isDirty()) {
752  // multiple appends are allowed,
753  // but only single update is allowed
754  if (srcBuffer->isUpdated() && chunk->isUpdated()) {
755  LOG(FATAL) << "Aborting attempt to write a chunk marked dirty. Chunk inconsistency "
756  "for key: "
757  << show_chunk(key);
758  }
759  }
760  if (srcBuffer->isUpdated()) {
761  // chunk size is not changed when fixed rows are updated or are marked as deleted.
762  // but when rows are vacuumed or varlen rows are updated (new rows are appended),
763  // chunk size will change. For vacuum, checkpoint should sync size from cpu to disk.
764  // For varlen update, it takes another route via fragmenter using disk-level buffer.
765  if (0 == numBytes && !chunk->isDirty()) {
766  chunk->setSize(newChunkSize);
767  }
768  //@todo use dirty flags to only flush pages of chunk that need to
769  // be flushed
770  chunk->write((int8_t*)srcBuffer->getMemoryPtr(),
771  newChunkSize,
772  0,
773  srcBuffer->getType(),
774  srcBuffer->getDeviceId());
775  } else if (srcBuffer->isAppended()) {
776  CHECK_LT(oldChunkSize, newChunkSize);
777  chunk->append((int8_t*)srcBuffer->getMemoryPtr() + oldChunkSize,
778  newChunkSize - oldChunkSize,
779  srcBuffer->getType(),
780  srcBuffer->getDeviceId());
781  } else {
782  UNREACHABLE() << "putBuffer() expects a buffer marked is_updated or is_appended";
783  }
784  // chunk->clearDirtyBits(); // Hack: because write and append will set dirty bits
785  //@todo commenting out line above will make sure this metadata is set
786  // but will trigger error on fetch chunk
787  srcBuffer->clearDirtyBits();
788  chunk->syncEncoder(srcBuffer);
789  return chunk;
790 }
791 
792 AbstractBuffer* FileMgr::alloc(const size_t numBytes = 0) {
793  LOG(FATAL) << "Operation not supported";
794  return nullptr; // satisfy return-type warning
795 }
796 
798  LOG(FATAL) << "Operation not supported";
799 }
800 
801 Page FileMgr::requestFreePage(size_t pageSize, const bool isMetadata) {
802  std::lock_guard<std::mutex> lock(getPageMutex_);
803 
804  auto candidateFiles = fileIndex_.equal_range(pageSize);
805  int pageNum = -1;
806  for (auto fileIt = candidateFiles.first; fileIt != candidateFiles.second; ++fileIt) {
807  FileInfo* fileInfo = files_[fileIt->second];
808  pageNum = fileInfo->getFreePage();
809  if (pageNum != -1) {
810  return (Page(fileInfo->fileId, pageNum));
811  }
812  }
813  // if here then we need to add a file
814  FileInfo* fileInfo;
815  if (isMetadata) {
816  fileInfo = createFile(pageSize, MAX_FILE_N_METADATA_PAGES);
817  } else {
818  fileInfo = createFile(pageSize, MAX_FILE_N_PAGES);
819  }
820  pageNum = fileInfo->getFreePage();
821  CHECK(pageNum != -1);
822  return (Page(fileInfo->fileId, pageNum));
823 }
824 
825 void FileMgr::requestFreePages(size_t numPagesRequested,
826  size_t pageSize,
827  std::vector<Page>& pages,
828  const bool isMetadata) {
829  // not used currently
830  // @todo add method to FileInfo to get more than one page
831  std::lock_guard<std::mutex> lock(getPageMutex_);
832  auto candidateFiles = fileIndex_.equal_range(pageSize);
833  size_t numPagesNeeded = numPagesRequested;
834  for (auto fileIt = candidateFiles.first; fileIt != candidateFiles.second; ++fileIt) {
835  FileInfo* fileInfo = files_[fileIt->second];
836  int pageNum;
837  do {
838  pageNum = fileInfo->getFreePage();
839  if (pageNum != -1) {
840  pages.emplace_back(fileInfo->fileId, pageNum);
841  numPagesNeeded--;
842  }
843  } while (pageNum != -1 && numPagesNeeded > 0);
844  if (numPagesNeeded == 0) {
845  break;
846  }
847  }
848  while (numPagesNeeded > 0) {
849  FileInfo* fileInfo;
850  if (isMetadata) {
851  fileInfo = createFile(pageSize, MAX_FILE_N_METADATA_PAGES);
852  } else {
853  fileInfo = createFile(pageSize, MAX_FILE_N_PAGES);
854  }
855  int pageNum;
856  do {
857  pageNum = fileInfo->getFreePage();
858  if (pageNum != -1) {
859  pages.emplace_back(fileInfo->fileId, pageNum);
860  numPagesNeeded--;
861  }
862  } while (pageNum != -1 && numPagesNeeded > 0);
863  if (numPagesNeeded == 0) {
864  break;
865  }
866  }
867  CHECK(pages.size() == numPagesRequested);
868 }
869 
870 FileInfo* FileMgr::openExistingFile(const std::string& path,
871  const int fileId,
872  const size_t pageSize,
873  const size_t numPages,
874  std::vector<HeaderInfo>& headerVec) {
875  FILE* f = open(path);
876  FileInfo* fInfo = new FileInfo(
877  this, fileId, f, pageSize, numPages, false); // false means don't init file
878 
879  fInfo->openExistingFile(headerVec, epoch_);
880  mapd_unique_lock<mapd_shared_mutex> write_lock(files_rw_mutex_);
881  if (fileId >= static_cast<int>(files_.size())) {
882  files_.resize(fileId + 1);
883  }
884  files_[fileId] = fInfo;
885  fileIndex_.insert(std::pair<size_t, int>(pageSize, fileId));
886  return fInfo;
887 }
888 
889 FileInfo* FileMgr::createFile(const size_t pageSize, const size_t numPages) {
890  // check arguments
891  if (pageSize == 0 || numPages == 0) {
892  LOG(FATAL) << "File creation failed: pageSize and numPages must be greater than 0.";
893  }
894 
895  // create the new file
896  FILE* f = create(fileMgrBasePath_,
897  nextFileId_,
898  pageSize,
899  numPages); // TM: not sure if I like naming scheme here - should be in
900  // separate namespace?
901  CHECK(f);
902 
903  // instantiate a new FileInfo for the newly created file
904  int fileId = nextFileId_++;
905  FileInfo* fInfo =
906  new FileInfo(this, fileId, f, pageSize, numPages, true); // true means init file
907  CHECK(fInfo);
908 
909  mapd_unique_lock<mapd_shared_mutex> write_lock(files_rw_mutex_);
910  // update file manager data structures
911  files_.push_back(fInfo);
912  fileIndex_.insert(std::pair<size_t, int>(pageSize, fileId));
913 
914  CHECK(files_.back() == fInfo); // postcondition
915  return fInfo;
916 }
917 
918 FILE* FileMgr::getFileForFileId(const int fileId) {
919  CHECK(fileId >= 0 && static_cast<size_t>(fileId) < files_.size());
920  return files_[fileId]->f;
921 }
922 
924  const ChunkKey& keyPrefix) {
925  mapd_unique_lock<mapd_shared_mutex> chunkIndexWriteLock(
926  chunkIndexMutex_); // is this guarding the right structure? it look slike we oly
927  // read here for chunk
928  auto chunkIt = chunkIndex_.lower_bound(keyPrefix);
929  if (chunkIt == chunkIndex_.end()) {
930  return; // throw?
931  }
932  while (chunkIt != chunkIndex_.end() &&
933  std::search(chunkIt->first.begin(),
934  chunkIt->first.begin() + keyPrefix.size(),
935  keyPrefix.begin(),
936  keyPrefix.end()) != chunkIt->first.begin() + keyPrefix.size()) {
937  if (chunkIt->second->hasEncoder()) {
938  auto chunk_metadata = std::make_shared<ChunkMetadata>();
939  chunkIt->second->encoder_->getMetadata(chunk_metadata);
940  chunkMetadataVec.emplace_back(chunkIt->first, chunk_metadata);
941  }
942  chunkIt++;
943  }
944 }
945 
947  return gfm_->getDBVersion();
948 }
949 
950 bool FileMgr::getDBConvert() const {
951  return gfm_->getDBConvert();
952 }
953 
956  if (db_version_ > getDBVersion()) {
957  LOG(FATAL) << "DB forward compatibility is not supported. Version of OmniSci "
958  "software used is older than the version of DB being read: "
959  << db_version_;
960  }
961  } else {
963  }
964 }
965 
966 void FileMgr::setEpoch(int epoch) {
967  epoch_ = epoch;
969 }
970 
971 void FileMgr::free_page(std::pair<FileInfo*, int>&& page) {
972  std::unique_lock<mapd_shared_mutex> lock(mutex_free_page);
973  free_pages.push_back(page);
974 }
975 
976 void FileMgr::removeTableRelatedDS(const int db_id, const int table_id) {
977  UNREACHABLE();
978 }
979 } // namespace File_Namespace
virtual std::vector< MultiPage > getMultiPage() const
Returns vector of MultiPages in the FileBuffer.
Definition: FileBuffer.h:139
#define CHECK_EQ(x, y)
Definition: Logger.h:205
std::vector< int > ChunkKey
Definition: types.h:37
AbstractBuffer * alloc(const size_t numBytes) override
Definition: FileMgr.cpp:792
FileInfo * getFileInfoForFileId(const int fileId)
Definition: FileMgr.h:155
std::string getBasePath() const
void createEpochFile(const std::string &epochFileName)
Definition: FileMgr.cpp:503
Page requestFreePage(size_t pagesize, const bool isMetadata)
Definition: FileMgr.cpp:801
void syncEncoder(const AbstractBuffer *src_buffer)
FileBuffer * createBuffer(const ChunkKey &key, size_t pageSize=0, const size_t numBytes=0) override
Creates a chunk with the specified key and page size.
Definition: FileMgr.cpp:620
void openEpochFile(const std::string &epochFileName)
Definition: FileMgr.cpp:514
void removeTableRelatedDS(const int db_id, const int table_id) override
Definition: FileMgr.cpp:976
void writeHeader(Page &page, const int pageId, const int epoch, const bool writeMetadata=false)
Returns the total number of used bytes in the FileBuffer.
Definition: FileBuffer.cpp:380
A logical page (Page) belongs to a file on disk.
Definition: Page.h:46
#define LOG(tag)
Definition: Logger.h:188
int getDBVersion() const
Index for looking up chunks.
Definition: FileMgr.cpp:946
Stores Pair of ChunkKey and Page id and version, in a pair with a Page struct itself (File id and Pag...
Definition: Page.h:121
virtual int8_t * getMemoryPtr()=0
void free(AbstractBuffer *buffer) override
Definition: FileMgr.cpp:797
virtual MemoryLevel getType() const =0
FILE * getFileForFileId(const int fileId)
Returns FILE pointer associated with requested fileId.
Definition: FileMgr.cpp:918
void copyPage(Page &srcPage, FileMgr *destFileMgr, Page &destPage, const size_t reservedHeaderSize, const size_t numBytes, const size_t offset)
Definition: FileMgr.cpp:481
#define MAPD_FILE_EXT
Definition: File.h:25
#define UNREACHABLE()
Definition: Logger.h:241
std::vector< std::pair< FileInfo *, int > > free_pages
Definition: FileMgr.h:260
TypeR::rep timer_stop(Type clock_begin)
Definition: measure.h:48
void deleteBuffer(const ChunkKey &key, const bool purge=true) override
Deletes the chunk with the specified key.
Definition: FileMgr.cpp:653
std::vector< MultiPage > multiPages_
Definition: FileBuffer.h:167
std::string getFileMgrBasePath() const
Definition: FileMgr.h:227
GlobalFileMgr * gfm_
Definition: FileMgr.h:240
std::mutex getPageMutex_
pointer to DB level metadata
Definition: FileMgr.h:255
void write(int8_t *src, const size_t numBytes, const size_t offset=0, const MemoryLevel srcMemoryLevel=CPU_LEVEL, const int deviceId=-1) override
Writes the contents of source (src) into new versions of the affected logical pages.
Definition: FileBuffer.cpp:499
Represents/provides access to contiguous data stored in the file system.
Definition: FileBuffer.h:55
void createDBMetaFile(const std::string &DBMetaFileName)
Definition: FileMgr.cpp:551
void checkpoint() override
Fsyncs data files, writes out epoch and fsyncs that.
Definition: FileMgr.cpp:592
std::string fileMgrBasePath_
Definition: FileMgr.h:242
bool headerCompare(const HeaderInfo &firstElem, const HeaderInfo &secondElem)
Definition: FileMgr.cpp:49
std::string to_string(char const *&&v)
FILE * create(const std::string &basePath, const int fileId, const size_t pageSize, const size_t numPages)
Definition: File.cpp:40
void fetchBuffer(const ChunkKey &key, AbstractBuffer *destBuffer, const size_t numBytes) override
Definition: FileMgr.cpp:705
std::string show_chunk(const ChunkKey &key)
Definition: types.h:73
FILE * epochFile_
the current epoch (time of last checkpoint)
Definition: FileMgr.h:250
int epoch()
Returns current value of epoch - should be one greater than recorded at last checkpoint.
Definition: FileMgr.h:201
size_t write(const size_t offset, const size_t size, int8_t *buf)
Definition: FileInfo.cpp:59
void free_page(std::pair< FileInfo *, int > &&page)
Definition: FileMgr.cpp:971
FileMgr(const int deviceId, GlobalFileMgr *gfm, const std::pair< const int, const int > fileMgrKey, const size_t num_reader_threads=0, const int epoch=-1, const size_t defaultPageSize=2097152)
Constructor.
Definition: FileMgr.cpp:67
std::vector< FileInfo * > files_
Definition: FileMgr.h:244
size_t read(FILE *f, const size_t offset, const size_t size, int8_t *buf)
Reads the specified number of bytes from the offset position in file f into buf.
Definition: File.cpp:118
void processFileFutures(std::vector< std::future< std::vector< HeaderInfo >>> &file_futures, std::vector< HeaderInfo > &headerVec)
Definition: FileMgr.cpp:296
void * checked_malloc(const size_t size)
Definition: checked_alloc.h:44
size_t pageSize() const override
Returns the size in bytes of each page in the FileBuffer.
Definition: FileBuffer.h:129
void init(const size_t num_reader_threads)
Definition: FileMgr.cpp:137
int epoch_
the index of the next file id
Definition: FileMgr.h:249
#define MAX_FILE_N_METADATA_PAGES
Definition: File.h:27
ChunkKeyToChunkMap chunkIndex_
Definition: FileMgr.h:222
PageSizeFileMMap fileIndex_
A vector of files accessible via a file identifier.
Definition: FileMgr.h:245
#define DB_META_FILENAME
Definition: FileMgr.cpp:43
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
An AbstractBuffer is a unit of data management for a data manager.
size_t num_reader_threads_
Maps page sizes to FileInfo objects.
Definition: FileMgr.h:246
int fsync(int fd)
Definition: omnisci_fs.cpp:60
bool isBufferOnDevice(const ChunkKey &key) override
Definition: FileMgr.cpp:648
void writeAndSyncEpochToDisk()
Definition: FileMgr.cpp:533
AbstractBufferMgr * getFileMgr(const int db_id, const int tb_id)
size_t pageNum
unique identifier of the owning file
Definition: Page.h:48
void deleteBuffersWithPrefix(const ChunkKey &keyPrefix, const bool purge=true) override
Definition: FileMgr.cpp:670
size_t fileSize(FILE *f)
Returns the size of the specified file.
Definition: File.cpp:176
void getChunkMetadataVecForKeyPrefix(ChunkMetadataVector &chunkMetadataVec, const ChunkKey &keyPrefix) override
Definition: FileMgr.cpp:923
#define CHECK_LT(x, y)
Definition: Logger.h:207
bool openDBMetaFile(const std::string &DBMetaFileName)
Definition: FileMgr.cpp:562
#define MAX_FILE_N_PAGES
Definition: File.h:26
size_t defaultPageSize_
number of threads used when loading data
Definition: FileMgr.h:247
FileBuffer * createBufferUnlocked(const ChunkKey &key, size_t pageSize=0, const size_t numBytes=0)
Definition: FileMgr.cpp:630
std::deque< int > epochs
Definition: Page.h:72
virtual size_t reservedHeaderSize() const
Definition: FileBuffer.h:136
size_t read(const size_t offset, const size_t size, int8_t *buf)
Definition: FileInfo.cpp:64
~FileMgr() override
Destructor.
Definition: FileMgr.cpp:116
bool getDBConvert() const
Definition: FileMgr.cpp:950
void setSize(const size_t size)
void copyTo(AbstractBuffer *destination_buffer, const size_t num_bytes=0)
void openExistingFile(std::vector< HeaderInfo > &headerVec, const int fileMgrEpoch)
Definition: FileInfo.cpp:69
mapd_shared_lock< mapd_shared_mutex > read_lock
FILE * open(int fileId)
Opens/creates the file with the given id; returns NULL on error.
Definition: File.cpp:88
FileInfo * createFile(const size_t pageSize, const size_t numPages)
Adds a file to the file manager repository.
Definition: FileMgr.cpp:889
size_t write(FILE *f, const size_t offset, const size_t size, int8_t *buf)
Writes the specified number of bytes to the offset position in file f from buf.
Definition: File.cpp:126
#define CHECK(condition)
Definition: Logger.h:197
FileBuffer * getBuffer(const ChunkKey &key, const size_t numBytes=0) override
Returns the a pointer to the chunk with the specified key.
Definition: FileMgr.cpp:697
void close(FILE *f)
Closes the file pointed to by the FILE pointer.
Definition: File.cpp:107
mapd_shared_mutex mutex_free_page
Definition: FileMgr.h:259
void requestFreePages(size_t npages, size_t pagesize, std::vector< Page > &pages, const bool isMetadata)
Obtains free pages – creates new files if necessary – of the requested size.
Definition: FileMgr.cpp:825
mapd_unique_lock< mapd_shared_mutex > write_lock
void setEpoch(int epoch)
Definition: FileMgr.cpp:966
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_&lt;EPOCH&gt;_&lt;oldname&gt;.
Definition: File.cpp:188
void writeAndSyncDBMetaToDisk()
Definition: FileMgr.cpp:583
#define EPOCH_FILENAME
Definition: FileMgr.cpp:42
mapd_shared_mutex chunkIndexMutex_
Definition: FileMgr.h:256
virtual size_t pageDataSize() const
Returns the size in bytes of the data portion of each page in the FileBuffer.
Definition: FileBuffer.h:132
mapd_shared_mutex files_rw_mutex_
Definition: FileMgr.h:257
The MultiPage stores versions of the same logical page in a deque.
Definition: Page.h:69
void append(int8_t *src, const size_t numBytes, const MemoryLevel srcMemoryLevel=CPU_LEVEL, const int deviceId=-1) override
Definition: FileBuffer.cpp:455
A selection of helper methods for File I/O.
std::pair< const int, const int > fileMgrKey_
Global FileMgr.
Definition: FileMgr.h:241
FileBuffer * putBuffer(const ChunkKey &key, AbstractBuffer *d, const size_t numBytes=0) override
Puts the contents of d into the Chunk with the given key.
Definition: FileMgr.cpp:734
#define VLOG(n)
Definition: Logger.h:291
Type timer_start()
Definition: measure.h:42
FileInfo * openExistingFile(const std::string &path, const int fileId, const size_t pageSize, const size_t numPages, std::vector< HeaderInfo > &headerVec)
Definition: FileMgr.cpp:870
size_t file_size(const int fd)
Definition: omnisci_fs.cpp:31