OmniSciDB  94e8789169
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
File_Namespace::FileMgr Class Reference

#include <FileMgr.h>

+ Inheritance diagram for File_Namespace::FileMgr:
+ Collaboration diagram for File_Namespace::FileMgr:

Public Member Functions

 FileMgr (const int32_t deviceId, GlobalFileMgr *gfm, const std::pair< const int32_t, const int32_t > fileMgrKey, const int32_t max_rollback_epochs=-1, const size_t num_reader_threads=0, const int32_t epoch=-1, const size_t defaultPageSize=DEFAULT_PAGE_SIZE)
 Constructor. More...
 
 FileMgr (const int32_t deviceId, GlobalFileMgr *gfm, const std::pair< const int32_t, const int32_t > fileMgrKey, const size_t defaultPageSize, const bool runCoreInit)
 
 FileMgr (GlobalFileMgr *gfm, const size_t defaultPageSize, std::string basePath)
 
 ~FileMgr () override
 Destructor. More...
 
StorageStats getStorageStats ()
 
FileBuffercreateBuffer (const ChunkKey &key, size_t pageSize=0, const size_t numBytes=0) override
 Creates a chunk with the specified key and page size. More...
 
bool isBufferOnDevice (const ChunkKey &key) override
 
void deleteBuffer (const ChunkKey &key, const bool purge=true) override
 Deletes the chunk with the specified key. More...
 
void deleteBuffersWithPrefix (const ChunkKey &keyPrefix, const bool purge=true) override
 
FileBuffergetBuffer (const ChunkKey &key, const size_t numBytes=0) override
 Returns the a pointer to the chunk with the specified key. More...
 
void fetchBuffer (const ChunkKey &key, AbstractBuffer *destBuffer, const size_t numBytes) override
 
FileBufferputBuffer (const ChunkKey &key, AbstractBuffer *d, const size_t numBytes=0) override
 Puts the contents of d into the Chunk with the given key. More...
 
AbstractBufferalloc (const size_t numBytes) override
 
void free (AbstractBuffer *buffer) override
 
Page requestFreePage (size_t pagesize, const bool isMetadata)
 
MgrType getMgrType () override
 
std::string getStringMgrType () override
 
std::string printSlabs () override
 
void clearSlabs () override
 
size_t getMaxSize () override
 
size_t getInUseSize () override
 
size_t getAllocated () override
 
bool isAllocationCapped () override
 
FileInfogetFileInfoForFileId (const int32_t fileId)
 
uint64_t getTotalFileSize () const
 
FileMetadata getMetadataForFile (const boost::filesystem::directory_iterator &fileIterator)
 
void init (const size_t num_reader_threads, const int32_t epochOverride)
 
void init (const std::string &dataPathToConvertFrom, const int32_t epochOverride)
 
bool coreInit ()
 Determines file path, and if exists, runs file migration and opens and reads epoch file. More...
 
void copyPage (Page &srcPage, FileMgr *destFileMgr, Page &destPage, const size_t reservedHeaderSize, const size_t numBytes, const size_t offset)
 
void requestFreePages (size_t npages, size_t pagesize, std::vector< Page > &pages, const bool isMetadata)
 Obtains free pages – creates new files if necessary – of the requested size. More...
 
void getChunkMetadataVecForKeyPrefix (ChunkMetadataVector &chunkMetadataVec, const ChunkKey &keyPrefix) override
 
void checkpoint () override
 Fsyncs data files, writes out epoch and fsyncs that. More...
 
void checkpoint (const int32_t db_id, const int32_t tb_id) override
 
int32_t epoch ()
 Returns current value of epoch - should be one greater than recorded at last checkpoint. More...
 
int32_t epochFloor ()
 
int32_t incrementEpoch ()
 
int32_t lastCheckpointedEpoch ()
 Returns value of epoch at last checkpoint. More...
 
int32_t maxRollbackEpochs ()
 Returns value max_rollback_epochs. More...
 
size_t getNumReaderThreads ()
 Returns number of threads defined by parameter num-reader-threads which should be used during initial load and consequent read of data. More...
 
FILE * getFileForFileId (const int32_t fileId)
 Returns FILE pointer associated with requested fileId. More...
 
size_t getNumChunks () override
 
size_t getNumUsedPages () const
 
size_t getNumUsedMetadataPages () const
 
size_t getNumUsedMetadataPagesForChunkKey (const ChunkKey &chunkKey) const
 
int32_t getDBVersion () const
 Index for looking up chunks. More...
 
bool getDBConvert () const
 
void createTopLevelMetadata ()
 
std::string getFileMgrBasePath () const
 
void closeRemovePhysical ()
 
void removeTableRelatedDS (const int32_t db_id, const int32_t table_id) override
 
void free_page (std::pair< FileInfo *, int32_t > &&page)
 
const std::pair< const int32_t,
const int32_t > 
get_fileMgrKey () const
 

Public Attributes

ChunkKeyToChunkMap chunkIndex_
 

Protected Member Functions

 FileMgr (const int epoch)
 

Private Member Functions

FileInfocreateFile (const size_t pageSize, const size_t numPages)
 Adds a file to the file manager repository. More...
 
FileInfoopenExistingFile (const std::string &path, const int32_t fileId, const size_t pageSize, const size_t numPages, std::vector< HeaderInfo > &headerVec)
 
void createEpochFile (const std::string &epochFileName)
 
int32_t openAndReadLegacyEpochFile (const std::string &epochFileName)
 
void openAndReadEpochFile (const std::string &epochFileName)
 
void writeAndSyncEpochToDisk ()
 
void setEpoch (const int32_t newEpoch)
 
void freePagesBeforeEpoch (const int32_t minRollbackEpoch)
 
void rollOffOldData (const int32_t epochCeiling, const bool shouldCheckpoint)
 
int32_t readVersionFromDisk (const std::string &versionFileName) const
 
void writeAndSyncVersionToDisk (const std::string &versionFileName, const int32_t version)
 
void processFileFutures (std::vector< std::future< std::vector< HeaderInfo >>> &file_futures, std::vector< HeaderInfo > &headerVec)
 
FileBuffercreateBufferUnlocked (const ChunkKey &key, size_t pageSize=0, const size_t numBytes=0)
 
void migrateToLatestFileMgrVersion ()
 
void migrateEpochFileV0 ()
 

Private Attributes

GlobalFileMgrgfm_
 
std::pair< const int32_t,
const int32_t > 
fileMgrKey_
 Global FileMgr. More...
 
int32_t maxRollbackEpochs_
 
std::string fileMgrBasePath_
 
std::vector< FileInfo * > files_
 
PageSizeFileMMap fileIndex_
 A vector of files accessible via a file identifier. More...
 
size_t num_reader_threads_
 Maps page sizes to FileInfo objects. More...
 
size_t defaultPageSize_
 number of threads used when loading data More...
 
unsigned nextFileId_
 
Epoch epoch_
 the index of the next file id More...
 
bool epochIsCheckpointed_ = true
 
FILE * epochFile_ = nullptr
 
int32_t db_version_
 
int32_t fileMgrVersion_
 
const int32_t latestFileMgrVersion_ {1}
 
FILE * DBMetaFile_ = nullptr
 
std::mutex getPageMutex_
 pointer to DB level metadata More...
 
mapd_shared_mutex chunkIndexMutex_
 
mapd_shared_mutex files_rw_mutex_
 
mapd_shared_mutex mutex_free_page_
 
std::vector< std::pair
< FileInfo *, int32_t > > 
free_pages_
 
bool isFullyInitted_ {false}
 

Friends

class GlobalFileMgr
 

Detailed Description

Definition at line 114 of file FileMgr.h.

Constructor & Destructor Documentation

File_Namespace::FileMgr::FileMgr ( const int32_t  deviceId,
GlobalFileMgr gfm,
const std::pair< const int32_t, const int32_t >  fileMgrKey,
const int32_t  max_rollback_epochs = -1,
const size_t  num_reader_threads = 0,
const int32_t  epoch = -1,
const size_t  defaultPageSize = DEFAULT_PAGE_SIZE 
)

Constructor.

File_Namespace::FileMgr::FileMgr ( const int32_t  deviceId,
GlobalFileMgr gfm,
const std::pair< const int32_t, const int32_t >  fileMgrKey,
const size_t  defaultPageSize,
const bool  runCoreInit 
)

Definition at line 82 of file FileMgr.cpp.

References coreInit(), epochFile_, fileMgrBasePath_, fileMgrKey_, files_, File_Namespace::GlobalFileMgr::getBasePath(), gfm_, and to_string().

87  : AbstractBufferMgr(deviceId)
88  , gfm_(gfm)
89  , fileMgrKey_(fileMgrKey)
90  , maxRollbackEpochs_(-1)
91  , defaultPageSize_(defaultPageSize)
92  , nextFileId_(0) {
93  const std::string fileMgrDirPrefix("table");
94  const std::string FileMgrDirDelim("_");
95  fileMgrBasePath_ = (gfm_->getBasePath() + fileMgrDirPrefix + FileMgrDirDelim +
96  std::to_string(fileMgrKey_.first) + // db_id
97  FileMgrDirDelim + std::to_string(fileMgrKey_.second)); // tb_id
98  epochFile_ = nullptr;
99  files_.clear();
100  if (runCoreInit) {
101  coreInit();
102  }
103 }
std::string getBasePath() const
GlobalFileMgr * gfm_
Definition: FileMgr.h:317
std::string fileMgrBasePath_
Definition: FileMgr.h:320
std::string to_string(char const *&&v)
std::vector< FileInfo * > files_
Definition: FileMgr.h:322
std::pair< const int32_t, const int32_t > fileMgrKey_
Global FileMgr.
Definition: FileMgr.h:318
size_t defaultPageSize_
number of threads used when loading data
Definition: FileMgr.h:325
int32_t maxRollbackEpochs_
Definition: FileMgr.h:319
bool coreInit()
Determines file path, and if exists, runs file migration and opens and reads epoch file...
Definition: FileMgr.cpp:141

+ Here is the call graph for this function:

File_Namespace::FileMgr::FileMgr ( GlobalFileMgr gfm,
const size_t  defaultPageSize,
std::string  basePath 
)

Definition at line 105 of file FileMgr.cpp.

References init().

106  : AbstractBufferMgr(0)
107  , gfm_(gfm)
108  , fileMgrKey_(0, 0)
109  , maxRollbackEpochs_(-1)
110  , fileMgrBasePath_(basePath)
111  , defaultPageSize_(defaultPageSize)
112  , nextFileId_(0) {
113  init(basePath, -1);
114 }
void init(const size_t num_reader_threads, const int32_t epochOverride)
Definition: FileMgr.cpp:196
GlobalFileMgr * gfm_
Definition: FileMgr.h:317
std::string fileMgrBasePath_
Definition: FileMgr.h:320
std::pair< const int32_t, const int32_t > fileMgrKey_
Global FileMgr.
Definition: FileMgr.h:318
size_t defaultPageSize_
number of threads used when loading data
Definition: FileMgr.h:325
int32_t maxRollbackEpochs_
Definition: FileMgr.h:319

+ Here is the call graph for this function:

File_Namespace::FileMgr::~FileMgr ( )
override

Destructor.

Definition at line 121 of file FileMgr.cpp.

References chunkIndex_, File_Namespace::close(), DBMetaFile_, epochFile_, and files_.

121  {
122  // free memory used by FileInfo objects
123  for (auto chunkIt = chunkIndex_.begin(); chunkIt != chunkIndex_.end(); ++chunkIt) {
124  delete chunkIt->second;
125  }
126  for (auto file_info : files_) {
127  delete file_info;
128  }
129 
130  if (epochFile_) {
131  close(epochFile_);
132  epochFile_ = nullptr;
133  }
134 
135  if (DBMetaFile_) {
137  DBMetaFile_ = nullptr;
138  }
139 }
std::vector< FileInfo * > files_
Definition: FileMgr.h:322
ChunkKeyToChunkMap chunkIndex_
Definition: FileMgr.h:297
void close(FILE *f)
Closes the file pointed to by the FILE pointer.
Definition: File.cpp:107

+ Here is the call graph for this function:

File_Namespace::FileMgr::FileMgr ( const int  epoch)
protected

Definition at line 117 of file FileMgr.cpp.

References Epoch::ceiling(), and epoch_.

117  : AbstractBufferMgr(-1) {
119 }
int32_t ceiling() const
Definition: Epoch.h:44
int32_t epoch()
Returns current value of epoch - should be one greater than recorded at last checkpoint.
Definition: FileMgr.h:245
Epoch epoch_
the index of the next file id
Definition: FileMgr.h:327

+ Here is the call graph for this function:

Member Function Documentation

AbstractBuffer * File_Namespace::FileMgr::alloc ( const size_t  numBytes = 0)
override

Definition at line 862 of file FileMgr.cpp.

References logger::FATAL, and LOG.

862  {
863  LOG(FATAL) << "Operation not supported";
864  return nullptr; // satisfy return-type warning
865 }
#define LOG(tag)
Definition: Logger.h:188
void File_Namespace::FileMgr::checkpoint ( )
override

Fsyncs data files, writes out epoch and fsyncs that.

Definition at line 666 of file FileMgr.cpp.

References chunkIndex_, chunkIndexMutex_, epoch(), logger::FATAL, fileMgrKey_, files_, files_rw_mutex_, free_page(), free_pages_, incrementEpoch(), lastCheckpointedEpoch(), LOG, mutex_free_page_, rollOffOldData(), VLOG, and writeAndSyncEpochToDisk().

Referenced by rollOffOldData().

666  {
667  VLOG(2) << "Checkpointing table (" << fileMgrKey_.first << ", " << fileMgrKey_.second
668  << " epoch: " << epoch();
669  mapd_unique_lock<mapd_shared_mutex> chunkIndexWriteLock(chunkIndexMutex_);
670  for (auto chunkIt = chunkIndex_.begin(); chunkIt != chunkIndex_.end(); ++chunkIt) {
671  if (chunkIt->second->isDirty()) {
672  chunkIt->second->writeMetadata(epoch());
673  chunkIt->second->clearDirtyBits();
674  }
675  }
676  chunkIndexWriteLock.unlock();
677 
678  mapd_shared_lock<mapd_shared_mutex> read_lock(files_rw_mutex_);
679  for (auto fileIt = files_.begin(); fileIt != files_.end(); ++fileIt) {
680  int32_t status = (*fileIt)->syncToDisk();
681  if (status != 0) {
682  LOG(FATAL) << "Could not sync file to disk";
683  }
684  }
685 
687  incrementEpoch();
688  rollOffOldData(lastCheckpointedEpoch(), false /* shouldCheckpoint */);
689 
690  mapd_unique_lock<mapd_shared_mutex> freePagesWriteLock(mutex_free_page_);
691  for (auto& free_page : free_pages_) {
692  free_page.first->freePageDeferred(free_page.second);
693  }
694  free_pages_.clear();
695 }
mapd_shared_mutex mutex_free_page_
Definition: FileMgr.h:342
#define LOG(tag)
Definition: Logger.h:188
void rollOffOldData(const int32_t epochCeiling, const bool shouldCheckpoint)
Definition: FileMgr.cpp:652
std::vector< FileInfo * > files_
Definition: FileMgr.h:322
int32_t incrementEpoch()
Definition: FileMgr.h:249
std::pair< const int32_t, const int32_t > fileMgrKey_
Global FileMgr.
Definition: FileMgr.h:318
ChunkKeyToChunkMap chunkIndex_
Definition: FileMgr.h:297
std::vector< std::pair< FileInfo *, int32_t > > free_pages_
Definition: FileMgr.h:343
void writeAndSyncEpochToDisk()
Definition: FileMgr.cpp:628
int32_t epoch()
Returns current value of epoch - should be one greater than recorded at last checkpoint.
Definition: FileMgr.h:245
void free_page(std::pair< FileInfo *, int32_t > &&page)
Definition: FileMgr.cpp:1177
mapd_shared_lock< mapd_shared_mutex > read_lock
mapd_shared_mutex chunkIndexMutex_
Definition: FileMgr.h:339
mapd_shared_mutex files_rw_mutex_
Definition: FileMgr.h:340
#define VLOG(n)
Definition: Logger.h:291
int32_t lastCheckpointedEpoch()
Returns value of epoch at last checkpoint.
Definition: FileMgr.h:265

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void File_Namespace::FileMgr::checkpoint ( const int32_t  db_id,
const int32_t  tb_id 
)
inlineoverride

Definition at line 238 of file FileMgr.h.

References logger::FATAL, and LOG.

238  {
239  LOG(FATAL) << "Operation not supported, api checkpoint() should be used instead";
240  }
#define LOG(tag)
Definition: Logger.h:188
void File_Namespace::FileMgr::clearSlabs ( )
inlineoverride

Definition at line 180 of file FileMgr.h.

180  { /* noop */
181  }
void File_Namespace::FileMgr::closeRemovePhysical ( )

Definition at line 534 of file FileMgr.cpp.

References File_Namespace::close(), DBMetaFile_, epochFile_, files_, getFileMgrBasePath(), and File_Namespace::renameForDelete().

Referenced by File_Namespace::GlobalFileMgr::removeTableRelatedDS().

534  {
535  for (auto file_info : files_) {
536  if (file_info->f) {
537  close(file_info->f);
538  file_info->f = nullptr;
539  }
540  }
541 
542  if (DBMetaFile_) {
544  DBMetaFile_ = nullptr;
545  }
546 
547  if (epochFile_) {
548  close(epochFile_);
549  epochFile_ = nullptr;
550  }
551 
552  /* rename for later deletion the directory containing table related data */
554 }
std::string getFileMgrBasePath() const
Definition: FileMgr.h:302
std::vector< FileInfo * > files_
Definition: FileMgr.h:322
void close(FILE *f)
Closes the file pointed to by the FILE pointer.
Definition: File.cpp:107
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_&lt;EPOCH&gt;_&lt;oldname&gt;.
Definition: File.cpp:188

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void File_Namespace::FileMgr::copyPage ( Page srcPage,
FileMgr destFileMgr,
Page destPage,
const size_t  reservedHeaderSize,
const size_t  numBytes,
const size_t  offset 
)

Definition at line 556 of file FileMgr.cpp.

References CHECK, checked_malloc(), defaultPageSize_, File_Namespace::Page::fileId, free(), getFileInfoForFileId(), File_Namespace::Page::pageNum, File_Namespace::FileInfo::read(), and File_Namespace::FileInfo::write().

Referenced by init().

561  {
562  CHECK(offset + numBytes <= defaultPageSize_);
563  FileInfo* srcFileInfo = getFileInfoForFileId(srcPage.fileId);
564  FileInfo* destFileInfo = destFileMgr->getFileInfoForFileId(destPage.fileId);
565  int8_t* buffer = reinterpret_cast<int8_t*>(checked_malloc(numBytes));
566 
567  size_t bytesRead = srcFileInfo->read(
568  srcPage.pageNum * defaultPageSize_ + offset + reservedHeaderSize, numBytes, buffer);
569  CHECK(bytesRead == numBytes);
570  size_t bytesWritten = destFileInfo->write(
571  destPage.pageNum * defaultPageSize_ + offset + reservedHeaderSize,
572  numBytes,
573  buffer);
574  CHECK(bytesWritten == numBytes);
575  ::free(buffer);
576 }
void free(AbstractBuffer *buffer) override
Definition: FileMgr.cpp:867
void * checked_malloc(const size_t size)
Definition: checked_alloc.h:44
FileInfo * getFileInfoForFileId(const int32_t fileId)
Definition: FileMgr.h:187
size_t defaultPageSize_
number of threads used when loading data
Definition: FileMgr.h:325
#define CHECK(condition)
Definition: Logger.h:197

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

bool File_Namespace::FileMgr::coreInit ( )

Determines file path, and if exists, runs file migration and opens and reads epoch file.

Returns
a boolean representing whether the directory path existed

Definition at line 141 of file FileMgr.cpp.

References EPOCH_FILENAME, logger::FATAL, fileMgrBasePath_, fileMgrKey_, files_rw_mutex_, File_Namespace::GlobalFileMgr::getBasePath(), gfm_, LOG, migrateToLatestFileMgrVersion(), openAndReadEpochFile(), and to_string().

Referenced by FileMgr(), and init().

141  {
142  mapd_unique_lock<mapd_shared_mutex> write_lock(files_rw_mutex_);
143  const std::string fileMgrDirPrefix("table");
144  const std::string FileMgrDirDelim("_");
145  fileMgrBasePath_ = (gfm_->getBasePath() + fileMgrDirPrefix + FileMgrDirDelim +
146  std::to_string(fileMgrKey_.first) + // db_id
147  FileMgrDirDelim + std::to_string(fileMgrKey_.second)); // tb_id
148  boost::filesystem::path path(fileMgrBasePath_);
149  if (boost::filesystem::exists(path)) {
150  if (!boost::filesystem::is_directory(path)) {
151  LOG(FATAL) << "Specified path '" << fileMgrBasePath_
152  << "' for table data is not a directory.";
153  }
156  return true;
157  }
158  return false;
159 }
std::string getBasePath() const
#define LOG(tag)
Definition: Logger.h:188
void migrateToLatestFileMgrVersion()
Definition: FileMgr.cpp:1125
GlobalFileMgr * gfm_
Definition: FileMgr.h:317
std::string fileMgrBasePath_
Definition: FileMgr.h:320
std::string to_string(char const *&&v)
std::pair< const int32_t, const int32_t > fileMgrKey_
Global FileMgr.
Definition: FileMgr.h:318
constexpr char EPOCH_FILENAME[]
Definition: FileMgr.cpp:43
void openAndReadEpochFile(const std::string &epochFileName)
Definition: FileMgr.cpp:609
mapd_unique_lock< mapd_shared_mutex > write_lock
mapd_shared_mutex files_rw_mutex_
Definition: FileMgr.h:340

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

FileBuffer * File_Namespace::FileMgr::createBuffer ( const ChunkKey key,
size_t  pageSize = 0,
const size_t  numBytes = 0 
)
override

Creates a chunk with the specified key and page size.

Definition at line 697 of file FileMgr.cpp.

References chunkIndexMutex_, and createBufferUnlocked().

699  {
700  mapd_unique_lock<mapd_shared_mutex> chunkIndexWriteLock(chunkIndexMutex_);
701  return createBufferUnlocked(key, pageSize, numBytes);
702 }
FileBuffer * createBufferUnlocked(const ChunkKey &key, size_t pageSize=0, const size_t numBytes=0)
Definition: FileMgr.cpp:707
mapd_shared_mutex chunkIndexMutex_
Definition: FileMgr.h:339

+ Here is the call graph for this function:

FileBuffer * File_Namespace::FileMgr::createBufferUnlocked ( const ChunkKey key,
size_t  pageSize = 0,
const size_t  numBytes = 0 
)
private
Todo:
Make all accesses to chunkIndex_ thread-safe

Definition at line 707 of file FileMgr.cpp.

References chunkIndex_, defaultPageSize_, logger::FATAL, LOG, and show_chunk().

Referenced by createBuffer(), and putBuffer().

709  {
710  size_t actualPageSize = pageSize;
711  if (actualPageSize == 0) {
712  actualPageSize = defaultPageSize_;
713  }
715  // we will do this lazily and not allocate space for the Chunk (i.e.
716  // FileBuffer yet)
717 
718  if (chunkIndex_.find(key) != chunkIndex_.end()) {
719  LOG(FATAL) << "Chunk already exists for key: " << show_chunk(key);
720  }
721  chunkIndex_[key] = new FileBuffer(this, actualPageSize, key, numBytes);
722  return (chunkIndex_[key]);
723 }
#define LOG(tag)
Definition: Logger.h:188
std::string show_chunk(const ChunkKey &key)
Definition: types.h:73
ChunkKeyToChunkMap chunkIndex_
Definition: FileMgr.h:297
size_t defaultPageSize_
number of threads used when loading data
Definition: FileMgr.h:325

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void File_Namespace::FileMgr::createEpochFile ( const std::string &  epochFileName)
private

Definition at line 578 of file FileMgr.cpp.

References Epoch::byte_size(), File_Namespace::create(), epochFile_, logger::FATAL, fileMgrBasePath_, LOG, and writeAndSyncEpochToDisk().

Referenced by init(), and migrateEpochFileV0().

578  {
579  std::string epochFilePath(fileMgrBasePath_ + "/" + epochFileName);
580  if (boost::filesystem::exists(epochFilePath)) {
581  LOG(FATAL) << "Epoch file `" << epochFilePath << "` already exists";
582  }
583  epochFile_ = create(epochFilePath, sizeof(Epoch::byte_size()));
584  // Write out current epoch to file - which if this
585  // function is being called should be 0
587 }
#define LOG(tag)
Definition: Logger.h:188
std::string fileMgrBasePath_
Definition: FileMgr.h:320
FILE * create(const std::string &basePath, const int fileId, const size_t pageSize, const size_t numPages)
Definition: File.cpp:40
void writeAndSyncEpochToDisk()
Definition: FileMgr.cpp:628
static size_t byte_size()
Definition: Epoch.h:63

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

FileInfo * File_Namespace::FileMgr::createFile ( const size_t  pageSize,
const size_t  numPages 
)
private

Adds a file to the file manager repository.

This method will create a FileInfo object for the file being added, and it will create the corresponding file on physical disk with the indicated number of pages pre-allocated.

A pointer to the FileInfo object is returned, which itself has a file pointer (FILE*) and a file identifier (int32_t fileId).

Parameters
fileNameThe name given to the file in physical storage.
pageSizeThe logical page size for the pages in the file.
numPagesThe number of logical pages to initially allocate for the file.
Returns
FileInfo* A pointer to the FileInfo object of the added file.

Definition at line 959 of file FileMgr.cpp.

References CHECK, File_Namespace::create(), generate_TableFunctionsFactory_init::f, logger::FATAL, fileIndex_, fileMgrBasePath_, files_, files_rw_mutex_, LOG, and nextFileId_.

Referenced by requestFreePage(), and requestFreePages().

959  {
960  // check arguments
961  if (pageSize == 0 || numPages == 0) {
962  LOG(FATAL) << "File creation failed: pageSize and numPages must be greater than 0.";
963  }
964 
965  // create the new file
966  FILE* f = create(fileMgrBasePath_,
967  nextFileId_,
968  pageSize,
969  numPages); // TM: not sure if I like naming scheme here - should be in
970  // separate namespace?
971  CHECK(f);
972 
973  // instantiate a new FileInfo for the newly created file
974  int32_t fileId = nextFileId_++;
975  FileInfo* fInfo =
976  new FileInfo(this, fileId, f, pageSize, numPages, true); // true means init file
977  CHECK(fInfo);
978 
979  mapd_unique_lock<mapd_shared_mutex> write_lock(files_rw_mutex_);
980  // update file manager data structures
981  files_.push_back(fInfo);
982  fileIndex_.insert(std::pair<size_t, int32_t>(pageSize, fileId));
983 
984  CHECK(files_.back() == fInfo); // postcondition
985  return fInfo;
986 }
#define LOG(tag)
Definition: Logger.h:188
std::string fileMgrBasePath_
Definition: FileMgr.h:320
FILE * create(const std::string &basePath, const int fileId, const size_t pageSize, const size_t numPages)
Definition: File.cpp:40
std::vector< FileInfo * > files_
Definition: FileMgr.h:322
PageSizeFileMMap fileIndex_
A vector of files accessible via a file identifier.
Definition: FileMgr.h:323
#define CHECK(condition)
Definition: Logger.h:197
mapd_unique_lock< mapd_shared_mutex > write_lock
mapd_shared_mutex files_rw_mutex_
Definition: FileMgr.h:340

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void File_Namespace::FileMgr::createTopLevelMetadata ( )

Definition at line 1052 of file FileMgr.cpp.

References DB_META_FILENAME, db_version_, logger::FATAL, getDBVersion(), INVALID_VERSION, LOG, readVersionFromDisk(), and writeAndSyncVersionToDisk().

1052  {
1054 
1055  if (db_version_ > getDBVersion()) {
1056  LOG(FATAL) << "DB forward compatibility is not supported. Version of OmniSci "
1057  "software used is older than the version of DB being read: "
1058  << db_version_;
1059  }
1061  // new system, or we are moving forward versions
1062  // system wide migration would go here if required
1064  return;
1065  }
1066 }
int32_t readVersionFromDisk(const std::string &versionFileName) const
Definition: FileMgr.cpp:1068
#define LOG(tag)
Definition: Logger.h:188
void writeAndSyncVersionToDisk(const std::string &versionFileName, const int32_t version)
Definition: FileMgr.cpp:1086
int32_t getDBVersion() const
Index for looking up chunks.
Definition: FileMgr.cpp:1044
constexpr int32_t INVALID_VERSION
Definition: FileMgr.cpp:47
constexpr char DB_META_FILENAME[]
Definition: FileMgr.cpp:44

+ Here is the call graph for this function:

void File_Namespace::FileMgr::deleteBuffer ( const ChunkKey key,
const bool  purge = true 
)
override

Deletes the chunk with the specified key.

Definition at line 730 of file FileMgr.cpp.

References chunkIndex_, chunkIndexMutex_, logger::FATAL, LOG, and show_chunk().

730  {
731  mapd_unique_lock<mapd_shared_mutex> chunkIndexWriteLock(chunkIndexMutex_);
732  auto chunkIt = chunkIndex_.find(key);
733  // ensure the Chunk exists
734  if (chunkIt == chunkIndex_.end()) {
735  LOG(FATAL) << "Chunk does not exist for key: " << show_chunk(key);
736  }
737  chunkIndexWriteLock.unlock();
738  // chunkIt->second->writeMetadata(-1); // writes -1 as epoch - signifies deleted
739  if (purge) {
740  chunkIt->second->freePages();
741  }
742  //@todo need a way to represent delete in non purge case
743  delete chunkIt->second;
744  chunkIndex_.erase(chunkIt);
745 }
#define LOG(tag)
Definition: Logger.h:188
std::string show_chunk(const ChunkKey &key)
Definition: types.h:73
ChunkKeyToChunkMap chunkIndex_
Definition: FileMgr.h:297
mapd_shared_mutex chunkIndexMutex_
Definition: FileMgr.h:339

+ Here is the call graph for this function:

void File_Namespace::FileMgr::deleteBuffersWithPrefix ( const ChunkKey keyPrefix,
const bool  purge = true 
)
override

Definition at line 747 of file FileMgr.cpp.

References chunkIndex_, and chunkIndexMutex_.

747  {
748  mapd_unique_lock<mapd_shared_mutex> chunkIndexWriteLock(chunkIndexMutex_);
749  auto chunkIt = chunkIndex_.lower_bound(keyPrefix);
750  if (chunkIt == chunkIndex_.end()) {
751  return; // should we throw?
752  }
753  while (chunkIt != chunkIndex_.end() &&
754  std::search(chunkIt->first.begin(),
755  chunkIt->first.begin() + keyPrefix.size(),
756  keyPrefix.begin(),
757  keyPrefix.end()) != chunkIt->first.begin() + keyPrefix.size()) {
758  if (purge) {
759  chunkIt->second->freePages();
760  }
761  //@todo need a way to represent delete in non purge case
762  delete chunkIt->second;
763  chunkIndex_.erase(chunkIt++);
764  }
765 }
ChunkKeyToChunkMap chunkIndex_
Definition: FileMgr.h:297
mapd_shared_mutex chunkIndexMutex_
Definition: FileMgr.h:339
int32_t File_Namespace::FileMgr::epoch ( )
inline

Returns current value of epoch - should be one greater than recorded at last checkpoint.

Definition at line 245 of file FileMgr.h.

Referenced by File_Namespace::FileBuffer::append(), checkpoint(), File_Namespace::FileInfo::freePage(), File_Namespace::FileBuffer::freePagesBeforeEpoch(), init(), openAndReadLegacyEpochFile(), openExistingFile(), File_Namespace::FileBuffer::reserve(), and File_Namespace::FileBuffer::write().

245 { return static_cast<int32_t>(epoch_.ceiling()); }
int32_t ceiling() const
Definition: Epoch.h:44
Epoch epoch_
the index of the next file id
Definition: FileMgr.h:327

+ Here is the caller graph for this function:

int32_t File_Namespace::FileMgr::epochFloor ( )
inline

Definition at line 247 of file FileMgr.h.

Referenced by getStorageStats().

247 { return static_cast<int32_t>(epoch_.floor()); }
int32_t floor() const
Definition: Epoch.h:43
Epoch epoch_
the index of the next file id
Definition: FileMgr.h:327

+ Here is the caller graph for this function:

void File_Namespace::FileMgr::fetchBuffer ( const ChunkKey key,
AbstractBuffer destBuffer,
const size_t  numBytes 
)
override

Definition at line 775 of file FileMgr.cpp.

References chunkIndex_, chunkIndexMutex_, Data_Namespace::AbstractBuffer::copyTo(), logger::FATAL, Data_Namespace::AbstractBuffer::isDirty(), LOG, show_chunk(), and Data_Namespace::AbstractBuffer::size().

777  {
778  // reads chunk specified by ChunkKey into AbstractBuffer provided by
779  // destBuffer
780  if (destBuffer->isDirty()) {
781  LOG(FATAL)
782  << "Aborting attempt to fetch a chunk marked dirty. Chunk inconsistency for key: "
783  << show_chunk(key);
784  }
785  mapd_shared_lock<mapd_shared_mutex> chunkIndexReadLock(chunkIndexMutex_);
786  auto chunkIt = chunkIndex_.find(key);
787  if (chunkIt == chunkIndex_.end()) {
788  LOG(FATAL) << "Chunk does not exist for key: " << show_chunk(key);
789  }
790  chunkIndexReadLock.unlock();
791 
792  AbstractBuffer* chunk = chunkIt->second;
793  // chunk's size is either specified in function call with numBytes or we
794  // just look at pageSize * numPages in FileBuffer
795  if (numBytes > 0 && numBytes > chunk->size()) {
796  LOG(FATAL) << "Chunk retrieved for key `" << show_chunk(key) << "` is smaller ("
797  << chunk->size() << ") than number of bytes requested (" << numBytes
798  << ")";
799  }
800 
801  chunk->copyTo(destBuffer, numBytes);
802 }
#define LOG(tag)
Definition: Logger.h:188
std::string show_chunk(const ChunkKey &key)
Definition: types.h:73
ChunkKeyToChunkMap chunkIndex_
Definition: FileMgr.h:297
An AbstractBuffer is a unit of data management for a data manager.
void copyTo(AbstractBuffer *destination_buffer, const size_t num_bytes=0)
mapd_shared_mutex chunkIndexMutex_
Definition: FileMgr.h:339

+ Here is the call graph for this function:

void File_Namespace::FileMgr::free ( AbstractBuffer buffer)
override

Definition at line 867 of file FileMgr.cpp.

References logger::FATAL, and LOG.

Referenced by copyPage().

867  {
868  LOG(FATAL) << "Operation not supported";
869 }
#define LOG(tag)
Definition: Logger.h:188

+ Here is the caller graph for this function:

void File_Namespace::FileMgr::free_page ( std::pair< FileInfo *, int32_t > &&  page)

Definition at line 1177 of file FileMgr.cpp.

References free_pages_, and mutex_free_page_.

Referenced by checkpoint(), File_Namespace::FileInfo::freePage(), and init().

1177  {
1178  std::unique_lock<mapd_shared_mutex> lock(mutex_free_page_);
1179  free_pages_.push_back(page);
1180 }
mapd_shared_mutex mutex_free_page_
Definition: FileMgr.h:342
std::vector< std::pair< FileInfo *, int32_t > > free_pages_
Definition: FileMgr.h:343

+ Here is the caller graph for this function:

void File_Namespace::FileMgr::freePagesBeforeEpoch ( const int32_t  minRollbackEpoch)
private

Definition at line 646 of file FileMgr.cpp.

References chunkIndex_.

Referenced by rollOffOldData().

646  {
647  for (auto chunkIt = chunkIndex_.begin(); chunkIt != chunkIndex_.end(); ++chunkIt) {
648  chunkIt->second->freePagesBeforeEpoch(minRollbackEpoch);
649  }
650 }
ChunkKeyToChunkMap chunkIndex_
Definition: FileMgr.h:297

+ Here is the caller graph for this function:

const std::pair<const int32_t, const int32_t> File_Namespace::FileMgr::get_fileMgrKey ( ) const
inline

Definition at line 308 of file FileMgr.h.

Referenced by File_Namespace::FileInfo::openExistingFile().

308  {
309  return fileMgrKey_;
310  }
std::pair< const int32_t, const int32_t > fileMgrKey_
Global FileMgr.
Definition: FileMgr.h:318

+ Here is the caller graph for this function:

size_t File_Namespace::FileMgr::getAllocated ( )
inlineoverride

Definition at line 184 of file FileMgr.h.

184 { return 0; }
FileBuffer * File_Namespace::FileMgr::getBuffer ( const ChunkKey key,
const size_t  numBytes = 0 
)
override

Returns the a pointer to the chunk with the specified key.

Definition at line 767 of file FileMgr.cpp.

References CHECK, chunkIndex_, chunkIndexMutex_, and show_chunk().

767  {
768  mapd_shared_lock<mapd_shared_mutex> chunkIndexReadLock(chunkIndexMutex_);
769  auto chunkIt = chunkIndex_.find(key);
770  CHECK(chunkIt != chunkIndex_.end())
771  << "Chunk does not exist for key: " << show_chunk(key);
772  return chunkIt->second;
773 }
std::string show_chunk(const ChunkKey &key)
Definition: types.h:73
ChunkKeyToChunkMap chunkIndex_
Definition: FileMgr.h:297
#define CHECK(condition)
Definition: Logger.h:197
mapd_shared_mutex chunkIndexMutex_
Definition: FileMgr.h:339

+ Here is the call graph for this function:

void File_Namespace::FileMgr::getChunkMetadataVecForKeyPrefix ( ChunkMetadataVector chunkMetadataVec,
const ChunkKey keyPrefix 
)
override

Definition at line 993 of file FileMgr.cpp.

References chunkIndex_, and chunkIndexMutex_.

994  {
995  mapd_unique_lock<mapd_shared_mutex> chunkIndexWriteLock(
996  chunkIndexMutex_); // is this guarding the right structure? it look slike we oly
997  // read here for chunk
998  auto chunkIt = chunkIndex_.lower_bound(keyPrefix);
999  if (chunkIt == chunkIndex_.end()) {
1000  return; // throw?
1001  }
1002  while (chunkIt != chunkIndex_.end() &&
1003  std::search(chunkIt->first.begin(),
1004  chunkIt->first.begin() + keyPrefix.size(),
1005  keyPrefix.begin(),
1006  keyPrefix.end()) != chunkIt->first.begin() + keyPrefix.size()) {
1007  if (chunkIt->second->hasEncoder()) {
1008  auto chunk_metadata = std::make_shared<ChunkMetadata>();
1009  chunkIt->second->encoder_->getMetadata(chunk_metadata);
1010  chunkMetadataVec.emplace_back(chunkIt->first, chunk_metadata);
1011  }
1012  chunkIt++;
1013  }
1014 }
ChunkKeyToChunkMap chunkIndex_
Definition: FileMgr.h:297
mapd_shared_mutex chunkIndexMutex_
Definition: FileMgr.h:339
bool File_Namespace::FileMgr::getDBConvert ( ) const

Definition at line 1048 of file FileMgr.cpp.

References File_Namespace::GlobalFileMgr::getDBConvert(), and gfm_.

1048  {
1049  return gfm_->getDBConvert();
1050 }
GlobalFileMgr * gfm_
Definition: FileMgr.h:317

+ Here is the call graph for this function:

int32_t File_Namespace::FileMgr::getDBVersion ( ) const

Index for looking up chunks.

Definition at line 1044 of file FileMgr.cpp.

References File_Namespace::GlobalFileMgr::getDBVersion(), and gfm_.

Referenced by createTopLevelMetadata().

1044  {
1045  return gfm_->getDBVersion();
1046 }
GlobalFileMgr * gfm_
Definition: FileMgr.h:317

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

FILE * File_Namespace::FileMgr::getFileForFileId ( const int32_t  fileId)

Returns FILE pointer associated with requested fileId.

See Also
FileBuffer

Definition at line 988 of file FileMgr.cpp.

References CHECK, and files_.

Referenced by File_Namespace::FileBuffer::readMetadata(), and File_Namespace::FileBuffer::writeMetadata().

988  {
989  CHECK(fileId >= 0 && static_cast<size_t>(fileId) < files_.size());
990  return files_[fileId]->f;
991 }
std::vector< FileInfo * > files_
Definition: FileMgr.h:322
#define CHECK(condition)
Definition: Logger.h:197

+ Here is the caller graph for this function:

FileInfo* File_Namespace::FileMgr::getFileInfoForFileId ( const int32_t  fileId)
inline

Definition at line 187 of file FileMgr.h.

Referenced by File_Namespace::FileBuffer::append(), File_Namespace::FileBuffer::copyPage(), copyPage(), File_Namespace::FileBuffer::freePage(), File_Namespace::readForThread(), File_Namespace::FileBuffer::write(), and File_Namespace::FileBuffer::writeHeader().

187 { return files_[fileId]; }
std::vector< FileInfo * > files_
Definition: FileMgr.h:322

+ Here is the caller graph for this function:

std::string File_Namespace::FileMgr::getFileMgrBasePath ( ) const
inline

Definition at line 302 of file FileMgr.h.

Referenced by closeRemovePhysical().

302 { return fileMgrBasePath_; }
std::string fileMgrBasePath_
Definition: FileMgr.h:320

+ Here is the caller graph for this function:

size_t File_Namespace::FileMgr::getInUseSize ( )
inlineoverride

Definition at line 183 of file FileMgr.h.

183 { return 0; }
size_t File_Namespace::FileMgr::getMaxSize ( )
inlineoverride

Definition at line 182 of file FileMgr.h.

182 { return 0; }
FileMetadata File_Namespace::FileMgr::getMetadataForFile ( const boost::filesystem::directory_iterator &  fileIterator)

Definition at line 161 of file FileMgr.cpp.

References CHECK_EQ, logger::FATAL, File_Namespace::FileMetadata::file_id, File_Namespace::FileMetadata::file_path, omnisci::file_size(), File_Namespace::FileMetadata::file_size, File_Namespace::FileMetadata::is_data_file, LOG, MAPD_FILE_EXT, File_Namespace::FileMetadata::num_pages, and File_Namespace::FileMetadata::page_size.

Referenced by getStorageStats(), and init().

162  {
163  FileMetadata fileMetadata;
164  fileMetadata.is_data_file = false;
165  fileMetadata.file_path = fileIterator->path().string();
166  if (!boost::filesystem::is_regular_file(fileIterator->status())) {
167  return fileMetadata;
168  }
169  // note that boost::filesystem leaves preceding dot on
170  // extension - hence MAPD_FILE_EXT is ".mapd"
171  std::string extension(fileIterator->path().extension().string());
172  if (extension == MAPD_FILE_EXT) {
173  std::string fileStem(fileIterator->path().stem().string());
174  // remove trailing dot if any
175  if (fileStem.size() > 0 && fileStem.back() == '.') {
176  fileStem = fileStem.substr(0, fileStem.size() - 1);
177  }
178  size_t dotPos = fileStem.find_last_of("."); // should only be one
179  if (dotPos == std::string::npos) {
180  LOG(FATAL) << "File `" << fileIterator->path()
181  << "` does not carry page size information in the filename.";
182  }
183  fileMetadata.is_data_file = true;
184  fileMetadata.file_id = boost::lexical_cast<int>(fileStem.substr(0, dotPos));
185  fileMetadata.page_size =
186  boost::lexical_cast<size_t>(fileStem.substr(dotPos + 1, fileStem.size()));
187 
188  fileMetadata.file_size = boost::filesystem::file_size(fileMetadata.file_path);
189  CHECK_EQ(fileMetadata.file_size % fileMetadata.page_size,
190  size_t(0)); // should be no partial pages
191  fileMetadata.num_pages = fileMetadata.file_size / fileMetadata.page_size;
192  }
193  return fileMetadata;
194 }
#define CHECK_EQ(x, y)
Definition: Logger.h:205
#define LOG(tag)
Definition: Logger.h:188
#define MAPD_FILE_EXT
Definition: File.h:25
size_t file_size(const int fd)
Definition: omnisci_fs.cpp:31

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

MgrType File_Namespace::FileMgr::getMgrType ( )
inlineoverride

Definition at line 177 of file FileMgr.h.

177 { return FILE_MGR; };
size_t File_Namespace::FileMgr::getNumChunks ( )
inlineoverride

Definition at line 289 of file FileMgr.h.

289  {
290  mapd_shared_lock<mapd_shared_mutex> read_lock(chunkIndexMutex_);
291  return chunkIndex_.size();
292  }
ChunkKeyToChunkMap chunkIndex_
Definition: FileMgr.h:297
mapd_shared_lock< mapd_shared_mutex > read_lock
mapd_shared_mutex chunkIndexMutex_
Definition: FileMgr.h:339
size_t File_Namespace::FileMgr::getNumReaderThreads ( )
inline

Returns number of threads defined by parameter num-reader-threads which should be used during initial load and consequent read of data.

Definition at line 278 of file FileMgr.h.

Referenced by File_Namespace::FileBuffer::read().

278 { return num_reader_threads_; }
size_t num_reader_threads_
Maps page sizes to FileInfo objects.
Definition: FileMgr.h:324

+ Here is the caller graph for this function:

size_t File_Namespace::FileMgr::getNumUsedMetadataPages ( ) const

Definition at line 1025 of file FileMgr.cpp.

References chunkIndex_, and chunkIndexMutex_.

1025  {
1026  size_t num_used_metadata_pages = 0;
1027  mapd_shared_lock<mapd_shared_mutex> read_lock(chunkIndexMutex_);
1028  for (const auto& chunkIt : chunkIndex_) {
1029  num_used_metadata_pages += chunkIt.second->numMetadataPages();
1030  }
1031  return num_used_metadata_pages;
1032 }
ChunkKeyToChunkMap chunkIndex_
Definition: FileMgr.h:297
mapd_shared_lock< mapd_shared_mutex > read_lock
mapd_shared_mutex chunkIndexMutex_
Definition: FileMgr.h:339
size_t File_Namespace::FileMgr::getNumUsedMetadataPagesForChunkKey ( const ChunkKey chunkKey) const

Definition at line 1034 of file FileMgr.cpp.

References chunkIndex_, and chunkIndexMutex_.

1034  {
1035  mapd_shared_lock<mapd_shared_mutex> read_lock(chunkIndexMutex_);
1036  const auto& chunkIt = chunkIndex_.find(chunkKey);
1037  if (chunkIt != chunkIndex_.end()) {
1038  return chunkIt->second->numMetadataPages();
1039  } else {
1040  throw std::runtime_error("Chunk was not found.");
1041  }
1042 }
ChunkKeyToChunkMap chunkIndex_
Definition: FileMgr.h:297
mapd_shared_lock< mapd_shared_mutex > read_lock
mapd_shared_mutex chunkIndexMutex_
Definition: FileMgr.h:339
size_t File_Namespace::FileMgr::getNumUsedPages ( ) const

Definition at line 1016 of file FileMgr.cpp.

References files_, and files_rw_mutex_.

1016  {
1017  size_t num_used_pages = 0;
1018  mapd_shared_lock<mapd_shared_mutex> read_lock(files_rw_mutex_);
1019  for (const auto file : files_) {
1020  num_used_pages += (file->numPages - file->freePages.size());
1021  }
1022  return num_used_pages;
1023 }
std::vector< FileInfo * > files_
Definition: FileMgr.h:322
mapd_shared_lock< mapd_shared_mutex > read_lock
mapd_shared_mutex files_rw_mutex_
Definition: FileMgr.h:340
StorageStats File_Namespace::FileMgr::getStorageStats ( )

Definition at line 319 of file FileMgr.cpp.

References CHECK, File_Namespace::StorageStats::data_file_count, defaultPageSize_, File_Namespace::StorageStats::epoch, File_Namespace::StorageStats::epoch_floor, epochFloor(), logger::FATAL, File_Namespace::FileMetadata::file_path, File_Namespace::FileMetadata::file_size, fileMgrBasePath_, files_, files_rw_mutex_, getMetadataForFile(), File_Namespace::FileMetadata::is_data_file, isFullyInitted_, lastCheckpointedEpoch(), LOG, File_Namespace::StorageStats::metadata_file_count, METADATA_PAGE_SIZE, File_Namespace::FileMetadata::num_pages, File_Namespace::FileMetadata::page_size, File_Namespace::StorageStats::total_data_file_size, File_Namespace::StorageStats::total_data_page_count, File_Namespace::StorageStats::total_free_data_page_count, File_Namespace::StorageStats::total_free_metadata_page_count, File_Namespace::StorageStats::total_metadata_file_size, File_Namespace::StorageStats::total_metadata_page_count, and UNREACHABLE.

319  {
320  StorageStats storage_stats;
321  mapd_shared_lock<mapd_shared_mutex> read_lock(files_rw_mutex_);
322  if (!isFullyInitted_) {
323  CHECK(!fileMgrBasePath_.empty());
324  boost::filesystem::path path(fileMgrBasePath_);
325  if (boost::filesystem::exists(path)) {
326  if (!boost::filesystem::is_directory(path)) {
327  LOG(FATAL) << "getStorageStats: Specified path '" << fileMgrBasePath_
328  << "' for table data is not a directory.";
329  }
330 
331  storage_stats.epoch = lastCheckpointedEpoch();
332  storage_stats.epoch_floor = epochFloor();
333  boost::filesystem::directory_iterator
334  endItr; // default construction yields past-the-end
335  for (boost::filesystem::directory_iterator fileIt(path); fileIt != endItr;
336  ++fileIt) {
337  FileMetadata file_metadata = getMetadataForFile(fileIt);
338  if (file_metadata.is_data_file) {
339  if (file_metadata.page_size == METADATA_PAGE_SIZE) {
340  storage_stats.metadata_file_count++;
341  storage_stats.total_metadata_file_size += file_metadata.file_size;
342  storage_stats.total_metadata_page_count += file_metadata.num_pages;
343  } else if (file_metadata.page_size == defaultPageSize_) {
344  storage_stats.data_file_count++;
345  storage_stats.total_data_file_size += file_metadata.file_size;
346  storage_stats.total_data_page_count += file_metadata.num_pages;
347  } else {
348  UNREACHABLE() << "Found file with unexpected page size. Page size: "
349  << file_metadata.page_size
350  << ", file path: " << file_metadata.file_path;
351  }
352  }
353  }
354  }
355  } else {
356  storage_stats.epoch = lastCheckpointedEpoch();
357  storage_stats.epoch_floor = epochFloor();
358 
359  // We already initialized this table so take the faster path of walking through the
360  // FileInfo objects and getting metadata from there
361  for (const auto& file_info : files_) {
362  if (file_info->pageSize == METADATA_PAGE_SIZE) {
363  storage_stats.metadata_file_count++;
364  storage_stats.total_metadata_file_size +=
365  file_info->pageSize * file_info->numPages;
366  storage_stats.total_metadata_page_count += file_info->numPages;
367  if (storage_stats.total_free_metadata_page_count) {
368  storage_stats.total_free_metadata_page_count.value() +=
369  file_info->freePages.size();
370  } else {
371  storage_stats.total_free_metadata_page_count = file_info->freePages.size();
372  }
373  } else if (file_info->pageSize == defaultPageSize_) {
374  storage_stats.data_file_count++;
375  storage_stats.total_data_file_size += file_info->pageSize * file_info->numPages;
376  storage_stats.total_data_page_count += file_info->numPages;
377  if (storage_stats.total_free_data_page_count) {
378  storage_stats.total_free_data_page_count.value() += file_info->freePages.size();
379  } else {
380  storage_stats.total_free_data_page_count = file_info->freePages.size();
381  }
382  } else {
383  UNREACHABLE() << "Found file with unexpected page size. Page size: "
384  << file_info->pageSize;
385  }
386  }
387  }
388  return storage_stats;
389 }
#define METADATA_PAGE_SIZE
Definition: FileBuffer.h:37
FileMetadata getMetadataForFile(const boost::filesystem::directory_iterator &fileIterator)
Definition: FileMgr.cpp:161
#define LOG(tag)
Definition: Logger.h:188
#define UNREACHABLE()
Definition: Logger.h:241
std::string fileMgrBasePath_
Definition: FileMgr.h:320
int32_t epochFloor()
Definition: FileMgr.h:247
std::vector< FileInfo * > files_
Definition: FileMgr.h:322
size_t defaultPageSize_
number of threads used when loading data
Definition: FileMgr.h:325
mapd_shared_lock< mapd_shared_mutex > read_lock
#define CHECK(condition)
Definition: Logger.h:197
mapd_shared_mutex files_rw_mutex_
Definition: FileMgr.h:340
int32_t lastCheckpointedEpoch()
Returns value of epoch at last checkpoint.
Definition: FileMgr.h:265

+ Here is the call graph for this function:

std::string File_Namespace::FileMgr::getStringMgrType ( )
inlineoverride

Definition at line 178 of file FileMgr.h.

178 { return ToString(FILE_MGR); }
uint64_t File_Namespace::FileMgr::getTotalFileSize ( ) const

Definition at line 1186 of file FileMgr.cpp.

References DBMetaFile_, epochFile_, files_, and File_Namespace::fileSize().

Referenced by ShowDiskCacheUsageCommand::execute().

1186  {
1187  uint64_t total_size = 0;
1188  for (const auto& file : files_) {
1189  total_size += file->size();
1190  }
1191  if (epochFile_) {
1192  total_size += fileSize(epochFile_);
1193  }
1194  if (DBMetaFile_) {
1195  total_size += fileSize(DBMetaFile_);
1196  }
1197  return total_size;
1198 }
std::vector< FileInfo * > files_
Definition: FileMgr.h:322
size_t fileSize(FILE *f)
Returns the size of the specified file.
Definition: File.cpp:176

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

int32_t File_Namespace::FileMgr::incrementEpoch ( )
inline

Definition at line 249 of file FileMgr.h.

References logger::FATAL, LOG, and Epoch::max_allowable_epoch().

Referenced by checkpoint(), and init().

249  {
250  int32_t newEpoch = epoch_.increment();
251  epochIsCheckpointed_ = false;
252  // We test for error here instead of in Epoch::increment so we can log FileMgr
253  // metadata
254  if (newEpoch > Epoch::max_allowable_epoch()) {
255  LOG(FATAL) << "Epoch for table (" << fileMgrKey_.first << ", " << fileMgrKey_.second
256  << ") greater than maximum allowed value of "
257  << Epoch::max_allowable_epoch() << ".";
258  }
259  return newEpoch;
260  }
#define LOG(tag)
Definition: Logger.h:188
static int64_t max_allowable_epoch()
Definition: Epoch.h:69
int32_t increment()
Definition: Epoch.h:54
std::pair< const int32_t, const int32_t > fileMgrKey_
Global FileMgr.
Definition: FileMgr.h:318
Epoch epoch_
the index of the next file id
Definition: FileMgr.h:327

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void File_Namespace::FileMgr::init ( const size_t  num_reader_threads,
const int32_t  epochOverride 
)

Definition at line 196 of file FileMgr.cpp.

References Epoch::ceiling(), chunkIndex_, coreInit(), createEpochFile(), epoch(), epoch_, EPOCH_FILENAME, logger::FATAL, File_Namespace::FileMetadata::file_id, FILE_MGR_VERSION_FILENAME, File_Namespace::FileMetadata::file_path, fileMgrBasePath_, fileMgrVersion_, Epoch::floor(), free_page(), free_pages_, getMetadataForFile(), File_Namespace::headerCompare(), incrementEpoch(), logger::INFO, File_Namespace::FileMetadata::is_data_file, isFullyInitted_, latestFileMgrVersion_, LOG, mutex_free_page_, nextFileId_, File_Namespace::FileMetadata::num_pages, num_reader_threads_, openExistingFile(), File_Namespace::FileMetadata::page_size, processFileFutures(), rollOffOldData(), setEpoch(), gpu_enabled::sort(), timer_start(), timer_stop(), VLOG, writeAndSyncEpochToDisk(), and writeAndSyncVersionToDisk().

Referenced by FileMgr().

196  {
197  // if epochCeiling = -1 this means open from epoch file
198 
199  const bool dataExists = coreInit();
200  if (dataExists) {
201  if (epochOverride != -1) { // if opening at specified epoch
202  setEpoch(epochOverride);
203  }
204  auto clock_begin = timer_start();
205 
206  boost::filesystem::directory_iterator
207  endItr; // default construction yields past-the-end
208  int32_t maxFileId = -1;
209  int32_t fileCount = 0;
210  int32_t threadCount = std::thread::hardware_concurrency();
211  std::vector<HeaderInfo> headerVec;
212  std::vector<std::future<std::vector<HeaderInfo>>> file_futures;
213  boost::filesystem::path path(fileMgrBasePath_);
214  for (boost::filesystem::directory_iterator fileIt(path); fileIt != endItr; ++fileIt) {
215  FileMetadata fileMetadata = getMetadataForFile(fileIt);
216  if (fileMetadata.is_data_file) {
217  maxFileId = std::max(maxFileId, fileMetadata.file_id);
218  file_futures.emplace_back(std::async(std::launch::async, [fileMetadata, this] {
219  std::vector<HeaderInfo> tempHeaderVec;
220  openExistingFile(fileMetadata.file_path,
221  fileMetadata.file_id,
222  fileMetadata.page_size,
223  fileMetadata.num_pages,
224  tempHeaderVec);
225  return tempHeaderVec;
226  }));
227  fileCount++;
228  if (fileCount % threadCount == 0) {
229  processFileFutures(file_futures, headerVec);
230  }
231  }
232  }
233 
234  if (file_futures.size() > 0) {
235  processFileFutures(file_futures, headerVec);
236  }
237  int64_t queue_time_ms = timer_stop(clock_begin);
238 
239  LOG(INFO) << "Completed Reading table's file metadata, Elapsed time : "
240  << queue_time_ms << "ms Epoch: " << epoch_.ceiling()
241  << " files read: " << fileCount << " table location: '" << fileMgrBasePath_
242  << "'";
243 
244  /* Sort headerVec so that all HeaderInfos
245  * from a chunk will be grouped together
246  * and in order of increasing PageId
247  * - Version Epoch */
248 
249  std::sort(headerVec.begin(), headerVec.end(), headerCompare);
250 
251  /* Goal of next section is to find sequences in the
252  * sorted headerVec of the same ChunkId, which we
253  * can then initiate a FileBuffer with */
254 
255  VLOG(4) << "Number of Headers in Vector: " << headerVec.size();
256  if (headerVec.size() > 0) {
257  ChunkKey lastChunkKey = headerVec.begin()->chunkKey;
258  auto startIt = headerVec.begin();
259 
260  for (auto headerIt = headerVec.begin() + 1; headerIt != headerVec.end();
261  ++headerIt) {
262  if (headerIt->chunkKey != lastChunkKey) {
263  chunkIndex_[lastChunkKey] =
264  new FileBuffer(this, /*pageSize,*/ lastChunkKey, startIt, headerIt);
265  lastChunkKey = headerIt->chunkKey;
266  startIt = headerIt;
267  }
268  }
269  // now need to insert last Chunk
270  chunkIndex_[lastChunkKey] =
271  new FileBuffer(this, /*pageSize,*/ lastChunkKey, startIt, headerVec.end());
272  }
273  nextFileId_ = maxFileId + 1;
275  true /* shouldCheckpoint - only happens if data is rolled off */);
276  incrementEpoch();
277  mapd_unique_lock<mapd_shared_mutex> freePagesWriteLock(mutex_free_page_);
278  for (auto& free_page : free_pages_) {
279  free_page.first->freePageDeferred(free_page.second);
280  }
281  free_pages_.clear();
282  } else {
283  boost::filesystem::path path(fileMgrBasePath_);
284  if (!boost::filesystem::create_directory(path)) {
285  LOG(FATAL) << "Could not create data directory: " << path;
286  }
288  if (epochOverride != -1) {
289  epoch_.floor(epochOverride);
290  epoch_.ceiling(epochOverride);
291  } else {
292  // These are default constructor values for epoch_, but resetting here for clarity
293  epoch_.floor(0);
294  epoch_.ceiling(0);
295  }
296 
300  incrementEpoch();
301  }
302 
303  /* define number of reader threads to be used */
304  size_t num_hardware_based_threads =
305  std::thread::hardware_concurrency(); // # of threads is based on # of cores on the
306  // host
307  if (num_reader_threads == 0) { // # of threads has not been defined by user
308  num_reader_threads_ = num_hardware_based_threads;
309  } else {
310  if (num_reader_threads > num_hardware_based_threads) {
311  num_reader_threads_ = num_hardware_based_threads;
312  } else {
313  num_reader_threads_ = num_reader_threads;
314  }
315  }
316  isFullyInitted_ = true;
317 }
std::vector< int > ChunkKey
Definition: types.h:37
FileMetadata getMetadataForFile(const boost::filesystem::directory_iterator &fileIterator)
Definition: FileMgr.cpp:161
mapd_shared_mutex mutex_free_page_
Definition: FileMgr.h:342
void createEpochFile(const std::string &epochFileName)
Definition: FileMgr.cpp:578
#define LOG(tag)
Definition: Logger.h:188
void rollOffOldData(const int32_t epochCeiling, const bool shouldCheckpoint)
Definition: FileMgr.cpp:652
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
TypeR::rep timer_stop(Type clock_begin)
Definition: measure.h:48
int32_t floor() const
Definition: Epoch.h:43
int32_t ceiling() const
Definition: Epoch.h:44
std::string fileMgrBasePath_
Definition: FileMgr.h:320
bool headerCompare(const HeaderInfo &firstElem, const HeaderInfo &secondElem)
Definition: FileMgr.cpp:53
const int32_t latestFileMgrVersion_
Definition: FileMgr.h:335
void writeAndSyncVersionToDisk(const std::string &versionFileName, const int32_t version)
Definition: FileMgr.cpp:1086
int32_t incrementEpoch()
Definition: FileMgr.h:249
void processFileFutures(std::vector< std::future< std::vector< HeaderInfo >>> &file_futures, std::vector< HeaderInfo > &headerVec)
Definition: FileMgr.cpp:391
ChunkKeyToChunkMap chunkIndex_
Definition: FileMgr.h:297
constexpr char EPOCH_FILENAME[]
Definition: FileMgr.cpp:43
std::vector< std::pair< FileInfo *, int32_t > > free_pages_
Definition: FileMgr.h:343
size_t num_reader_threads_
Maps page sizes to FileInfo objects.
Definition: FileMgr.h:324
FileInfo * openExistingFile(const std::string &path, const int32_t fileId, const size_t pageSize, const size_t numPages, std::vector< HeaderInfo > &headerVec)
Definition: FileMgr.cpp:940
void writeAndSyncEpochToDisk()
Definition: FileMgr.cpp:628
int32_t epoch()
Returns current value of epoch - should be one greater than recorded at last checkpoint.
Definition: FileMgr.h:245
void free_page(std::pair< FileInfo *, int32_t > &&page)
Definition: FileMgr.cpp:1177
constexpr char FILE_MGR_VERSION_FILENAME[]
Definition: FileMgr.cpp:45
void setEpoch(const int32_t newEpoch)
Definition: FileMgr.cpp:1165
bool coreInit()
Determines file path, and if exists, runs file migration and opens and reads epoch file...
Definition: FileMgr.cpp:141
Epoch epoch_
the index of the next file id
Definition: FileMgr.h:327
#define VLOG(n)
Definition: Logger.h:291
Type timer_start()
Definition: measure.h:42

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void File_Namespace::FileMgr::init ( const std::string &  dataPathToConvertFrom,
const int32_t  epochOverride 
)

Definition at line 405 of file FileMgr.cpp.

References CHECK, chunkIndex_, copyPage(), EPOCH_FILENAME, logger::FATAL, File_Namespace::FileMetadata::file_id, File_Namespace::FileMetadata::file_path, File_Namespace::GlobalFileMgr::getFileMgr(), getMetadataForFile(), File_Namespace::FileBuffer::getMultiPage(), gfm_, File_Namespace::headerCompare(), File_Namespace::FileMetadata::is_data_file, isFullyInitted_, LOG, File_Namespace::FileBuffer::multiPages_, nextFileId_, File_Namespace::FileMetadata::num_pages, openAndReadEpochFile(), openExistingFile(), File_Namespace::FileMetadata::page_size, File_Namespace::FileBuffer::pageDataSize(), File_Namespace::FileBuffer::pageSize(), processFileFutures(), File_Namespace::MultiPage::push(), requestFreePage(), File_Namespace::FileBuffer::reservedHeaderSize(), Data_Namespace::AbstractBuffer::setDirty(), setEpoch(), Data_Namespace::AbstractBuffer::setSize(), Data_Namespace::AbstractBuffer::size(), gpu_enabled::sort(), Data_Namespace::AbstractBuffer::syncEncoder(), and File_Namespace::FileBuffer::writeHeader().

406  {
407  int32_t converted_data_epoch = 0;
408  boost::filesystem::path path(dataPathToConvertFrom);
409  if (boost::filesystem::exists(path)) {
410  if (!boost::filesystem::is_directory(path)) {
411  LOG(FATAL) << "Specified path `" << path << "` is not a directory.";
412  }
414 
415  if (epochOverride != -1) { // if opening at previous epoch
416  setEpoch(epochOverride);
417  }
418 
419  boost::filesystem::directory_iterator
420  endItr; // default construction yields past-the-end
421  int32_t maxFileId = -1;
422  int32_t fileCount = 0;
423  int32_t threadCount = std::thread::hardware_concurrency();
424  std::vector<HeaderInfo> headerVec;
425  std::vector<std::future<std::vector<HeaderInfo>>> file_futures;
426  for (boost::filesystem::directory_iterator fileIt(path); fileIt != endItr; ++fileIt) {
427  FileMetadata fileMetadata = getMetadataForFile(fileIt);
428  if (fileMetadata.is_data_file) {
429  maxFileId = std::max(maxFileId, fileMetadata.file_id);
430  file_futures.emplace_back(std::async(std::launch::async, [fileMetadata, this] {
431  std::vector<HeaderInfo> tempHeaderVec;
432  openExistingFile(fileMetadata.file_path,
433  fileMetadata.file_id,
434  fileMetadata.page_size,
435  fileMetadata.num_pages,
436  tempHeaderVec);
437  return tempHeaderVec;
438  }));
439  fileCount++;
440  if (fileCount % threadCount) {
441  processFileFutures(file_futures, headerVec);
442  }
443  }
444  }
445 
446  if (file_futures.size() > 0) {
447  processFileFutures(file_futures, headerVec);
448  }
449 
450  /* Sort headerVec so that all HeaderInfos
451  * from a chunk will be grouped together
452  * and in order of increasing PageId
453  * - Version Epoch */
454 
455  std::sort(headerVec.begin(), headerVec.end(), headerCompare);
456 
457  /* Goal of next section is to find sequences in the
458  * sorted headerVec of the same ChunkId, which we
459  * can then initiate a FileBuffer with */
460 
461  if (headerVec.size() > 0) {
462  ChunkKey lastChunkKey = headerVec.begin()->chunkKey;
463  auto startIt = headerVec.begin();
464 
465  for (auto headerIt = headerVec.begin() + 1; headerIt != headerVec.end();
466  ++headerIt) {
467  if (headerIt->chunkKey != lastChunkKey) {
468  FileMgr* c_fm_ =
469  dynamic_cast<File_Namespace::FileMgr*>(gfm_->getFileMgr(lastChunkKey));
470  CHECK(c_fm_);
471  FileBuffer* srcBuf = new FileBuffer(this, lastChunkKey, startIt, headerIt);
472  chunkIndex_[lastChunkKey] = srcBuf;
473  FileBuffer* destBuf = new FileBuffer(c_fm_, srcBuf->pageSize(), lastChunkKey);
474  c_fm_->chunkIndex_[lastChunkKey] = destBuf;
475  destBuf->syncEncoder(srcBuf);
476  destBuf->setSize(srcBuf->size());
477  destBuf->setDirty(); // this needs to be set to force writing out metadata
478  // files from "checkpoint()" call
479 
480  size_t totalNumPages = srcBuf->getMultiPage().size();
481  for (size_t pageNum = 0; pageNum < totalNumPages; pageNum++) {
482  Page srcPage = srcBuf->getMultiPage()[pageNum].current().page;
483  Page destPage = c_fm_->requestFreePage(
484  srcBuf->pageSize(),
485  false); // may modify and use api "FileBuffer::addNewMultiPage" instead
486  MultiPage multiPage(srcBuf->pageSize());
487  multiPage.push(destPage, converted_data_epoch);
488  destBuf->multiPages_.push_back(multiPage);
489  size_t reservedHeaderSize = srcBuf->reservedHeaderSize();
490  copyPage(
491  srcPage, c_fm_, destPage, reservedHeaderSize, srcBuf->pageDataSize(), 0);
492  destBuf->writeHeader(destPage, pageNum, converted_data_epoch, false);
493  }
494  lastChunkKey = headerIt->chunkKey;
495  startIt = headerIt;
496  }
497  }
498 
499  // now need to insert last Chunk
500  FileMgr* c_fm_ =
501  dynamic_cast<File_Namespace::FileMgr*>(gfm_->getFileMgr(lastChunkKey));
502  FileBuffer* srcBuf = new FileBuffer(this, lastChunkKey, startIt, headerVec.end());
503  chunkIndex_[lastChunkKey] = srcBuf;
504  FileBuffer* destBuf = new FileBuffer(c_fm_, srcBuf->pageSize(), lastChunkKey);
505  c_fm_->chunkIndex_[lastChunkKey] = destBuf;
506  destBuf->syncEncoder(srcBuf);
507  destBuf->setSize(srcBuf->size());
508  destBuf->setDirty(); // this needs to be set to write out metadata file from the
509  // "checkpoint()" call
510 
511  size_t totalNumPages = srcBuf->getMultiPage().size();
512  for (size_t pageNum = 0; pageNum < totalNumPages; pageNum++) {
513  Page srcPage = srcBuf->getMultiPage()[pageNum].current().page;
514  Page destPage = c_fm_->requestFreePage(
515  srcBuf->pageSize(),
516  false); // may modify and use api "FileBuffer::addNewMultiPage" instead
517  MultiPage multiPage(srcBuf->pageSize());
518  multiPage.push(destPage, converted_data_epoch);
519  destBuf->multiPages_.push_back(multiPage);
520  size_t reservedHeaderSize = srcBuf->reservedHeaderSize();
521  copyPage(srcPage, c_fm_, destPage, reservedHeaderSize, srcBuf->pageDataSize(), 0);
522  destBuf->writeHeader(destPage, pageNum, converted_data_epoch, false);
523  }
524  }
525  nextFileId_ = maxFileId + 1;
526  } else {
527  if (!boost::filesystem::create_directory(path)) {
528  LOG(FATAL) << "Specified path does not exist: " << path;
529  }
530  }
531  isFullyInitted_ = true;
532 }
std::vector< int > ChunkKey
Definition: types.h:37
FileMetadata getMetadataForFile(const boost::filesystem::directory_iterator &fileIterator)
Definition: FileMgr.cpp:161
#define LOG(tag)
Definition: Logger.h:188
void copyPage(Page &srcPage, FileMgr *destFileMgr, Page &destPage, const size_t reservedHeaderSize, const size_t numBytes, const size_t offset)
Definition: FileMgr.cpp:556
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
GlobalFileMgr * gfm_
Definition: FileMgr.h:317
bool headerCompare(const HeaderInfo &firstElem, const HeaderInfo &secondElem)
Definition: FileMgr.cpp:53
FileMgr(const int32_t deviceId, GlobalFileMgr *gfm, const std::pair< const int32_t, const int32_t > fileMgrKey, const int32_t max_rollback_epochs=-1, const size_t num_reader_threads=0, const int32_t epoch=-1, const size_t defaultPageSize=DEFAULT_PAGE_SIZE)
Constructor.
void processFileFutures(std::vector< std::future< std::vector< HeaderInfo >>> &file_futures, std::vector< HeaderInfo > &headerVec)
Definition: FileMgr.cpp:391
ChunkKeyToChunkMap chunkIndex_
Definition: FileMgr.h:297
constexpr char EPOCH_FILENAME[]
Definition: FileMgr.cpp:43
FileInfo * openExistingFile(const std::string &path, const int32_t fileId, const size_t pageSize, const size_t numPages, std::vector< HeaderInfo > &headerVec)
Definition: FileMgr.cpp:940
void openAndReadEpochFile(const std::string &epochFileName)
Definition: FileMgr.cpp:609
AbstractBufferMgr * getFileMgr(const int32_t db_id, const int32_t tb_id)
#define CHECK(condition)
Definition: Logger.h:197
void setEpoch(const int32_t newEpoch)
Definition: FileMgr.cpp:1165

+ Here is the call graph for this function:

bool File_Namespace::FileMgr::isAllocationCapped ( )
inlineoverride

Definition at line 185 of file FileMgr.h.

185 { return false; }
bool File_Namespace::FileMgr::isBufferOnDevice ( const ChunkKey key)
override

Definition at line 725 of file FileMgr.cpp.

References chunkIndex_, and chunkIndexMutex_.

725  {
726  mapd_shared_lock<mapd_shared_mutex> chunkIndexReadLock(chunkIndexMutex_);
727  return chunkIndex_.find(key) != chunkIndex_.end();
728 }
ChunkKeyToChunkMap chunkIndex_
Definition: FileMgr.h:297
mapd_shared_mutex chunkIndexMutex_
Definition: FileMgr.h:339
int32_t File_Namespace::FileMgr::lastCheckpointedEpoch ( )
inline

Returns value of epoch at last checkpoint.

Definition at line 265 of file FileMgr.h.

Referenced by checkpoint(), File_Namespace::GlobalFileMgr::existsDiffBetweenFileMgrParamsAndFileMgr(), and getStorageStats().

265  {
266  return epoch() - (epochIsCheckpointed_ ? 0 : 1);
267  }
int32_t epoch()
Returns current value of epoch - should be one greater than recorded at last checkpoint.
Definition: FileMgr.h:245

+ Here is the caller graph for this function:

int32_t File_Namespace::FileMgr::maxRollbackEpochs ( )
inline

Returns value max_rollback_epochs.

Definition at line 272 of file FileMgr.h.

Referenced by File_Namespace::GlobalFileMgr::existsDiffBetweenFileMgrParamsAndFileMgr().

272 { return maxRollbackEpochs_; }
int32_t maxRollbackEpochs_
Definition: FileMgr.h:319

+ Here is the caller graph for this function:

void File_Namespace::FileMgr::migrateEpochFileV0 ( )
private

Definition at line 1114 of file FileMgr.cpp.

References Epoch::ceiling(), createEpochFile(), epoch_, EPOCH_FILENAME, FILE_MGR_VERSION_FILENAME, fileMgrBasePath_, Epoch::floor(), logger::INFO, LEGACY_EPOCH_FILENAME, LOG, Epoch::min_allowable_epoch(), openAndReadLegacyEpochFile(), writeAndSyncEpochToDisk(), and writeAndSyncVersionToDisk().

Referenced by migrateToLatestFileMgrVersion().

1114  {
1115  const std::string versionFilePath(fileMgrBasePath_ + "/" + FILE_MGR_VERSION_FILENAME);
1116  LOG(INFO) << "Migrating file format version from 0 to 1 for `" << versionFilePath;
1121  int32_t migrationCompleteVersion = 1;
1122  writeAndSyncVersionToDisk(FILE_MGR_VERSION_FILENAME, migrationCompleteVersion);
1123 }
int32_t openAndReadLegacyEpochFile(const std::string &epochFileName)
Definition: FileMgr.cpp:589
void createEpochFile(const std::string &epochFileName)
Definition: FileMgr.cpp:578
#define LOG(tag)
Definition: Logger.h:188
static int64_t min_allowable_epoch()
Definition: Epoch.h:65
int32_t floor() const
Definition: Epoch.h:43
int32_t ceiling() const
Definition: Epoch.h:44
std::string fileMgrBasePath_
Definition: FileMgr.h:320
void writeAndSyncVersionToDisk(const std::string &versionFileName, const int32_t version)
Definition: FileMgr.cpp:1086
constexpr char LEGACY_EPOCH_FILENAME[]
Definition: FileMgr.cpp:42
constexpr char EPOCH_FILENAME[]
Definition: FileMgr.cpp:43
void writeAndSyncEpochToDisk()
Definition: FileMgr.cpp:628
constexpr char FILE_MGR_VERSION_FILENAME[]
Definition: FileMgr.cpp:45
Epoch epoch_
the index of the next file id
Definition: FileMgr.h:327

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void File_Namespace::FileMgr::migrateToLatestFileMgrVersion ( )
private

Definition at line 1125 of file FileMgr.cpp.

References logger::FATAL, FILE_MGR_VERSION_FILENAME, fileMgrVersion_, INVALID_VERSION, latestFileMgrVersion_, LOG, migrateEpochFileV0(), readVersionFromDisk(), UNREACHABLE, and writeAndSyncVersionToDisk().

Referenced by coreInit().

1125  {
1128  fileMgrVersion_ = 0;
1130  } else if (fileMgrVersion_ > latestFileMgrVersion_) {
1131  LOG(FATAL)
1132  << "Table storage forward compatibility is not supported. Version of OmniSci "
1133  "software used is older than the version of table being read: "
1134  << fileMgrVersion_;
1135  }
1136 
1139  switch (fileMgrVersion_) {
1140  case 0: {
1142  break;
1143  }
1144  default: {
1145  UNREACHABLE();
1146  }
1147  }
1148  fileMgrVersion_++;
1149  }
1150  }
1151 }
int32_t readVersionFromDisk(const std::string &versionFileName) const
Definition: FileMgr.cpp:1068
#define LOG(tag)
Definition: Logger.h:188
#define UNREACHABLE()
Definition: Logger.h:241
const int32_t latestFileMgrVersion_
Definition: FileMgr.h:335
void writeAndSyncVersionToDisk(const std::string &versionFileName, const int32_t version)
Definition: FileMgr.cpp:1086
constexpr int32_t INVALID_VERSION
Definition: FileMgr.cpp:47
constexpr char FILE_MGR_VERSION_FILENAME[]
Definition: FileMgr.cpp:45

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void File_Namespace::FileMgr::openAndReadEpochFile ( const std::string &  epochFileName)
private

Definition at line 609 of file FileMgr.cpp.

References Epoch::byte_size(), epoch_, epochFile_, logger::FATAL, omnisci::file_size(), fileMgrBasePath_, LOG, File_Namespace::open(), File_Namespace::read(), and Epoch::storage_ptr().

Referenced by coreInit(), and init().

609  {
610  if (!epochFile_) { // Check to see if already open
611  std::string epochFilePath(fileMgrBasePath_ + "/" + epochFileName);
612  if (!boost::filesystem::exists(epochFilePath)) {
613  LOG(FATAL) << "Epoch file `" << epochFilePath << "` does not exist";
614  }
615  if (!boost::filesystem::is_regular_file(epochFilePath)) {
616  LOG(FATAL) << "Epoch file `" << epochFilePath << "` is not a regular file";
617  }
618  if (boost::filesystem::file_size(epochFilePath) != 16) {
619  LOG(FATAL) << "Epoch file `" << epochFilePath
620  << "` is not sized properly (current size: "
621  << boost::filesystem::file_size(epochFilePath) << ", expected size: 16)";
622  }
623  epochFile_ = open(epochFilePath);
624  }
626 }
int8_t * storage_ptr()
Definition: Epoch.h:61
#define LOG(tag)
Definition: Logger.h:188
std::string fileMgrBasePath_
Definition: FileMgr.h:320
size_t read(FILE *f, const size_t offset, const size_t size, int8_t *buf)
Reads the specified number of bytes from the offset position in file f into buf.
Definition: File.cpp:118
static size_t byte_size()
Definition: Epoch.h:63
FILE * open(int fileId)
Opens/creates the file with the given id; returns NULL on error.
Definition: File.cpp:88
Epoch epoch_
the index of the next file id
Definition: FileMgr.h:327
size_t file_size(const int fd)
Definition: omnisci_fs.cpp:31

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

int32_t File_Namespace::FileMgr::openAndReadLegacyEpochFile ( const std::string &  epochFileName)
private

Definition at line 589 of file FileMgr.cpp.

References epoch(), logger::FATAL, omnisci::file_size(), fileMgrBasePath_, LOG, File_Namespace::open(), and File_Namespace::read().

Referenced by migrateEpochFileV0().

589  {
590  std::string epochFilePath(fileMgrBasePath_ + "/" + epochFileName);
591  if (!boost::filesystem::exists(epochFilePath)) {
592  return 0;
593  }
594 
595  if (!boost::filesystem::is_regular_file(epochFilePath)) {
596  LOG(FATAL) << "Epoch file `" << epochFilePath << "` is not a regular file";
597  }
598  if (boost::filesystem::file_size(epochFilePath) < 4) {
599  LOG(FATAL) << "Epoch file `" << epochFilePath
600  << "` is not sized properly (current size: "
601  << boost::filesystem::file_size(epochFilePath) << ", expected size: 4)";
602  }
603  FILE* legacyEpochFile = open(epochFilePath);
604  int32_t epoch;
605  read(legacyEpochFile, 0, sizeof(int32_t), (int8_t*)&epoch);
606  return epoch;
607 }
#define LOG(tag)
Definition: Logger.h:188
std::string fileMgrBasePath_
Definition: FileMgr.h:320
size_t read(FILE *f, const size_t offset, const size_t size, int8_t *buf)
Reads the specified number of bytes from the offset position in file f into buf.
Definition: File.cpp:118
int32_t epoch()
Returns current value of epoch - should be one greater than recorded at last checkpoint.
Definition: FileMgr.h:245
FILE * open(int fileId)
Opens/creates the file with the given id; returns NULL on error.
Definition: File.cpp:88
size_t file_size(const int fd)
Definition: omnisci_fs.cpp:31

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

FileInfo * File_Namespace::FileMgr::openExistingFile ( const std::string &  path,
const int32_t  fileId,
const size_t  pageSize,
const size_t  numPages,
std::vector< HeaderInfo > &  headerVec 
)
private

Definition at line 940 of file FileMgr.cpp.

References epoch(), generate_TableFunctionsFactory_init::f, fileIndex_, files_, files_rw_mutex_, File_Namespace::open(), and File_Namespace::FileInfo::openExistingFile().

Referenced by init().

944  {
945  FILE* f = open(path);
946  FileInfo* fInfo = new FileInfo(
947  this, fileId, f, pageSize, numPages, false); // false means don't init file
948 
949  fInfo->openExistingFile(headerVec, epoch());
950  mapd_unique_lock<mapd_shared_mutex> write_lock(files_rw_mutex_);
951  if (fileId >= static_cast<int32_t>(files_.size())) {
952  files_.resize(fileId + 1);
953  }
954  files_[fileId] = fInfo;
955  fileIndex_.insert(std::pair<size_t, int32_t>(pageSize, fileId));
956  return fInfo;
957 }
std::vector< FileInfo * > files_
Definition: FileMgr.h:322
PageSizeFileMMap fileIndex_
A vector of files accessible via a file identifier.
Definition: FileMgr.h:323
int32_t epoch()
Returns current value of epoch - should be one greater than recorded at last checkpoint.
Definition: FileMgr.h:245
FILE * open(int fileId)
Opens/creates the file with the given id; returns NULL on error.
Definition: File.cpp:88
mapd_unique_lock< mapd_shared_mutex > write_lock
mapd_shared_mutex files_rw_mutex_
Definition: FileMgr.h:340

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string File_Namespace::FileMgr::printSlabs ( )
inlineoverride

Definition at line 179 of file FileMgr.h.

179 { return "Not Implemented"; }
void File_Namespace::FileMgr::processFileFutures ( std::vector< std::future< std::vector< HeaderInfo >>> &  file_futures,
std::vector< HeaderInfo > &  headerVec 
)
private

Definition at line 391 of file FileMgr.cpp.

Referenced by init().

393  {
394  for (auto& file_future : file_futures) {
395  file_future.wait();
396  }
397  // concatenate the vectors after thread completes
398  for (auto& file_future : file_futures) {
399  auto tempHeaderVec = file_future.get();
400  headerVec.insert(headerVec.end(), tempHeaderVec.begin(), tempHeaderVec.end());
401  }
402  file_futures.clear();
403 }

+ Here is the caller graph for this function:

FileBuffer * File_Namespace::FileMgr::putBuffer ( const ChunkKey key,
AbstractBuffer d,
const size_t  numBytes = 0 
)
override

Puts the contents of d into the Chunk with the given key.

Parameters
key- Unique identifier for a Chunk.
d- An object representing the source data for the Chunk.
Returns
AbstractBuffer*

Definition at line 804 of file FileMgr.cpp.

References File_Namespace::FileBuffer::append(), CHECK_LT, chunkIndex_, chunkIndexMutex_, Data_Namespace::AbstractBuffer::clearDirtyBits(), createBufferUnlocked(), defaultPageSize_, logger::FATAL, Data_Namespace::AbstractBuffer::getDeviceId(), Data_Namespace::AbstractBuffer::getMemoryPtr(), Data_Namespace::AbstractBuffer::getType(), Data_Namespace::AbstractBuffer::isAppended(), Data_Namespace::AbstractBuffer::isDirty(), Data_Namespace::AbstractBuffer::isUpdated(), LOG, Data_Namespace::AbstractBuffer::setSize(), show_chunk(), Data_Namespace::AbstractBuffer::size(), Data_Namespace::AbstractBuffer::syncEncoder(), UNREACHABLE, and File_Namespace::FileBuffer::write().

806  {
807  // obtain a pointer to the Chunk
808  mapd_unique_lock<mapd_shared_mutex> chunkIndexWriteLock(chunkIndexMutex_);
809  auto chunkIt = chunkIndex_.find(key);
810  FileBuffer* chunk;
811  if (chunkIt == chunkIndex_.end()) {
813  } else {
814  chunk = chunkIt->second;
815  }
816  chunkIndexWriteLock.unlock();
817  size_t oldChunkSize = chunk->size();
818  // write the buffer's data to the Chunk
819  // size_t newChunkSize = numBytes == 0 ? srcBuffer->size() : numBytes;
820  size_t newChunkSize = numBytes == 0 ? srcBuffer->size() : numBytes;
821  if (chunk->isDirty()) {
822  // multiple appends are allowed,
823  // but only single update is allowed
824  if (srcBuffer->isUpdated() && chunk->isUpdated()) {
825  LOG(FATAL) << "Aborting attempt to write a chunk marked dirty. Chunk inconsistency "
826  "for key: "
827  << show_chunk(key);
828  }
829  }
830  if (srcBuffer->isUpdated()) {
831  // chunk size is not changed when fixed rows are updated or are marked as deleted.
832  // but when rows are vacuumed or varlen rows are updated (new rows are appended),
833  // chunk size will change. For vacuum, checkpoint should sync size from cpu to disk.
834  // For varlen update, it takes another route via fragmenter using disk-level buffer.
835  if (0 == numBytes && !chunk->isDirty()) {
836  chunk->setSize(newChunkSize);
837  }
838  //@todo use dirty flags to only flush pages of chunk that need to
839  // be flushed
840  chunk->write((int8_t*)srcBuffer->getMemoryPtr(),
841  newChunkSize,
842  0,
843  srcBuffer->getType(),
844  srcBuffer->getDeviceId());
845  } else if (srcBuffer->isAppended()) {
846  CHECK_LT(oldChunkSize, newChunkSize);
847  chunk->append((int8_t*)srcBuffer->getMemoryPtr() + oldChunkSize,
848  newChunkSize - oldChunkSize,
849  srcBuffer->getType(),
850  srcBuffer->getDeviceId());
851  } else {
852  UNREACHABLE() << "putBuffer() expects a buffer marked is_updated or is_appended";
853  }
854  // chunk->clearDirtyBits(); // Hack: because write and append will set dirty bits
855  //@todo commenting out line above will make sure this metadata is set
856  // but will trigger error on fetch chunk
857  srcBuffer->clearDirtyBits();
858  chunk->syncEncoder(srcBuffer);
859  return chunk;
860 }
#define LOG(tag)
Definition: Logger.h:188
#define UNREACHABLE()
Definition: Logger.h:241
std::string show_chunk(const ChunkKey &key)
Definition: types.h:73
ChunkKeyToChunkMap chunkIndex_
Definition: FileMgr.h:297
#define CHECK_LT(x, y)
Definition: Logger.h:207
size_t defaultPageSize_
number of threads used when loading data
Definition: FileMgr.h:325
FileBuffer * createBufferUnlocked(const ChunkKey &key, size_t pageSize=0, const size_t numBytes=0)
Definition: FileMgr.cpp:707
mapd_shared_mutex chunkIndexMutex_
Definition: FileMgr.h:339

+ Here is the call graph for this function:

int32_t File_Namespace::FileMgr::readVersionFromDisk ( const std::string &  versionFileName) const
private

Definition at line 1068 of file FileMgr.cpp.

References File_Namespace::close(), omnisci::file_size(), fileMgrBasePath_, File_Namespace::open(), File_Namespace::read(), and setup::version.

Referenced by createTopLevelMetadata(), migrateToLatestFileMgrVersion(), and writeAndSyncVersionToDisk().

1068  {
1069  const std::string versionFilePath(fileMgrBasePath_ + "/" + versionFileName);
1070  if (!boost::filesystem::exists(versionFilePath)) {
1071  return -1;
1072  }
1073  if (!boost::filesystem::is_regular_file(versionFilePath)) {
1074  return -1;
1075  }
1076  if (boost::filesystem::file_size(versionFilePath) < 4) {
1077  return -1;
1078  }
1079  FILE* versionFile = open(versionFilePath);
1080  int32_t version;
1081  read(versionFile, 0, sizeof(int32_t), (int8_t*)&version);
1082  close(versionFile);
1083  return version;
1084 }
std::string fileMgrBasePath_
Definition: FileMgr.h:320
size_t read(FILE *f, const size_t offset, const size_t size, int8_t *buf)
Reads the specified number of bytes from the offset position in file f into buf.
Definition: File.cpp:118
version
Definition: setup.py:65
FILE * open(int fileId)
Opens/creates the file with the given id; returns NULL on error.
Definition: File.cpp:88
void close(FILE *f)
Closes the file pointed to by the FILE pointer.
Definition: File.cpp:107
size_t file_size(const int fd)
Definition: omnisci_fs.cpp:31

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void File_Namespace::FileMgr::removeTableRelatedDS ( const int32_t  db_id,
const int32_t  table_id 
)
override

Definition at line 1182 of file FileMgr.cpp.

References UNREACHABLE.

1182  {
1183  UNREACHABLE();
1184 }
#define UNREACHABLE()
Definition: Logger.h:241
Page File_Namespace::FileMgr::requestFreePage ( size_t  pagesize,
const bool  isMetadata 
)

Definition at line 871 of file FileMgr.cpp.

References CHECK, createFile(), File_Namespace::FileInfo::fileId, fileIndex_, files_, File_Namespace::FileInfo::getFreePage(), getPageMutex_, MAX_FILE_N_METADATA_PAGES, and MAX_FILE_N_PAGES.

Referenced by File_Namespace::FileBuffer::addNewMultiPage(), init(), File_Namespace::FileBuffer::write(), and File_Namespace::FileBuffer::writeMetadata().

871  {
872  std::lock_guard<std::mutex> lock(getPageMutex_);
873 
874  auto candidateFiles = fileIndex_.equal_range(pageSize);
875  int32_t pageNum = -1;
876  for (auto fileIt = candidateFiles.first; fileIt != candidateFiles.second; ++fileIt) {
877  FileInfo* fileInfo = files_[fileIt->second];
878  pageNum = fileInfo->getFreePage();
879  if (pageNum != -1) {
880  return (Page(fileInfo->fileId, pageNum));
881  }
882  }
883  // if here then we need to add a file
884  FileInfo* fileInfo;
885  if (isMetadata) {
886  fileInfo = createFile(pageSize, MAX_FILE_N_METADATA_PAGES);
887  } else {
888  fileInfo = createFile(pageSize, MAX_FILE_N_PAGES);
889  }
890  pageNum = fileInfo->getFreePage();
891  CHECK(pageNum != -1);
892  return (Page(fileInfo->fileId, pageNum));
893 }
std::mutex getPageMutex_
pointer to DB level metadata
Definition: FileMgr.h:338
std::vector< FileInfo * > files_
Definition: FileMgr.h:322
#define MAX_FILE_N_METADATA_PAGES
Definition: File.h:27
PageSizeFileMMap fileIndex_
A vector of files accessible via a file identifier.
Definition: FileMgr.h:323
#define MAX_FILE_N_PAGES
Definition: File.h:26
FileInfo * createFile(const size_t pageSize, const size_t numPages)
Adds a file to the file manager repository.
Definition: FileMgr.cpp:959
#define CHECK(condition)
Definition: Logger.h:197

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void File_Namespace::FileMgr::requestFreePages ( size_t  npages,
size_t  pagesize,
std::vector< Page > &  pages,
const bool  isMetadata 
)

Obtains free pages – creates new files if necessary – of the requested size.

Given a page size and number of pages, this method updates the vector "pages" to include free pages of the requested size. These pages are immediately removed from the free list of the affected file(s). If there are not enough pages available among current files, new files are created and their pages are included in the vector.

Parameters
npagesThe number of free pages requested
pagesizeThe size of each requested page
pagesA vector containing the free pages obtained by this method

Definition at line 895 of file FileMgr.cpp.

References CHECK, createFile(), File_Namespace::FileInfo::fileId, fileIndex_, files_, File_Namespace::FileInfo::getFreePage(), getPageMutex_, MAX_FILE_N_METADATA_PAGES, and MAX_FILE_N_PAGES.

898  {
899  // not used currently
900  // @todo add method to FileInfo to get more than one page
901  std::lock_guard<std::mutex> lock(getPageMutex_);
902  auto candidateFiles = fileIndex_.equal_range(pageSize);
903  size_t numPagesNeeded = numPagesRequested;
904  for (auto fileIt = candidateFiles.first; fileIt != candidateFiles.second; ++fileIt) {
905  FileInfo* fileInfo = files_[fileIt->second];
906  int32_t pageNum;
907  do {
908  pageNum = fileInfo->getFreePage();
909  if (pageNum != -1) {
910  pages.emplace_back(fileInfo->fileId, pageNum);
911  numPagesNeeded--;
912  }
913  } while (pageNum != -1 && numPagesNeeded > 0);
914  if (numPagesNeeded == 0) {
915  break;
916  }
917  }
918  while (numPagesNeeded > 0) {
919  FileInfo* fileInfo;
920  if (isMetadata) {
921  fileInfo = createFile(pageSize, MAX_FILE_N_METADATA_PAGES);
922  } else {
923  fileInfo = createFile(pageSize, MAX_FILE_N_PAGES);
924  }
925  int32_t pageNum;
926  do {
927  pageNum = fileInfo->getFreePage();
928  if (pageNum != -1) {
929  pages.emplace_back(fileInfo->fileId, pageNum);
930  numPagesNeeded--;
931  }
932  } while (pageNum != -1 && numPagesNeeded > 0);
933  if (numPagesNeeded == 0) {
934  break;
935  }
936  }
937  CHECK(pages.size() == numPagesRequested);
938 }
std::mutex getPageMutex_
pointer to DB level metadata
Definition: FileMgr.h:338
std::vector< FileInfo * > files_
Definition: FileMgr.h:322
#define MAX_FILE_N_METADATA_PAGES
Definition: File.h:27
PageSizeFileMMap fileIndex_
A vector of files accessible via a file identifier.
Definition: FileMgr.h:323
#define MAX_FILE_N_PAGES
Definition: File.h:26
FileInfo * createFile(const size_t pageSize, const size_t numPages)
Adds a file to the file manager repository.
Definition: FileMgr.cpp:959
#define CHECK(condition)
Definition: Logger.h:197

+ Here is the call graph for this function:

void File_Namespace::FileMgr::rollOffOldData ( const int32_t  epochCeiling,
const bool  shouldCheckpoint 
)
private

Definition at line 652 of file FileMgr.cpp.

References checkpoint(), epoch_, Epoch::floor(), freePagesBeforeEpoch(), and maxRollbackEpochs_.

Referenced by checkpoint(), and init().

652  {
653  if (maxRollbackEpochs_ >= 0) {
654  const int32_t minRollbackEpoch =
655  std::max(epochCeiling - maxRollbackEpochs_, epoch_.floor());
656  if (minRollbackEpoch > epoch_.floor()) {
657  freePagesBeforeEpoch(minRollbackEpoch);
658  epoch_.floor(minRollbackEpoch);
659  if (shouldCheckpoint) {
660  checkpoint();
661  }
662  }
663  }
664 }
int32_t floor() const
Definition: Epoch.h:43
void checkpoint() override
Fsyncs data files, writes out epoch and fsyncs that.
Definition: FileMgr.cpp:666
int32_t maxRollbackEpochs_
Definition: FileMgr.h:319
void freePagesBeforeEpoch(const int32_t minRollbackEpoch)
Definition: FileMgr.cpp:646
Epoch epoch_
the index of the next file id
Definition: FileMgr.h:327

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void File_Namespace::FileMgr::setEpoch ( const int32_t  newEpoch)
private

Definition at line 1165 of file FileMgr.cpp.

References Epoch::ceiling(), epoch_, fileMgrKey_, Epoch::floor(), and writeAndSyncEpochToDisk().

Referenced by init().

1165  {
1166  if (newEpoch < epoch_.floor()) {
1167  std::stringstream error_message;
1168  error_message << "Cannot set epoch for table (" << fileMgrKey_.first << ","
1169  << fileMgrKey_.second << ") lower than the minimum rollback epoch ("
1170  << epoch_.floor() << ").";
1171  throw std::runtime_error(error_message.str());
1172  }
1173  epoch_.ceiling(newEpoch);
1175 }
int32_t floor() const
Definition: Epoch.h:43
int32_t ceiling() const
Definition: Epoch.h:44
std::pair< const int32_t, const int32_t > fileMgrKey_
Global FileMgr.
Definition: FileMgr.h:318
void writeAndSyncEpochToDisk()
Definition: FileMgr.cpp:628
Epoch epoch_
the index of the next file id
Definition: FileMgr.h:327

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void File_Namespace::FileMgr::writeAndSyncEpochToDisk ( )
private

Definition at line 628 of file FileMgr.cpp.

References Epoch::byte_size(), CHECK, epoch_, epochFile_, epochIsCheckpointed_, logger::FATAL, omnisci::fsync(), LOG, Epoch::storage_ptr(), and File_Namespace::write().

Referenced by checkpoint(), createEpochFile(), init(), migrateEpochFileV0(), and setEpoch().

628  {
629  CHECK(epochFile_);
631  int32_t status = fflush(epochFile_);
632  if (status != 0) {
633  LOG(FATAL) << "Could not flush epoch file to disk";
634  }
635 #ifdef __APPLE__
636  status = fcntl(fileno(epochFile_), 51);
637 #else
638  status = omnisci::fsync(fileno(epochFile_));
639 #endif
640  if (status != 0) {
641  LOG(FATAL) << "Could not sync epoch file to disk";
642  }
643  epochIsCheckpointed_ = true;
644 }
int8_t * storage_ptr()
Definition: Epoch.h:61
#define LOG(tag)
Definition: Logger.h:188
int fsync(int fd)
Definition: omnisci_fs.cpp:60
static size_t byte_size()
Definition: Epoch.h:63
size_t write(FILE *f, const size_t offset, const size_t size, int8_t *buf)
Writes the specified number of bytes to the offset position in file f from buf.
Definition: File.cpp:126
#define CHECK(condition)
Definition: Logger.h:197
Epoch epoch_
the index of the next file id
Definition: FileMgr.h:327

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void File_Namespace::FileMgr::writeAndSyncVersionToDisk ( const std::string &  versionFileName,
const int32_t  version 
)
private

Definition at line 1086 of file FileMgr.cpp.

References File_Namespace::close(), File_Namespace::create(), epochFile_, logger::FATAL, fileMgrBasePath_, omnisci::fsync(), logger::INFO, LOG, File_Namespace::open(), readVersionFromDisk(), and File_Namespace::write().

Referenced by createTopLevelMetadata(), init(), migrateEpochFileV0(), and migrateToLatestFileMgrVersion().

1087  {
1088  const std::string versionFilePath(fileMgrBasePath_ + "/" + versionFileName);
1089  FILE* versionFile;
1090  if (boost::filesystem::exists(versionFilePath)) {
1091  int32_t oldVersion = readVersionFromDisk(versionFileName);
1092  LOG(INFO) << "Storage version file `" << versionFilePath
1093  << "` already exists, its current version is " << oldVersion;
1094  versionFile = open(versionFilePath);
1095  } else {
1096  versionFile = create(versionFilePath, sizeof(int32_t));
1097  }
1098  write(versionFile, 0, sizeof(int32_t), (int8_t*)&version);
1099  int32_t status = fflush(versionFile);
1100  if (status != 0) {
1101  LOG(FATAL) << "Could not flush version file " << versionFilePath << " to disk";
1102  }
1103 #ifdef __APPLE__
1104  status = fcntl(fileno(epochFile_), 51);
1105 #else
1106  status = omnisci::fsync(fileno(versionFile));
1107 #endif
1108  if (status != 0) {
1109  LOG(FATAL) << "Could not sync version file " << versionFilePath << " to disk";
1110  }
1111  close(versionFile);
1112 }
int32_t readVersionFromDisk(const std::string &versionFileName) const
Definition: FileMgr.cpp:1068
#define LOG(tag)
Definition: Logger.h:188
std::string fileMgrBasePath_
Definition: FileMgr.h:320
FILE * create(const std::string &basePath, const int fileId, const size_t pageSize, const size_t numPages)
Definition: File.cpp:40
int fsync(int fd)
Definition: omnisci_fs.cpp:60
version
Definition: setup.py:65
FILE * open(int fileId)
Opens/creates the file with the given id; returns NULL on error.
Definition: File.cpp:88
size_t write(FILE *f, const size_t offset, const size_t size, int8_t *buf)
Writes the specified number of bytes to the offset position in file f from buf.
Definition: File.cpp:126
void close(FILE *f)
Closes the file pointed to by the FILE pointer.
Definition: File.cpp:107

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Friends And Related Function Documentation

friend class GlobalFileMgr
friend

Definition at line 115 of file FileMgr.h.

Member Data Documentation

int32_t File_Namespace::FileMgr::db_version_
private

Definition at line 332 of file FileMgr.h.

Referenced by createTopLevelMetadata().

FILE* File_Namespace::FileMgr::DBMetaFile_ = nullptr
private

Definition at line 336 of file FileMgr.h.

Referenced by closeRemovePhysical(), getTotalFileSize(), and ~FileMgr().

size_t File_Namespace::FileMgr::defaultPageSize_
private

number of threads used when loading data

Definition at line 325 of file FileMgr.h.

Referenced by copyPage(), createBufferUnlocked(), getStorageStats(), and putBuffer().

Epoch File_Namespace::FileMgr::epoch_
private

the index of the next file id

Definition at line 327 of file FileMgr.h.

Referenced by FileMgr(), init(), migrateEpochFileV0(), openAndReadEpochFile(), rollOffOldData(), setEpoch(), and writeAndSyncEpochToDisk().

FILE* File_Namespace::FileMgr::epochFile_ = nullptr
private
bool File_Namespace::FileMgr::epochIsCheckpointed_ = true
private

Definition at line 328 of file FileMgr.h.

Referenced by writeAndSyncEpochToDisk().

PageSizeFileMMap File_Namespace::FileMgr::fileIndex_
private

A vector of files accessible via a file identifier.

Definition at line 323 of file FileMgr.h.

Referenced by createFile(), openExistingFile(), requestFreePage(), and requestFreePages().

std::pair<const int32_t, const int32_t> File_Namespace::FileMgr::fileMgrKey_
private

Global FileMgr.

Definition at line 318 of file FileMgr.h.

Referenced by checkpoint(), coreInit(), FileMgr(), and setEpoch().

int32_t File_Namespace::FileMgr::fileMgrVersion_
private

DB version from dbmeta file, should be compatible with GlobalFileMgr::omnisci_db_version_

Definition at line 334 of file FileMgr.h.

Referenced by init(), and migrateToLatestFileMgrVersion().

std::vector<FileInfo*> File_Namespace::FileMgr::files_
private
mapd_shared_mutex File_Namespace::FileMgr::files_rw_mutex_
mutableprivate
std::vector<std::pair<FileInfo*, int32_t> > File_Namespace::FileMgr::free_pages_
private

Definition at line 343 of file FileMgr.h.

Referenced by checkpoint(), free_page(), and init().

std::mutex File_Namespace::FileMgr::getPageMutex_
private

pointer to DB level metadata

Definition at line 338 of file FileMgr.h.

Referenced by requestFreePage(), and requestFreePages().

GlobalFileMgr* File_Namespace::FileMgr::gfm_
private

Definition at line 317 of file FileMgr.h.

Referenced by coreInit(), FileMgr(), getDBConvert(), getDBVersion(), and init().

bool File_Namespace::FileMgr::isFullyInitted_ {false}
private

Definition at line 344 of file FileMgr.h.

Referenced by getStorageStats(), and init().

const int32_t File_Namespace::FileMgr::latestFileMgrVersion_ {1}
private

Definition at line 335 of file FileMgr.h.

Referenced by init(), and migrateToLatestFileMgrVersion().

int32_t File_Namespace::FileMgr::maxRollbackEpochs_
private

Definition at line 319 of file FileMgr.h.

Referenced by rollOffOldData().

mapd_shared_mutex File_Namespace::FileMgr::mutex_free_page_
mutableprivate

Definition at line 342 of file FileMgr.h.

Referenced by checkpoint(), free_page(), and init().

unsigned File_Namespace::FileMgr::nextFileId_
private

Definition at line 326 of file FileMgr.h.

Referenced by createFile(), and init().

size_t File_Namespace::FileMgr::num_reader_threads_
private

Maps page sizes to FileInfo objects.

Definition at line 324 of file FileMgr.h.

Referenced by init().


The documentation for this class was generated from the following files: