File_Namespace::FileMgr Class Reference

#include <FileMgr.h>

Inheritance diagram for File_Namespace::FileMgr:

Collaboration diagram for File_Namespace::FileMgr:

Public Member Functions
	FileMgr (const int32_t deviceId, GlobalFileMgr *gfm, const std::pair< const int32_t, const int32_t > fileMgrKey, const int32_t max_rollback_epochs=-1, const size_t num_reader_threads=0, const int32_t epoch=-1, const size_t defaultPageSize=DEFAULT_PAGE_SIZE)
	Constructor. More...
	FileMgr (const int32_t deviceId, GlobalFileMgr *gfm, const std::pair< const int32_t, const int32_t > fileMgrKey, const size_t defaultPageSize, const bool runCoreInit)
	FileMgr (GlobalFileMgr *gfm, const size_t defaultPageSize, std::string basePath)
	~FileMgr () override
	Destructor. More...
StorageStats	getStorageStats ()
FileBuffer *	createBuffer (const ChunkKey &key, size_t pageSize=0, const size_t numBytes=0) override
	Creates a chunk with the specified key and page size. More...
bool	isBufferOnDevice (const ChunkKey &key) override
void	deleteBuffer (const ChunkKey &key, const bool purge=true) override
	Deletes the chunk with the specified key. More...
void	deleteBuffersWithPrefix (const ChunkKey &keyPrefix, const bool purge=true) override
FileBuffer *	getBuffer (const ChunkKey &key, const size_t numBytes=0) override
	Returns the a pointer to the chunk with the specified key. More...
void	fetchBuffer (const ChunkKey &key, AbstractBuffer *destBuffer, const size_t numBytes) override
FileBuffer *	putBuffer (const ChunkKey &key, AbstractBuffer *d, const size_t numBytes=0) override
	Puts the contents of d into the Chunk with the given key. More...
AbstractBuffer *	alloc (const size_t numBytes) override
void	free (AbstractBuffer *buffer) override
Page	requestFreePage (size_t pagesize, const bool isMetadata)
MgrType	getMgrType () override
std::string	getStringMgrType () override
std::string	printSlabs () override
void	clearSlabs () override
size_t	getMaxSize () override
size_t	getInUseSize () override
size_t	getAllocated () override
bool	isAllocationCapped () override
FileInfo *	getFileInfoForFileId (const int32_t fileId)
uint64_t	getTotalFileSize () const
FileMetadata	getMetadataForFile (const boost::filesystem::directory_iterator &fileIterator)
void	init (const size_t num_reader_threads, const int32_t epochOverride)
void	init (const std::string &dataPathToConvertFrom, const int32_t epochOverride)
bool	coreInit ()
	Determines file path, and if exists, runs file migration and opens and reads epoch file. More...
void	copyPage (Page &srcPage, FileMgr *destFileMgr, Page &destPage, const size_t reservedHeaderSize, const size_t numBytes, const size_t offset)
void	requestFreePages (size_t npages, size_t pagesize, std::vector< Page > &pages, const bool isMetadata)
	Obtains free pages – creates new files if necessary – of the requested size. More...
void	getChunkMetadataVecForKeyPrefix (ChunkMetadataVector &chunkMetadataVec, const ChunkKey &keyPrefix) override
void	checkpoint () override
	Fsyncs data files, writes out epoch and fsyncs that. More...
void	checkpoint (const int32_t db_id, const int32_t tb_id) override
int32_t	epoch ()
	Returns current value of epoch - should be one greater than recorded at last checkpoint. More...
int32_t	epochFloor ()
int32_t	incrementEpoch ()
int32_t	lastCheckpointedEpoch ()
	Returns value of epoch at last checkpoint. More...
int32_t	maxRollbackEpochs ()
	Returns value max_rollback_epochs. More...
size_t	getNumReaderThreads ()
	Returns number of threads defined by parameter num-reader-threads which should be used during initial load and consequent read of data. More...
FILE *	getFileForFileId (const int32_t fileId)
	Returns FILE pointer associated with requested fileId. More...
size_t	getNumChunks () override
size_t	getNumUsedPages () const
size_t	getNumUsedMetadataPages () const
size_t	getNumUsedMetadataPagesForChunkKey (const ChunkKey &chunkKey) const
int32_t	getDBVersion () const
	Index for looking up chunks. More...
bool	getDBConvert () const
void	createTopLevelMetadata ()
std::string	getFileMgrBasePath () const
void	closeRemovePhysical ()
void	removeTableRelatedDS (const int32_t db_id, const int32_t table_id) override
void	free_page (std::pair< FileInfo *, int32_t > &&page)
const std::pair< const int32_t, const int32_t >	get_fileMgrKey () const

Public Attributes
ChunkKeyToChunkMap	chunkIndex_

Protected Member Functions
	FileMgr (const int epoch)

Private Member Functions
FileInfo *	createFile (const size_t pageSize, const size_t numPages)
	Adds a file to the file manager repository. More...
FileInfo *	openExistingFile (const std::string &path, const int32_t fileId, const size_t pageSize, const size_t numPages, std::vector< HeaderInfo > &headerVec)
void	createEpochFile (const std::string &epochFileName)
int32_t	openAndReadLegacyEpochFile (const std::string &epochFileName)
void	openAndReadEpochFile (const std::string &epochFileName)
void	writeAndSyncEpochToDisk ()
void	setEpoch (const int32_t newEpoch)
void	freePagesBeforeEpoch (const int32_t minRollbackEpoch)
void	rollOffOldData (const int32_t epochCeiling, const bool shouldCheckpoint)
int32_t	readVersionFromDisk (const std::string &versionFileName) const
void	writeAndSyncVersionToDisk (const std::string &versionFileName, const int32_t version)
void	processFileFutures (std::vector< std::future< std::vector< HeaderInfo >>> &file_futures, std::vector< HeaderInfo > &headerVec)
FileBuffer *	createBufferUnlocked (const ChunkKey &key, size_t pageSize=0, const size_t numBytes=0)
void	migrateToLatestFileMgrVersion ()
void	migrateEpochFileV0 ()

Private Attributes
GlobalFileMgr *	gfm_
std::pair< const int32_t, const int32_t >	fileMgrKey_
	Global FileMgr. More...
int32_t	maxRollbackEpochs_
std::string	fileMgrBasePath_
std::vector< FileInfo * >	files_
PageSizeFileMMap	fileIndex_
	A vector of files accessible via a file identifier. More...
size_t	num_reader_threads_
	Maps page sizes to FileInfo objects. More...
size_t	defaultPageSize_
	number of threads used when loading data More...
unsigned	nextFileId_
Epoch	epoch_
	the index of the next file id More...
bool	epochIsCheckpointed_ = true
FILE *	epochFile_ = nullptr
int32_t	db_version_
int32_t	fileMgrVersion_
const int32_t	latestFileMgrVersion_ {1}
FILE *	DBMetaFile_ = nullptr
std::mutex	getPageMutex_
	pointer to DB level metadata More...
mapd_shared_mutex	chunkIndexMutex_
mapd_shared_mutex	files_rw_mutex_
mapd_shared_mutex	mutex_free_page_
std::vector< std::pair < FileInfo *, int32_t > >	free_pages_
bool	isFullyInitted_ {false}

Friends
class	GlobalFileMgr

Detailed Description

Definition at line 114 of file FileMgr.h.

Constructor & Destructor Documentation

File_Namespace::FileMgr::FileMgr	(	const int32_t	deviceId,
		GlobalFileMgr *	gfm,
		const std::pair< const int32_t, const int32_t >	fileMgrKey,
		const int32_t	max_rollback_epochs = -1,
		const size_t	num_reader_threads = 0,
		const int32_t	epoch = -1,
		const size_t	defaultPageSize = DEFAULT_PAGE_SIZE
	)

Constructor.

File_Namespace::FileMgr::FileMgr	(	const int32_t	deviceId,
		GlobalFileMgr *	gfm,
		const std::pair< const int32_t, const int32_t >	fileMgrKey,
		const size_t	defaultPageSize,
		const bool	runCoreInit
	)

Definition at line 82 of file FileMgr.cpp.

References coreInit(), epochFile_, fileMgrBasePath_, fileMgrKey_, files_, File_Namespace::GlobalFileMgr::getBasePath(), gfm_, and to_string().

    : AbstractBufferMgr(deviceId)
    , gfm_(gfm)
    , fileMgrKey_(fileMgrKey)
    , maxRollbackEpochs_(-1)
    , defaultPageSize_(defaultPageSize)
    , nextFileId_(0) {
  const std::string fileMgrDirPrefix("table");
  const std::string FileMgrDirDelim("_");
  fileMgrBasePath_ = (gfm_->getBasePath() + fileMgrDirPrefix + FileMgrDirDelim +
                      std::to_string(fileMgrKey_.first) +                     // db_id
                      FileMgrDirDelim + std::to_string(fileMgrKey_.second));  // tb_id
  epochFile_ = nullptr;
  files_.clear();
  if (runCoreInit) {
    coreInit();
  }
}

Here is the call graph for this function:

File_Namespace::FileMgr::FileMgr	(	GlobalFileMgr *	gfm,
		const size_t	defaultPageSize,
		std::string	basePath
	)

Definition at line 105 of file FileMgr.cpp.

References init().

    : AbstractBufferMgr(0)
    , gfm_(gfm)
    , fileMgrKey_(0, 0)
    , maxRollbackEpochs_(-1)
    , fileMgrBasePath_(basePath)
    , defaultPageSize_(defaultPageSize)
    , nextFileId_(0) {
  init(basePath, -1);
}

Here is the call graph for this function:

File_Namespace::FileMgr::~FileMgr ( )

override

Destructor.

Definition at line 121 of file FileMgr.cpp.

References chunkIndex_, File_Namespace::close(), DBMetaFile_, epochFile_, and files_.

                  {
  // free memory used by FileInfo objects
  for (auto chunkIt = chunkIndex_.begin(); chunkIt != chunkIndex_.end(); ++chunkIt) {
    delete chunkIt->second;
  }
  for (auto file_info : files_) {
    delete file_info;
  }

  if (epochFile_) {
    close(epochFile_);
    epochFile_ = nullptr;
  }

  if (DBMetaFile_) {
    close(DBMetaFile_);
    DBMetaFile_ = nullptr;
  }
}

Here is the call graph for this function:

File_Namespace::FileMgr::FileMgr ( const int  epoch )

protected

Definition at line 117 of file FileMgr.cpp.

References Epoch::ceiling(), and epoch_.

                                : AbstractBufferMgr(-1) {
  epoch_.ceiling(epoch);
}

Here is the call graph for this function:

Member Function Documentation

AbstractBuffer * File_Namespace::FileMgr::alloc ( const size_t  numBytes = 0 )

override

Definition at line 862 of file FileMgr.cpp.

References logger::FATAL, and LOG.

                                                        {
  LOG(FATAL) << "Operation not supported";
  return nullptr;  // satisfy return-type warning
}

void File_Namespace::FileMgr::checkpoint ( )

override

Fsyncs data files, writes out epoch and fsyncs that.

Definition at line 666 of file FileMgr.cpp.

References chunkIndex_, chunkIndexMutex_, epoch(), logger::FATAL, fileMgrKey_, files_, files_rw_mutex_, free_page(), free_pages_, incrementEpoch(), lastCheckpointedEpoch(), LOG, mutex_free_page_, rollOffOldData(), VLOG, and writeAndSyncEpochToDisk().

Referenced by rollOffOldData().

                         {
  VLOG(2) << "Checkpointing table (" << fileMgrKey_.first << ", " << fileMgrKey_.second
          << " epoch: " << epoch();
  mapd_unique_lock<mapd_shared_mutex> chunkIndexWriteLock(chunkIndexMutex_);
  for (auto chunkIt = chunkIndex_.begin(); chunkIt != chunkIndex_.end(); ++chunkIt) {
    if (chunkIt->second->isDirty()) {
      chunkIt->second->writeMetadata(epoch());
      chunkIt->second->clearDirtyBits();
    }
  }
  chunkIndexWriteLock.unlock();

  mapd_shared_lock<mapd_shared_mutex> read_lock(files_rw_mutex_);
  for (auto fileIt = files_.begin(); fileIt != files_.end(); ++fileIt) {
    int32_t status = (*fileIt)->syncToDisk();
    if (status != 0) {
      LOG(FATAL) << "Could not sync file to disk";
    }
  }

  writeAndSyncEpochToDisk();
  incrementEpoch();
  rollOffOldData(lastCheckpointedEpoch(), false /* shouldCheckpoint */);

  mapd_unique_lock<mapd_shared_mutex> freePagesWriteLock(mutex_free_page_);
  for (auto& free_page : free_pages_) {
    free_page.first->freePageDeferred(free_page.second);
  }
  free_pages_.clear();
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::checkpoint ( const int32_t  db_id, const int32_t  tb_id  )

inlineoverride

Definition at line 238 of file FileMgr.h.

References logger::FATAL, and LOG.

                                                                     {
    LOG(FATAL) << "Operation not supported, api checkpoint() should be used instead";
  }

void File_Namespace::FileMgr::clearSlabs ( )

inlineoverride

Definition at line 180 of file FileMgr.h.

                                    { /* noop */
  }

void File_Namespace::FileMgr::closeRemovePhysical

(

)

Definition at line 534 of file FileMgr.cpp.

References File_Namespace::close(), DBMetaFile_, epochFile_, files_, getFileMgrBasePath(), and File_Namespace::renameForDelete().

Referenced by File_Namespace::GlobalFileMgr::removeTableRelatedDS().

                                  {
  for (auto file_info : files_) {
    if (file_info->f) {
      close(file_info->f);
      file_info->f = nullptr;
    }
  }

  if (DBMetaFile_) {
    close(DBMetaFile_);
    DBMetaFile_ = nullptr;
  }

  if (epochFile_) {
    close(epochFile_);
    epochFile_ = nullptr;
  }

  /* rename for later deletion the directory containing table related data */
  File_Namespace::renameForDelete(getFileMgrBasePath());
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::copyPage	(	Page &	srcPage,
		FileMgr *	destFileMgr,
		Page &	destPage,
		const size_t	reservedHeaderSize,
		const size_t	numBytes,
		const size_t	offset
	)

Definition at line 556 of file FileMgr.cpp.

References CHECK, checked_malloc(), defaultPageSize_, File_Namespace::Page::fileId, free(), getFileInfoForFileId(), File_Namespace::Page::pageNum, File_Namespace::FileInfo::read(), and File_Namespace::FileInfo::write().

Referenced by init().

                                            {
  CHECK(offset + numBytes <= defaultPageSize_);
  FileInfo* srcFileInfo = getFileInfoForFileId(srcPage.fileId);
  FileInfo* destFileInfo = destFileMgr->getFileInfoForFileId(destPage.fileId);
  int8_t* buffer = reinterpret_cast<int8_t*>(checked_malloc(numBytes));

  size_t bytesRead = srcFileInfo->read(
      srcPage.pageNum * defaultPageSize_ + offset + reservedHeaderSize, numBytes, buffer);
  CHECK(bytesRead == numBytes);
  size_t bytesWritten = destFileInfo->write(
      destPage.pageNum * defaultPageSize_ + offset + reservedHeaderSize,
      numBytes,
      buffer);
  CHECK(bytesWritten == numBytes);
  ::free(buffer);
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool File_Namespace::FileMgr::coreInit

(

)

Determines file path, and if exists, runs file migration and opens and reads epoch file.

Returns

a boolean representing whether the directory path existed

Definition at line 141 of file FileMgr.cpp.

References EPOCH_FILENAME, logger::FATAL, fileMgrBasePath_, fileMgrKey_, files_rw_mutex_, File_Namespace::GlobalFileMgr::getBasePath(), gfm_, LOG, migrateToLatestFileMgrVersion(), openAndReadEpochFile(), and to_string().

Referenced by FileMgr(), and init().

                       {
  mapd_unique_lock<mapd_shared_mutex> write_lock(files_rw_mutex_);
  const std::string fileMgrDirPrefix("table");
  const std::string FileMgrDirDelim("_");
  fileMgrBasePath_ = (gfm_->getBasePath() + fileMgrDirPrefix + FileMgrDirDelim +
                      std::to_string(fileMgrKey_.first) +                     // db_id
                      FileMgrDirDelim + std::to_string(fileMgrKey_.second));  // tb_id
  boost::filesystem::path path(fileMgrBasePath_);
  if (boost::filesystem::exists(path)) {
    if (!boost::filesystem::is_directory(path)) {
      LOG(FATAL) << "Specified path '" << fileMgrBasePath_
                 << "' for table data is not a directory.";
    }
    migrateToLatestFileMgrVersion();
    openAndReadEpochFile(EPOCH_FILENAME);
    return true;
  }
  return false;
}

Here is the call graph for this function:

Here is the caller graph for this function:

FileBuffer * File_Namespace::FileMgr::createBuffer ( const ChunkKey &  key, size_t  pageSize = 0, const size_t  numBytes = 0  )

override

Creates a chunk with the specified key and page size.

Definition at line 697 of file FileMgr.cpp.

References chunkIndexMutex_, and createBufferUnlocked().

                                                         {
  mapd_unique_lock<mapd_shared_mutex> chunkIndexWriteLock(chunkIndexMutex_);
  return createBufferUnlocked(key, pageSize, numBytes);
}

Here is the call graph for this function:

FileBuffer * File_Namespace::FileMgr::createBufferUnlocked ( const ChunkKey &  key, size_t  pageSize = 0, const size_t  numBytes = 0  )

private

Todo:

Make all accesses to chunkIndex_ thread-safe

Definition at line 707 of file FileMgr.cpp.

References chunkIndex_, defaultPageSize_, logger::FATAL, LOG, and show_chunk().

Referenced by createBuffer(), and putBuffer().

                                                                 {
  size_t actualPageSize = pageSize;
  if (actualPageSize == 0) {
    actualPageSize = defaultPageSize_;
  }
  // we will do this lazily and not allocate space for the Chunk (i.e.
  // FileBuffer yet)

  if (chunkIndex_.find(key) != chunkIndex_.end()) {
    LOG(FATAL) << "Chunk already exists for key: " << show_chunk(key);
  }
  chunkIndex_[key] = new FileBuffer(this, actualPageSize, key, numBytes);
  return (chunkIndex_[key]);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::createEpochFile ( const std::string &  epochFileName )

private

Definition at line 578 of file FileMgr.cpp.

References Epoch::byte_size(), File_Namespace::create(), epochFile_, logger::FATAL, fileMgrBasePath_, LOG, and writeAndSyncEpochToDisk().

Referenced by init(), and migrateEpochFileV0().

                                                            {
  std::string epochFilePath(fileMgrBasePath_ + "/" + epochFileName);
  if (boost::filesystem::exists(epochFilePath)) {
    LOG(FATAL) << "Epoch file `" << epochFilePath << "` already exists";
  }
  epochFile_ = create(epochFilePath, sizeof(Epoch::byte_size()));
  // Write out current epoch to file - which if this
  // function is being called should be 0
  writeAndSyncEpochToDisk();
}

Here is the call graph for this function:

Here is the caller graph for this function:

FileInfo * File_Namespace::FileMgr::createFile ( const size_t  pageSize, const size_t  numPages  )

private

Adds a file to the file manager repository.

This method will create a FileInfo object for the file being added, and it will create the corresponding file on physical disk with the indicated number of pages pre-allocated.

A pointer to the FileInfo object is returned, which itself has a file pointer (FILE*) and a file identifier (int32_t fileId).

Parameters

fileName	The name given to the file in physical storage.
pageSize	The logical page size for the pages in the file.
numPages	The number of logical pages to initially allocate for the file.

Returns

FileInfo* A pointer to the FileInfo object of the added file.

Definition at line 959 of file FileMgr.cpp.

References CHECK, File_Namespace::create(), generate_TableFunctionsFactory_init::f, logger::FATAL, fileIndex_, fileMgrBasePath_, files_, files_rw_mutex_, LOG, and nextFileId_.

Referenced by requestFreePage(), and requestFreePages().

                                                                          {
  // check arguments
  if (pageSize == 0 || numPages == 0) {
    LOG(FATAL) << "File creation failed: pageSize and numPages must be greater than 0.";
  }

  // create the new file
  FILE* f = create(fileMgrBasePath_,
                   nextFileId_,
                   pageSize,
                   numPages);  // TM: not sure if I like naming scheme here - should be in
                               // separate namespace?
  CHECK(f);

  // instantiate a new FileInfo for the newly created file
  int32_t fileId = nextFileId_++;
  FileInfo* fInfo =
      new FileInfo(this, fileId, f, pageSize, numPages, true);  // true means init file
  CHECK(fInfo);

  mapd_unique_lock<mapd_shared_mutex> write_lock(files_rw_mutex_);
  // update file manager data structures
  files_.push_back(fInfo);
  fileIndex_.insert(std::pair<size_t, int32_t>(pageSize, fileId));

  CHECK(files_.back() == fInfo);  // postcondition
  return fInfo;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::createTopLevelMetadata

(

)

Definition at line 1052 of file FileMgr.cpp.

References DB_META_FILENAME, db_version_, logger::FATAL, getDBVersion(), INVALID_VERSION, LOG, readVersionFromDisk(), and writeAndSyncVersionToDisk().

                                     {
  db_version_ = readVersionFromDisk(DB_META_FILENAME);

  if (db_version_ > getDBVersion()) {
    LOG(FATAL) << "DB forward compatibility is not supported. Version of OmniSci "
                  "software used is older than the version of DB being read: "
               << db_version_;
  }
  if (db_version_ == INVALID_VERSION || db_version_ < getDBVersion()) {
    // new system, or we are moving forward versions
    // system wide migration would go here if required
    writeAndSyncVersionToDisk(DB_META_FILENAME, getDBVersion());
    return;
  }
}

Here is the call graph for this function:

void File_Namespace::FileMgr::deleteBuffer ( const ChunkKey &  key, const bool  purge = true  )

override

Deletes the chunk with the specified key.

Definition at line 730 of file FileMgr.cpp.

References chunkIndex_, chunkIndexMutex_, logger::FATAL, LOG, and show_chunk().

                                                                {
  mapd_unique_lock<mapd_shared_mutex> chunkIndexWriteLock(chunkIndexMutex_);
  auto chunkIt = chunkIndex_.find(key);
  // ensure the Chunk exists
  if (chunkIt == chunkIndex_.end()) {
    LOG(FATAL) << "Chunk does not exist for key: " << show_chunk(key);
  }
  chunkIndexWriteLock.unlock();
  // chunkIt->second->writeMetadata(-1); // writes -1 as epoch - signifies deleted
  if (purge) {
    chunkIt->second->freePages();
  }
  //@todo need a way to represent delete in non purge case
  delete chunkIt->second;
  chunkIndex_.erase(chunkIt);
}

Here is the call graph for this function:

void File_Namespace::FileMgr::deleteBuffersWithPrefix ( const ChunkKey &  keyPrefix, const bool  purge = true  )

override

Definition at line 747 of file FileMgr.cpp.

References chunkIndex_, and chunkIndexMutex_.

                                                                                 {
  mapd_unique_lock<mapd_shared_mutex> chunkIndexWriteLock(chunkIndexMutex_);
  auto chunkIt = chunkIndex_.lower_bound(keyPrefix);
  if (chunkIt == chunkIndex_.end()) {
    return;  // should we throw?
  }
  while (chunkIt != chunkIndex_.end() &&
         std::search(chunkIt->first.begin(),
                     chunkIt->first.begin() + keyPrefix.size(),
                     keyPrefix.begin(),
                     keyPrefix.end()) != chunkIt->first.begin() + keyPrefix.size()) {
    if (purge) {
      chunkIt->second->freePages();
    }
    //@todo need a way to represent delete in non purge case
    delete chunkIt->second;
    chunkIndex_.erase(chunkIt++);
  }
}

int32_t File_Namespace::FileMgr::epoch ( )

inline

Returns current value of epoch - should be one greater than recorded at last checkpoint.

Definition at line 245 of file FileMgr.h.

Referenced by File_Namespace::FileBuffer::append(), checkpoint(), File_Namespace::FileInfo::freePage(), File_Namespace::FileBuffer::freePagesBeforeEpoch(), init(), openAndReadLegacyEpochFile(), openExistingFile(), File_Namespace::FileBuffer::reserve(), and File_Namespace::FileBuffer::write().

{ return static_cast<int32_t>(epoch_.ceiling()); }

Here is the caller graph for this function:

int32_t File_Namespace::FileMgr::epochFloor ( )

inline

Definition at line 247 of file FileMgr.h.

Referenced by getStorageStats().

{ return static_cast<int32_t>(epoch_.floor()); }

Here is the caller graph for this function:

void File_Namespace::FileMgr::fetchBuffer ( const ChunkKey &  key, AbstractBuffer *  destBuffer, const size_t  numBytes  )

override

Definition at line 775 of file FileMgr.cpp.

References chunkIndex_, chunkIndexMutex_, Data_Namespace::AbstractBuffer::copyTo(), logger::FATAL, Data_Namespace::AbstractBuffer::isDirty(), LOG, show_chunk(), and Data_Namespace::AbstractBuffer::size().

                                                 {
  // reads chunk specified by ChunkKey into AbstractBuffer provided by
  // destBuffer
  if (destBuffer->isDirty()) {
    LOG(FATAL)
        << "Aborting attempt to fetch a chunk marked dirty. Chunk inconsistency for key: "
        << show_chunk(key);
  }
  mapd_shared_lock<mapd_shared_mutex> chunkIndexReadLock(chunkIndexMutex_);
  auto chunkIt = chunkIndex_.find(key);
  if (chunkIt == chunkIndex_.end()) {
    LOG(FATAL) << "Chunk does not exist for key: " << show_chunk(key);
  }
  chunkIndexReadLock.unlock();

  AbstractBuffer* chunk = chunkIt->second;
  // chunk's size is either specified in function call with numBytes or we
  // just look at pageSize * numPages in FileBuffer
  if (numBytes > 0 && numBytes > chunk->size()) {
    LOG(FATAL) << "Chunk retrieved for key `" << show_chunk(key) << "` is smaller ("
               << chunk->size() << ") than number of bytes requested (" << numBytes
               << ")";
  }

  chunk->copyTo(destBuffer, numBytes);
}

Here is the call graph for this function:

void File_Namespace::FileMgr::free ( AbstractBuffer *  buffer )

override

Definition at line 867 of file FileMgr.cpp.

References logger::FATAL, and LOG.

Referenced by copyPage().

                                         {
  LOG(FATAL) << "Operation not supported";
}

Here is the caller graph for this function:

void File_Namespace::FileMgr::free_page

(

std::pair< FileInfo *, int32_t > &&

page

)

Definition at line 1177 of file FileMgr.cpp.

References free_pages_, and mutex_free_page_.

Referenced by checkpoint(), File_Namespace::FileInfo::freePage(), and init().

                                                          {
  std::unique_lock<mapd_shared_mutex> lock(mutex_free_page_);
  free_pages_.push_back(page);
}

Here is the caller graph for this function:

void File_Namespace::FileMgr::freePagesBeforeEpoch ( const int32_t  minRollbackEpoch )

private

Definition at line 646 of file FileMgr.cpp.

References chunkIndex_.

Referenced by rollOffOldData().

                                                                 {
  for (auto chunkIt = chunkIndex_.begin(); chunkIt != chunkIndex_.end(); ++chunkIt) {
    chunkIt->second->freePagesBeforeEpoch(minRollbackEpoch);
  }
}

Here is the caller graph for this function:

const std::pair<const int32_t, const int32_t> File_Namespace::FileMgr::get_fileMgrKey ( ) const

inline

Definition at line 308 of file FileMgr.h.

Referenced by File_Namespace::FileInfo::openExistingFile().

                                                                     {
    return fileMgrKey_;
  }

Here is the caller graph for this function:

size_t File_Namespace::FileMgr::getAllocated ( )

inlineoverride

Definition at line 184 of file FileMgr.h.

{ return 0; }

FileBuffer * File_Namespace::FileMgr::getBuffer ( const ChunkKey &  key, const size_t  numBytes = 0  )

override

Returns the a pointer to the chunk with the specified key.

Definition at line 767 of file FileMgr.cpp.

References CHECK, chunkIndex_, chunkIndexMutex_, and show_chunk().

                                                                         {
  mapd_shared_lock<mapd_shared_mutex> chunkIndexReadLock(chunkIndexMutex_);
  auto chunkIt = chunkIndex_.find(key);
  CHECK(chunkIt != chunkIndex_.end())
      << "Chunk does not exist for key: " << show_chunk(key);
  return chunkIt->second;
}

Here is the call graph for this function:

void File_Namespace::FileMgr::getChunkMetadataVecForKeyPrefix ( ChunkMetadataVector &  chunkMetadataVec, const ChunkKey &  keyPrefix  )

override

Definition at line 993 of file FileMgr.cpp.

References chunkIndex_, and chunkIndexMutex_.

                                                                         {
  mapd_unique_lock<mapd_shared_mutex> chunkIndexWriteLock(
      chunkIndexMutex_);  // is this guarding the right structure?  it look slike we oly
                          // read here for chunk
  auto chunkIt = chunkIndex_.lower_bound(keyPrefix);
  if (chunkIt == chunkIndex_.end()) {
    return;  // throw?
  }
  while (chunkIt != chunkIndex_.end() &&
         std::search(chunkIt->first.begin(),
                     chunkIt->first.begin() + keyPrefix.size(),
                     keyPrefix.begin(),
                     keyPrefix.end()) != chunkIt->first.begin() + keyPrefix.size()) {
    if (chunkIt->second->hasEncoder()) {
      auto chunk_metadata = std::make_shared<ChunkMetadata>();
      chunkIt->second->encoder_->getMetadata(chunk_metadata);
      chunkMetadataVec.emplace_back(chunkIt->first, chunk_metadata);
    }
    chunkIt++;
  }
}

bool File_Namespace::FileMgr::getDBConvert

(

)

const

Definition at line 1048 of file FileMgr.cpp.

References File_Namespace::GlobalFileMgr::getDBConvert(), and gfm_.

                                 {
  return gfm_->getDBConvert();
}

Here is the call graph for this function:

int32_t File_Namespace::FileMgr::getDBVersion

(

)

const

Index for looking up chunks.

Definition at line 1044 of file FileMgr.cpp.

References File_Namespace::GlobalFileMgr::getDBVersion(), and gfm_.

Referenced by createTopLevelMetadata().

                                    {
  return gfm_->getDBVersion();
}

Here is the call graph for this function:

Here is the caller graph for this function:

FILE * File_Namespace::FileMgr::getFileForFileId

(

const int32_t

fileId

)

Returns FILE pointer associated with requested fileId.

See Also

FileBuffer

Definition at line 988 of file FileMgr.cpp.

References CHECK, and files_.

Referenced by File_Namespace::FileBuffer::readMetadata(), and File_Namespace::FileBuffer::writeMetadata().

                                                    {
  CHECK(fileId >= 0 && static_cast<size_t>(fileId) < files_.size());
  return files_[fileId]->f;
}

Here is the caller graph for this function:

FileInfo* File_Namespace::FileMgr::getFileInfoForFileId ( const int32_t  fileId )

inline

Definition at line 187 of file FileMgr.h.

Referenced by File_Namespace::FileBuffer::append(), File_Namespace::FileBuffer::copyPage(), copyPage(), File_Namespace::FileBuffer::freePage(), File_Namespace::readForThread(), File_Namespace::FileBuffer::write(), and File_Namespace::FileBuffer::writeHeader().

{ return files_[fileId]; }

Here is the caller graph for this function:

std::string File_Namespace::FileMgr::getFileMgrBasePath ( ) const

inline

Definition at line 302 of file FileMgr.h.

Referenced by closeRemovePhysical().

{ return fileMgrBasePath_; }

Here is the caller graph for this function:

size_t File_Namespace::FileMgr::getInUseSize ( )

inlineoverride

Definition at line 183 of file FileMgr.h.

{ return 0; }

size_t File_Namespace::FileMgr::getMaxSize ( )

inlineoverride

Definition at line 182 of file FileMgr.h.

{ return 0; }

FileMetadata File_Namespace::FileMgr::getMetadataForFile

(

const boost::filesystem::directory_iterator &

fileIterator

)

Definition at line 161 of file FileMgr.cpp.

References CHECK_EQ, logger::FATAL, File_Namespace::FileMetadata::file_id, File_Namespace::FileMetadata::file_path, omnisci::file_size(), File_Namespace::FileMetadata::file_size, File_Namespace::FileMetadata::is_data_file, LOG, MAPD_FILE_EXT, File_Namespace::FileMetadata::num_pages, and File_Namespace::FileMetadata::page_size.

Referenced by getStorageStats(), and init().

                                                           {
  FileMetadata fileMetadata;
  fileMetadata.is_data_file = false;
  fileMetadata.file_path = fileIterator->path().string();
  if (!boost::filesystem::is_regular_file(fileIterator->status())) {
    return fileMetadata;
  }
  // note that boost::filesystem leaves preceding dot on
  // extension - hence MAPD_FILE_EXT is ".mapd"
  std::string extension(fileIterator->path().extension().string());
  if (extension == MAPD_FILE_EXT) {
    std::string fileStem(fileIterator->path().stem().string());
    // remove trailing dot if any
    if (fileStem.size() > 0 && fileStem.back() == '.') {
      fileStem = fileStem.substr(0, fileStem.size() - 1);
    }
    size_t dotPos = fileStem.find_last_of(".");  // should only be one
    if (dotPos == std::string::npos) {
      LOG(FATAL) << "File `" << fileIterator->path()
                 << "` does not carry page size information in the filename.";
    }
    fileMetadata.is_data_file = true;
    fileMetadata.file_id = boost::lexical_cast<int>(fileStem.substr(0, dotPos));
    fileMetadata.page_size =
        boost::lexical_cast<size_t>(fileStem.substr(dotPos + 1, fileStem.size()));

    fileMetadata.file_size = boost::filesystem::file_size(fileMetadata.file_path);
    CHECK_EQ(fileMetadata.file_size % fileMetadata.page_size,
             size_t(0));  // should be no partial pages
    fileMetadata.num_pages = fileMetadata.file_size / fileMetadata.page_size;
  }
  return fileMetadata;
}

Here is the call graph for this function:

Here is the caller graph for this function:

MgrType File_Namespace::FileMgr::getMgrType ( )

inlineoverride

Definition at line 177 of file FileMgr.h.

{ return FILE_MGR; };

size_t File_Namespace::FileMgr::getNumChunks ( )

inlineoverride

Definition at line 289 of file FileMgr.h.

                                        {
    mapd_shared_lock<mapd_shared_mutex> read_lock(chunkIndexMutex_);
    return chunkIndex_.size();
  }

size_t File_Namespace::FileMgr::getNumReaderThreads ( )

inline

Returns number of threads defined by parameter num-reader-threads which should be used during initial load and consequent read of data.

Definition at line 278 of file FileMgr.h.

Referenced by File_Namespace::FileBuffer::read().

{ return num_reader_threads_; }

Here is the caller graph for this function:

size_t File_Namespace::FileMgr::getNumUsedMetadataPages

(

)

const

Definition at line 1025 of file FileMgr.cpp.

References chunkIndex_, and chunkIndexMutex_.

                                              {
  size_t num_used_metadata_pages = 0;
  mapd_shared_lock<mapd_shared_mutex> read_lock(chunkIndexMutex_);
  for (const auto& chunkIt : chunkIndex_) {
    num_used_metadata_pages += chunkIt.second->numMetadataPages();
  }
  return num_used_metadata_pages;
}

size_t File_Namespace::FileMgr::getNumUsedMetadataPagesForChunkKey

(

const ChunkKey &

chunkKey

)

const

Definition at line 1034 of file FileMgr.cpp.

References chunkIndex_, and chunkIndexMutex_.

                                                                                 {
  mapd_shared_lock<mapd_shared_mutex> read_lock(chunkIndexMutex_);
  const auto& chunkIt = chunkIndex_.find(chunkKey);
  if (chunkIt != chunkIndex_.end()) {
    return chunkIt->second->numMetadataPages();
  } else {
    throw std::runtime_error("Chunk was not found.");
  }
}

size_t File_Namespace::FileMgr::getNumUsedPages

(

)

const

Definition at line 1016 of file FileMgr.cpp.

References files_, and files_rw_mutex_.

                                      {
  size_t num_used_pages = 0;
  mapd_shared_lock<mapd_shared_mutex> read_lock(files_rw_mutex_);
  for (const auto file : files_) {
    num_used_pages += (file->numPages - file->freePages.size());
  }
  return num_used_pages;
}

StorageStats File_Namespace::FileMgr::getStorageStats

(

)

Definition at line 319 of file FileMgr.cpp.

References CHECK, File_Namespace::StorageStats::data_file_count, defaultPageSize_, File_Namespace::StorageStats::epoch, File_Namespace::StorageStats::epoch_floor, epochFloor(), logger::FATAL, File_Namespace::FileMetadata::file_path, File_Namespace::FileMetadata::file_size, fileMgrBasePath_, files_, files_rw_mutex_, getMetadataForFile(), File_Namespace::FileMetadata::is_data_file, isFullyInitted_, lastCheckpointedEpoch(), LOG, File_Namespace::StorageStats::metadata_file_count, METADATA_PAGE_SIZE, File_Namespace::FileMetadata::num_pages, File_Namespace::FileMetadata::page_size, File_Namespace::StorageStats::total_data_file_size, File_Namespace::StorageStats::total_data_page_count, File_Namespace::StorageStats::total_free_data_page_count, File_Namespace::StorageStats::total_free_metadata_page_count, File_Namespace::StorageStats::total_metadata_file_size, File_Namespace::StorageStats::total_metadata_page_count, and UNREACHABLE.

                                      {
  StorageStats storage_stats;
  mapd_shared_lock<mapd_shared_mutex> read_lock(files_rw_mutex_);
  if (!isFullyInitted_) {
    CHECK(!fileMgrBasePath_.empty());
    boost::filesystem::path path(fileMgrBasePath_);
    if (boost::filesystem::exists(path)) {
      if (!boost::filesystem::is_directory(path)) {
        LOG(FATAL) << "getStorageStats: Specified path '" << fileMgrBasePath_
                   << "' for table data is not a directory.";
      }

      storage_stats.epoch = lastCheckpointedEpoch();
      storage_stats.epoch_floor = epochFloor();
      boost::filesystem::directory_iterator
          endItr;  // default construction yields past-the-end
      for (boost::filesystem::directory_iterator fileIt(path); fileIt != endItr;
           ++fileIt) {
        FileMetadata file_metadata = getMetadataForFile(fileIt);
        if (file_metadata.is_data_file) {
          if (file_metadata.page_size == METADATA_PAGE_SIZE) {
            storage_stats.metadata_file_count++;
            storage_stats.total_metadata_file_size += file_metadata.file_size;
            storage_stats.total_metadata_page_count += file_metadata.num_pages;
          } else if (file_metadata.page_size == defaultPageSize_) {
            storage_stats.data_file_count++;
            storage_stats.total_data_file_size += file_metadata.file_size;
            storage_stats.total_data_page_count += file_metadata.num_pages;
          } else {
            UNREACHABLE() << "Found file with unexpected page size. Page size: "
                          << file_metadata.page_size
                          << ", file path: " << file_metadata.file_path;
          }
        }
      }
    }
  } else {
    storage_stats.epoch = lastCheckpointedEpoch();
    storage_stats.epoch_floor = epochFloor();

    // We already initialized this table so take the faster path of walking through the
    // FileInfo objects and getting metadata from there
    for (const auto& file_info : files_) {
      if (file_info->pageSize == METADATA_PAGE_SIZE) {
        storage_stats.metadata_file_count++;
        storage_stats.total_metadata_file_size +=
            file_info->pageSize * file_info->numPages;
        storage_stats.total_metadata_page_count += file_info->numPages;
        if (storage_stats.total_free_metadata_page_count) {
          storage_stats.total_free_metadata_page_count.value() +=
              file_info->freePages.size();
        } else {
          storage_stats.total_free_metadata_page_count = file_info->freePages.size();
        }
      } else if (file_info->pageSize == defaultPageSize_) {
        storage_stats.data_file_count++;
        storage_stats.total_data_file_size += file_info->pageSize * file_info->numPages;
        storage_stats.total_data_page_count += file_info->numPages;
        if (storage_stats.total_free_data_page_count) {
          storage_stats.total_free_data_page_count.value() += file_info->freePages.size();
        } else {
          storage_stats.total_free_data_page_count = file_info->freePages.size();
        }
      } else {
        UNREACHABLE() << "Found file with unexpected page size. Page size: "
                      << file_info->pageSize;
      }
    }
  }
  return storage_stats;
}

Here is the call graph for this function:

std::string File_Namespace::FileMgr::getStringMgrType ( )

inlineoverride

Definition at line 178 of file FileMgr.h.

{ return ToString(FILE_MGR); }

uint64_t File_Namespace::FileMgr::getTotalFileSize

(

)

const

Definition at line 1186 of file FileMgr.cpp.

References DBMetaFile_, epochFile_, files_, and File_Namespace::fileSize().

Referenced by ShowDiskCacheUsageCommand::execute().

                                         {
  uint64_t total_size = 0;
  for (const auto& file : files_) {
    total_size += file->size();
  }
  if (epochFile_) {
    total_size += fileSize(epochFile_);
  }
  if (DBMetaFile_) {
    total_size += fileSize(DBMetaFile_);
  }
  return total_size;
}

Here is the call graph for this function:

Here is the caller graph for this function:

int32_t File_Namespace::FileMgr::incrementEpoch ( )

inline

Definition at line 249 of file FileMgr.h.

References logger::FATAL, LOG, and Epoch::max_allowable_epoch().

Referenced by checkpoint(), and init().

                                  {
    int32_t newEpoch = epoch_.increment();
    epochIsCheckpointed_ = false;
    // We test for error here instead of in Epoch::increment so we can log FileMgr
    // metadata
    if (newEpoch > Epoch::max_allowable_epoch()) {
      LOG(FATAL) << "Epoch for table (" << fileMgrKey_.first << ", " << fileMgrKey_.second
                 << ") greater than maximum allowed value of "
                 << Epoch::max_allowable_epoch() << ".";
    }
    return newEpoch;
  }

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::init	(	const size_t	num_reader_threads,
		const int32_t	epochOverride
	)

Definition at line 196 of file FileMgr.cpp.

References Epoch::ceiling(), chunkIndex_, coreInit(), createEpochFile(), epoch(), epoch_, EPOCH_FILENAME, logger::FATAL, File_Namespace::FileMetadata::file_id, FILE_MGR_VERSION_FILENAME, File_Namespace::FileMetadata::file_path, fileMgrBasePath_, fileMgrVersion_, Epoch::floor(), free_page(), free_pages_, getMetadataForFile(), File_Namespace::headerCompare(), incrementEpoch(), logger::INFO, File_Namespace::FileMetadata::is_data_file, isFullyInitted_, latestFileMgrVersion_, LOG, mutex_free_page_, nextFileId_, File_Namespace::FileMetadata::num_pages, num_reader_threads_, openExistingFile(), File_Namespace::FileMetadata::page_size, processFileFutures(), rollOffOldData(), setEpoch(), gpu_enabled::sort(), timer_start(), timer_stop(), VLOG, writeAndSyncEpochToDisk(), and writeAndSyncVersionToDisk().

Referenced by FileMgr().

                                                                               {
  // if epochCeiling = -1 this means open from epoch file

  const bool dataExists = coreInit();
  if (dataExists) {
    if (epochOverride != -1) {  // if opening at specified epoch
      setEpoch(epochOverride);
    }
    auto clock_begin = timer_start();

    boost::filesystem::directory_iterator
        endItr;  // default construction yields past-the-end
    int32_t maxFileId = -1;
    int32_t fileCount = 0;
    int32_t threadCount = std::thread::hardware_concurrency();
    std::vector<HeaderInfo> headerVec;
    std::vector<std::future<std::vector<HeaderInfo>>> file_futures;
    boost::filesystem::path path(fileMgrBasePath_);
    for (boost::filesystem::directory_iterator fileIt(path); fileIt != endItr; ++fileIt) {
      FileMetadata fileMetadata = getMetadataForFile(fileIt);
      if (fileMetadata.is_data_file) {
        maxFileId = std::max(maxFileId, fileMetadata.file_id);
        file_futures.emplace_back(std::async(std::launch::async, [fileMetadata, this] {
          std::vector<HeaderInfo> tempHeaderVec;
          openExistingFile(fileMetadata.file_path,
                           fileMetadata.file_id,
                           fileMetadata.page_size,
                           fileMetadata.num_pages,
                           tempHeaderVec);
          return tempHeaderVec;
        }));
        fileCount++;
        if (fileCount % threadCount == 0) {
          processFileFutures(file_futures, headerVec);
        }
      }
    }

    if (file_futures.size() > 0) {
      processFileFutures(file_futures, headerVec);
    }
    int64_t queue_time_ms = timer_stop(clock_begin);

    LOG(INFO) << "Completed Reading table's file metadata, Elapsed time : "
              << queue_time_ms << "ms Epoch: " << epoch_.ceiling()
              << " files read: " << fileCount << " table location: '" << fileMgrBasePath_
              << "'";

    /* Sort headerVec so that all HeaderInfos
     * from a chunk will be grouped together
     * and in order of increasing PageId
     * - Version Epoch */

    std::sort(headerVec.begin(), headerVec.end(), headerCompare);

    /* Goal of next section is to find sequences in the
     * sorted headerVec of the same ChunkId, which we
     * can then initiate a FileBuffer with */

    VLOG(4) << "Number of Headers in Vector: " << headerVec.size();
    if (headerVec.size() > 0) {
      ChunkKey lastChunkKey = headerVec.begin()->chunkKey;
      auto startIt = headerVec.begin();

      for (auto headerIt = headerVec.begin() + 1; headerIt != headerVec.end();
           ++headerIt) {
        if (headerIt->chunkKey != lastChunkKey) {
          chunkIndex_[lastChunkKey] =
              new FileBuffer(this, /*pageSize,*/ lastChunkKey, startIt, headerIt);
          lastChunkKey = headerIt->chunkKey;
          startIt = headerIt;
        }
      }
      // now need to insert last Chunk
      chunkIndex_[lastChunkKey] =
          new FileBuffer(this, /*pageSize,*/ lastChunkKey, startIt, headerVec.end());
    }
    nextFileId_ = maxFileId + 1;
    rollOffOldData(epoch(),
                   true /* shouldCheckpoint - only happens if data is rolled off */);
    incrementEpoch();
    mapd_unique_lock<mapd_shared_mutex> freePagesWriteLock(mutex_free_page_);
    for (auto& free_page : free_pages_) {
      free_page.first->freePageDeferred(free_page.second);
    }
    free_pages_.clear();
  } else {
    boost::filesystem::path path(fileMgrBasePath_);
    if (!boost::filesystem::create_directory(path)) {
      LOG(FATAL) << "Could not create data directory: " << path;
    }
    fileMgrVersion_ = latestFileMgrVersion_;
    if (epochOverride != -1) {
      epoch_.floor(epochOverride);
      epoch_.ceiling(epochOverride);
    } else {
      // These are default constructor values for epoch_, but resetting here for clarity
      epoch_.floor(0);
      epoch_.ceiling(0);
    }

    createEpochFile(EPOCH_FILENAME);
    writeAndSyncEpochToDisk();
    writeAndSyncVersionToDisk(FILE_MGR_VERSION_FILENAME, fileMgrVersion_);
    incrementEpoch();
  }

  /* define number of reader threads to be used */
  size_t num_hardware_based_threads =
      std::thread::hardware_concurrency();  // # of threads is based on # of cores on the
                                            // host
  if (num_reader_threads == 0) {            // # of threads has not been defined by user
    num_reader_threads_ = num_hardware_based_threads;
  } else {
    if (num_reader_threads > num_hardware_based_threads) {
      num_reader_threads_ = num_hardware_based_threads;
    } else {
      num_reader_threads_ = num_reader_threads;
    }
  }
  isFullyInitted_ = true;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::init	(	const std::string &	dataPathToConvertFrom,
		const int32_t	epochOverride
	)

Definition at line 405 of file FileMgr.cpp.

References CHECK, chunkIndex_, copyPage(), EPOCH_FILENAME, logger::FATAL, File_Namespace::FileMetadata::file_id, File_Namespace::FileMetadata::file_path, File_Namespace::GlobalFileMgr::getFileMgr(), getMetadataForFile(), File_Namespace::FileBuffer::getMultiPage(), gfm_, File_Namespace::headerCompare(), File_Namespace::FileMetadata::is_data_file, isFullyInitted_, LOG, File_Namespace::FileBuffer::multiPages_, nextFileId_, File_Namespace::FileMetadata::num_pages, openAndReadEpochFile(), openExistingFile(), File_Namespace::FileMetadata::page_size, File_Namespace::FileBuffer::pageDataSize(), File_Namespace::FileBuffer::pageSize(), processFileFutures(), File_Namespace::MultiPage::push(), requestFreePage(), File_Namespace::FileBuffer::reservedHeaderSize(), Data_Namespace::AbstractBuffer::setDirty(), setEpoch(), Data_Namespace::AbstractBuffer::setSize(), Data_Namespace::AbstractBuffer::size(), gpu_enabled::sort(), Data_Namespace::AbstractBuffer::syncEncoder(), and File_Namespace::FileBuffer::writeHeader().

                                                {
  int32_t converted_data_epoch = 0;
  boost::filesystem::path path(dataPathToConvertFrom);
  if (boost::filesystem::exists(path)) {
    if (!boost::filesystem::is_directory(path)) {
      LOG(FATAL) << "Specified path `" << path << "` is not a directory.";
    }
    openAndReadEpochFile(EPOCH_FILENAME);

    if (epochOverride != -1) {  // if opening at previous epoch
      setEpoch(epochOverride);
    }

    boost::filesystem::directory_iterator
        endItr;  // default construction yields past-the-end
    int32_t maxFileId = -1;
    int32_t fileCount = 0;
    int32_t threadCount = std::thread::hardware_concurrency();
    std::vector<HeaderInfo> headerVec;
    std::vector<std::future<std::vector<HeaderInfo>>> file_futures;
    for (boost::filesystem::directory_iterator fileIt(path); fileIt != endItr; ++fileIt) {
      FileMetadata fileMetadata = getMetadataForFile(fileIt);
      if (fileMetadata.is_data_file) {
        maxFileId = std::max(maxFileId, fileMetadata.file_id);
        file_futures.emplace_back(std::async(std::launch::async, [fileMetadata, this] {
          std::vector<HeaderInfo> tempHeaderVec;
          openExistingFile(fileMetadata.file_path,
                           fileMetadata.file_id,
                           fileMetadata.page_size,
                           fileMetadata.num_pages,
                           tempHeaderVec);
          return tempHeaderVec;
        }));
        fileCount++;
        if (fileCount % threadCount) {
          processFileFutures(file_futures, headerVec);
        }
      }
    }

    if (file_futures.size() > 0) {
      processFileFutures(file_futures, headerVec);
    }

    /* Sort headerVec so that all HeaderInfos
     * from a chunk will be grouped together
     * and in order of increasing PageId
     * - Version Epoch */

    std::sort(headerVec.begin(), headerVec.end(), headerCompare);

    /* Goal of next section is to find sequences in the
     * sorted headerVec of the same ChunkId, which we
     * can then initiate a FileBuffer with */

    if (headerVec.size() > 0) {
      ChunkKey lastChunkKey = headerVec.begin()->chunkKey;
      auto startIt = headerVec.begin();

      for (auto headerIt = headerVec.begin() + 1; headerIt != headerVec.end();
           ++headerIt) {
        if (headerIt->chunkKey != lastChunkKey) {
          FileMgr* c_fm_ =
              dynamic_cast<File_Namespace::FileMgr*>(gfm_->getFileMgr(lastChunkKey));
          CHECK(c_fm_);
          FileBuffer* srcBuf = new FileBuffer(this, lastChunkKey, startIt, headerIt);
          chunkIndex_[lastChunkKey] = srcBuf;
          FileBuffer* destBuf = new FileBuffer(c_fm_, srcBuf->pageSize(), lastChunkKey);
          c_fm_->chunkIndex_[lastChunkKey] = destBuf;
          destBuf->syncEncoder(srcBuf);
          destBuf->setSize(srcBuf->size());
          destBuf->setDirty();  // this needs to be set to force writing out metadata
                                // files from "checkpoint()" call

          size_t totalNumPages = srcBuf->getMultiPage().size();
          for (size_t pageNum = 0; pageNum < totalNumPages; pageNum++) {
            Page srcPage = srcBuf->getMultiPage()[pageNum].current().page;
            Page destPage = c_fm_->requestFreePage(
                srcBuf->pageSize(),
                false);  // may modify and use api "FileBuffer::addNewMultiPage" instead
            MultiPage multiPage(srcBuf->pageSize());
            multiPage.push(destPage, converted_data_epoch);
            destBuf->multiPages_.push_back(multiPage);
            size_t reservedHeaderSize = srcBuf->reservedHeaderSize();
            copyPage(
                srcPage, c_fm_, destPage, reservedHeaderSize, srcBuf->pageDataSize(), 0);
            destBuf->writeHeader(destPage, pageNum, converted_data_epoch, false);
          }
          lastChunkKey = headerIt->chunkKey;
          startIt = headerIt;
        }
      }

      // now need to insert last Chunk
      FileMgr* c_fm_ =
          dynamic_cast<File_Namespace::FileMgr*>(gfm_->getFileMgr(lastChunkKey));
      FileBuffer* srcBuf = new FileBuffer(this, lastChunkKey, startIt, headerVec.end());
      chunkIndex_[lastChunkKey] = srcBuf;
      FileBuffer* destBuf = new FileBuffer(c_fm_, srcBuf->pageSize(), lastChunkKey);
      c_fm_->chunkIndex_[lastChunkKey] = destBuf;
      destBuf->syncEncoder(srcBuf);
      destBuf->setSize(srcBuf->size());
      destBuf->setDirty();  // this needs to be set to write out metadata file from the
                            // "checkpoint()" call

      size_t totalNumPages = srcBuf->getMultiPage().size();
      for (size_t pageNum = 0; pageNum < totalNumPages; pageNum++) {
        Page srcPage = srcBuf->getMultiPage()[pageNum].current().page;
        Page destPage = c_fm_->requestFreePage(
            srcBuf->pageSize(),
            false);  // may modify and use api "FileBuffer::addNewMultiPage" instead
        MultiPage multiPage(srcBuf->pageSize());
        multiPage.push(destPage, converted_data_epoch);
        destBuf->multiPages_.push_back(multiPage);
        size_t reservedHeaderSize = srcBuf->reservedHeaderSize();
        copyPage(srcPage, c_fm_, destPage, reservedHeaderSize, srcBuf->pageDataSize(), 0);
        destBuf->writeHeader(destPage, pageNum, converted_data_epoch, false);
      }
    }
    nextFileId_ = maxFileId + 1;
  } else {
    if (!boost::filesystem::create_directory(path)) {
      LOG(FATAL) << "Specified path does not exist: " << path;
    }
  }
  isFullyInitted_ = true;
}

Here is the call graph for this function:

bool File_Namespace::FileMgr::isAllocationCapped ( )

inlineoverride

Definition at line 185 of file FileMgr.h.

{ return false; }

bool File_Namespace::FileMgr::isBufferOnDevice ( const ChunkKey &  key )

override

Definition at line 725 of file FileMgr.cpp.

References chunkIndex_, and chunkIndexMutex_.

                                                  {
  mapd_shared_lock<mapd_shared_mutex> chunkIndexReadLock(chunkIndexMutex_);
  return chunkIndex_.find(key) != chunkIndex_.end();
}

int32_t File_Namespace::FileMgr::lastCheckpointedEpoch ( )

inline

Returns value of epoch at last checkpoint.

Definition at line 265 of file FileMgr.h.

Referenced by checkpoint(), File_Namespace::GlobalFileMgr::existsDiffBetweenFileMgrParamsAndFileMgr(), and getStorageStats().

                                         {
    return epoch() - (epochIsCheckpointed_ ? 0 : 1);
  }

Here is the caller graph for this function:

int32_t File_Namespace::FileMgr::maxRollbackEpochs ( )

inline

Returns value max_rollback_epochs.

Definition at line 272 of file FileMgr.h.

Referenced by File_Namespace::GlobalFileMgr::existsDiffBetweenFileMgrParamsAndFileMgr().

{ return maxRollbackEpochs_; }

Here is the caller graph for this function:

void File_Namespace::FileMgr::migrateEpochFileV0 ( )

private

Definition at line 1114 of file FileMgr.cpp.

References Epoch::ceiling(), createEpochFile(), epoch_, EPOCH_FILENAME, FILE_MGR_VERSION_FILENAME, fileMgrBasePath_, Epoch::floor(), logger::INFO, LEGACY_EPOCH_FILENAME, LOG, Epoch::min_allowable_epoch(), openAndReadLegacyEpochFile(), writeAndSyncEpochToDisk(), and writeAndSyncVersionToDisk().

Referenced by migrateToLatestFileMgrVersion().

                                 {
  const std::string versionFilePath(fileMgrBasePath_ + "/" + FILE_MGR_VERSION_FILENAME);
  LOG(INFO) << "Migrating file format version from 0 to 1 for  `" << versionFilePath;
  epoch_.floor(Epoch::min_allowable_epoch());
  epoch_.ceiling(openAndReadLegacyEpochFile(LEGACY_EPOCH_FILENAME));
  createEpochFile(EPOCH_FILENAME);
  writeAndSyncEpochToDisk();
  int32_t migrationCompleteVersion = 1;
  writeAndSyncVersionToDisk(FILE_MGR_VERSION_FILENAME, migrationCompleteVersion);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::migrateToLatestFileMgrVersion ( )

private

Definition at line 1125 of file FileMgr.cpp.

References logger::FATAL, FILE_MGR_VERSION_FILENAME, fileMgrVersion_, INVALID_VERSION, latestFileMgrVersion_, LOG, migrateEpochFileV0(), readVersionFromDisk(), UNREACHABLE, and writeAndSyncVersionToDisk().

Referenced by coreInit().

                                            {
  fileMgrVersion_ = readVersionFromDisk(FILE_MGR_VERSION_FILENAME);
  if (fileMgrVersion_ == INVALID_VERSION) {
    fileMgrVersion_ = 0;
    writeAndSyncVersionToDisk(FILE_MGR_VERSION_FILENAME, fileMgrVersion_);
  } else if (fileMgrVersion_ > latestFileMgrVersion_) {
    LOG(FATAL)
        << "Table storage forward compatibility is not supported. Version of OmniSci "
           "software used is older than the version of table being read: "
        << fileMgrVersion_;
  }

  if (fileMgrVersion_ < latestFileMgrVersion_) {
    while (fileMgrVersion_ < latestFileMgrVersion_) {
      switch (fileMgrVersion_) {
        case 0: {
          migrateEpochFileV0();
          break;
        }
        default: {
          UNREACHABLE();
        }
      }
      fileMgrVersion_++;
    }
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::openAndReadEpochFile ( const std::string &  epochFileName )

private

Definition at line 609 of file FileMgr.cpp.

References Epoch::byte_size(), epoch_, epochFile_, logger::FATAL, omnisci::file_size(), fileMgrBasePath_, LOG, File_Namespace::open(), File_Namespace::read(), and Epoch::storage_ptr().

Referenced by coreInit(), and init().

                                                                 {
  if (!epochFile_) {  // Check to see if already open
    std::string epochFilePath(fileMgrBasePath_ + "/" + epochFileName);
    if (!boost::filesystem::exists(epochFilePath)) {
      LOG(FATAL) << "Epoch file `" << epochFilePath << "` does not exist";
    }
    if (!boost::filesystem::is_regular_file(epochFilePath)) {
      LOG(FATAL) << "Epoch file `" << epochFilePath << "` is not a regular file";
    }
    if (boost::filesystem::file_size(epochFilePath) != 16) {
      LOG(FATAL) << "Epoch file `" << epochFilePath
                 << "` is not sized properly (current size: "
                 << boost::filesystem::file_size(epochFilePath) << ", expected size: 16)";
    }
    epochFile_ = open(epochFilePath);
  }
  read(epochFile_, 0, Epoch::byte_size(), epoch_.storage_ptr());
}

Here is the call graph for this function:

Here is the caller graph for this function:

int32_t File_Namespace::FileMgr::openAndReadLegacyEpochFile ( const std::string &  epochFileName )

private

Definition at line 589 of file FileMgr.cpp.

References epoch(), logger::FATAL, omnisci::file_size(), fileMgrBasePath_, LOG, File_Namespace::open(), and File_Namespace::read().

Referenced by migrateEpochFileV0().

                                                                          {
  std::string epochFilePath(fileMgrBasePath_ + "/" + epochFileName);
  if (!boost::filesystem::exists(epochFilePath)) {
    return 0;
  }

  if (!boost::filesystem::is_regular_file(epochFilePath)) {
    LOG(FATAL) << "Epoch file `" << epochFilePath << "` is not a regular file";
  }
  if (boost::filesystem::file_size(epochFilePath) < 4) {
    LOG(FATAL) << "Epoch file `" << epochFilePath
               << "` is not sized properly (current size: "
               << boost::filesystem::file_size(epochFilePath) << ", expected size: 4)";
  }
  FILE* legacyEpochFile = open(epochFilePath);
  int32_t epoch;
  read(legacyEpochFile, 0, sizeof(int32_t), (int8_t*)&epoch);
  return epoch;
}

Here is the call graph for this function:

Here is the caller graph for this function:

FileInfo * File_Namespace::FileMgr::openExistingFile ( const std::string &  path, const int32_t  fileId, const size_t  pageSize, const size_t  numPages, std::vector< HeaderInfo > &  headerVec  )

private

Definition at line 940 of file FileMgr.cpp.

References epoch(), generate_TableFunctionsFactory_init::f, fileIndex_, files_, files_rw_mutex_, File_Namespace::open(), and File_Namespace::FileInfo::openExistingFile().

Referenced by init().

                                                                      {
  FILE* f = open(path);
  FileInfo* fInfo = new FileInfo(
      this, fileId, f, pageSize, numPages, false);  // false means don't init file

  fInfo->openExistingFile(headerVec, epoch());
  mapd_unique_lock<mapd_shared_mutex> write_lock(files_rw_mutex_);
  if (fileId >= static_cast<int32_t>(files_.size())) {
    files_.resize(fileId + 1);
  }
  files_[fileId] = fInfo;
  fileIndex_.insert(std::pair<size_t, int32_t>(pageSize, fileId));
  return fInfo;
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::string File_Namespace::FileMgr::printSlabs ( )

inlineoverride

Definition at line 179 of file FileMgr.h.

{ return "Not Implemented"; }

void File_Namespace::FileMgr::processFileFutures ( std::vector< std::future< std::vector< HeaderInfo >>> &  file_futures, std::vector< HeaderInfo > &  headerVec  )

private

Definition at line 391 of file FileMgr.cpp.

Referenced by init().

                                      {
  for (auto& file_future : file_futures) {
    file_future.wait();
  }
  // concatenate the vectors after thread completes
  for (auto& file_future : file_futures) {
    auto tempHeaderVec = file_future.get();
    headerVec.insert(headerVec.end(), tempHeaderVec.begin(), tempHeaderVec.end());
  }
  file_futures.clear();
}

Here is the caller graph for this function:

FileBuffer * File_Namespace::FileMgr::putBuffer ( const ChunkKey &  key, AbstractBuffer *  d, const size_t  numBytes = 0  )

override

Puts the contents of d into the Chunk with the given key.

Parameters

key	- Unique identifier for a Chunk.
d	- An object representing the source data for the Chunk.

Returns

AbstractBuffer*

Definition at line 804 of file FileMgr.cpp.

References File_Namespace::FileBuffer::append(), CHECK_LT, chunkIndex_, chunkIndexMutex_, Data_Namespace::AbstractBuffer::clearDirtyBits(), createBufferUnlocked(), defaultPageSize_, logger::FATAL, Data_Namespace::AbstractBuffer::getDeviceId(), Data_Namespace::AbstractBuffer::getMemoryPtr(), Data_Namespace::AbstractBuffer::getType(), Data_Namespace::AbstractBuffer::isAppended(), Data_Namespace::AbstractBuffer::isDirty(), Data_Namespace::AbstractBuffer::isUpdated(), LOG, Data_Namespace::AbstractBuffer::setSize(), show_chunk(), Data_Namespace::AbstractBuffer::size(), Data_Namespace::AbstractBuffer::syncEncoder(), UNREACHABLE, and File_Namespace::FileBuffer::write().

                                                      {
  // obtain a pointer to the Chunk
  mapd_unique_lock<mapd_shared_mutex> chunkIndexWriteLock(chunkIndexMutex_);
  auto chunkIt = chunkIndex_.find(key);
  FileBuffer* chunk;
  if (chunkIt == chunkIndex_.end()) {
    chunk = createBufferUnlocked(key, defaultPageSize_);
  } else {
    chunk = chunkIt->second;
  }
  chunkIndexWriteLock.unlock();
  size_t oldChunkSize = chunk->size();
  // write the buffer's data to the Chunk
  // size_t newChunkSize = numBytes == 0 ? srcBuffer->size() : numBytes;
  size_t newChunkSize = numBytes == 0 ? srcBuffer->size() : numBytes;
  if (chunk->isDirty()) {
    // multiple appends are allowed,
    // but only single update is allowed
    if (srcBuffer->isUpdated() && chunk->isUpdated()) {
      LOG(FATAL) << "Aborting attempt to write a chunk marked dirty. Chunk inconsistency "
                    "for key: "
                 << show_chunk(key);
    }
  }
  if (srcBuffer->isUpdated()) {
    // chunk size is not changed when fixed rows are updated or are marked as deleted.
    // but when rows are vacuumed or varlen rows are updated (new rows are appended),
    // chunk size will change. For vacuum, checkpoint should sync size from cpu to disk.
    // For varlen update, it takes another route via fragmenter using disk-level buffer.
    if (0 == numBytes && !chunk->isDirty()) {
      chunk->setSize(newChunkSize);
    }
    //@todo use dirty flags to only flush pages of chunk that need to
    // be flushed
    chunk->write((int8_t*)srcBuffer->getMemoryPtr(),
                 newChunkSize,
                 0,
                 srcBuffer->getType(),
                 srcBuffer->getDeviceId());
  } else if (srcBuffer->isAppended()) {
    CHECK_LT(oldChunkSize, newChunkSize);
    chunk->append((int8_t*)srcBuffer->getMemoryPtr() + oldChunkSize,
                  newChunkSize - oldChunkSize,
                  srcBuffer->getType(),
                  srcBuffer->getDeviceId());
  } else {
    UNREACHABLE() << "putBuffer() expects a buffer marked is_updated or is_appended";
  }
  // chunk->clearDirtyBits(); // Hack: because write and append will set dirty bits
  //@todo commenting out line above will make sure this metadata is set
  // but will trigger error on fetch chunk
  srcBuffer->clearDirtyBits();
  chunk->syncEncoder(srcBuffer);
  return chunk;
}

Here is the call graph for this function:

int32_t File_Namespace::FileMgr::readVersionFromDisk ( const std::string &  versionFileName ) const

private

Definition at line 1068 of file FileMgr.cpp.

References File_Namespace::close(), omnisci::file_size(), fileMgrBasePath_, File_Namespace::open(), File_Namespace::read(), and setup::version.

Referenced by createTopLevelMetadata(), migrateToLatestFileMgrVersion(), and writeAndSyncVersionToDisk().

                                                                           {
  const std::string versionFilePath(fileMgrBasePath_ + "/" + versionFileName);
  if (!boost::filesystem::exists(versionFilePath)) {
    return -1;
  }
  if (!boost::filesystem::is_regular_file(versionFilePath)) {
    return -1;
  }
  if (boost::filesystem::file_size(versionFilePath) < 4) {
    return -1;
  }
  FILE* versionFile = open(versionFilePath);
  int32_t version;
  read(versionFile, 0, sizeof(int32_t), (int8_t*)&version);
  close(versionFile);
  return version;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::removeTableRelatedDS ( const int32_t  db_id, const int32_t  table_id  )

override

Definition at line 1182 of file FileMgr.cpp.

References UNREACHABLE.

                                                                              {
  UNREACHABLE();
}

Page File_Namespace::FileMgr::requestFreePage	(	size_t	pagesize,
		const bool	isMetadata
	)

Definition at line 871 of file FileMgr.cpp.

References CHECK, createFile(), File_Namespace::FileInfo::fileId, fileIndex_, files_, File_Namespace::FileInfo::getFreePage(), getPageMutex_, MAX_FILE_N_METADATA_PAGES, and MAX_FILE_N_PAGES.

Referenced by File_Namespace::FileBuffer::addNewMultiPage(), init(), File_Namespace::FileBuffer::write(), and File_Namespace::FileBuffer::writeMetadata().

                                                                    {
  std::lock_guard<std::mutex> lock(getPageMutex_);

  auto candidateFiles = fileIndex_.equal_range(pageSize);
  int32_t pageNum = -1;
  for (auto fileIt = candidateFiles.first; fileIt != candidateFiles.second; ++fileIt) {
    FileInfo* fileInfo = files_[fileIt->second];
    pageNum = fileInfo->getFreePage();
    if (pageNum != -1) {
      return (Page(fileInfo->fileId, pageNum));
    }
  }
  // if here then we need to add a file
  FileInfo* fileInfo;
  if (isMetadata) {
    fileInfo = createFile(pageSize, MAX_FILE_N_METADATA_PAGES);
  } else {
    fileInfo = createFile(pageSize, MAX_FILE_N_PAGES);
  }
  pageNum = fileInfo->getFreePage();
  CHECK(pageNum != -1);
  return (Page(fileInfo->fileId, pageNum));
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::requestFreePages	(	size_t	npages,
		size_t	pagesize,
		std::vector< Page > &	pages,
		const bool	isMetadata
	)

Obtains free pages – creates new files if necessary – of the requested size.

Given a page size and number of pages, this method updates the vector "pages" to include free pages of the requested size. These pages are immediately removed from the free list of the affected file(s). If there are not enough pages available among current files, new files are created and their pages are included in the vector.

Parameters

npages	The number of free pages requested
pagesize	The size of each requested page
pages	A vector containing the free pages obtained by this method

Definition at line 895 of file FileMgr.cpp.

References CHECK, createFile(), File_Namespace::FileInfo::fileId, fileIndex_, files_, File_Namespace::FileInfo::getFreePage(), getPageMutex_, MAX_FILE_N_METADATA_PAGES, and MAX_FILE_N_PAGES.

                                                      {
  // not used currently
  // @todo add method to FileInfo to get more than one page
  std::lock_guard<std::mutex> lock(getPageMutex_);
  auto candidateFiles = fileIndex_.equal_range(pageSize);
  size_t numPagesNeeded = numPagesRequested;
  for (auto fileIt = candidateFiles.first; fileIt != candidateFiles.second; ++fileIt) {
    FileInfo* fileInfo = files_[fileIt->second];
    int32_t pageNum;
    do {
      pageNum = fileInfo->getFreePage();
      if (pageNum != -1) {
        pages.emplace_back(fileInfo->fileId, pageNum);
        numPagesNeeded--;
      }
    } while (pageNum != -1 && numPagesNeeded > 0);
    if (numPagesNeeded == 0) {
      break;
    }
  }
  while (numPagesNeeded > 0) {
    FileInfo* fileInfo;
    if (isMetadata) {
      fileInfo = createFile(pageSize, MAX_FILE_N_METADATA_PAGES);
    } else {
      fileInfo = createFile(pageSize, MAX_FILE_N_PAGES);
    }
    int32_t pageNum;
    do {
      pageNum = fileInfo->getFreePage();
      if (pageNum != -1) {
        pages.emplace_back(fileInfo->fileId, pageNum);
        numPagesNeeded--;
      }
    } while (pageNum != -1 && numPagesNeeded > 0);
    if (numPagesNeeded == 0) {
      break;
    }
  }
  CHECK(pages.size() == numPagesRequested);
}

Here is the call graph for this function:

void File_Namespace::FileMgr::rollOffOldData ( const int32_t  epochCeiling, const bool  shouldCheckpoint  )

private

Definition at line 652 of file FileMgr.cpp.

References checkpoint(), epoch_, Epoch::floor(), freePagesBeforeEpoch(), and maxRollbackEpochs_.

Referenced by checkpoint(), and init().

                                                                                    {
  if (maxRollbackEpochs_ >= 0) {
    const int32_t minRollbackEpoch =
        std::max(epochCeiling - maxRollbackEpochs_, epoch_.floor());
    if (minRollbackEpoch > epoch_.floor()) {
      freePagesBeforeEpoch(minRollbackEpoch);
      epoch_.floor(minRollbackEpoch);
      if (shouldCheckpoint) {
        checkpoint();
      }
    }
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::setEpoch ( const int32_t  newEpoch )

private

Definition at line 1165 of file FileMgr.cpp.

References Epoch::ceiling(), epoch_, fileMgrKey_, Epoch::floor(), and writeAndSyncEpochToDisk().

Referenced by init().

                                             {
  if (newEpoch < epoch_.floor()) {
    std::stringstream error_message;
    error_message << "Cannot set epoch for table (" << fileMgrKey_.first << ","
                  << fileMgrKey_.second << ") lower than the minimum rollback epoch ("
                  << epoch_.floor() << ").";
    throw std::runtime_error(error_message.str());
  }
  epoch_.ceiling(newEpoch);
  writeAndSyncEpochToDisk();
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::writeAndSyncEpochToDisk ( )

private

Definition at line 628 of file FileMgr.cpp.

References Epoch::byte_size(), CHECK, epoch_, epochFile_, epochIsCheckpointed_, logger::FATAL, omnisci::fsync(), LOG, Epoch::storage_ptr(), and File_Namespace::write().

Referenced by checkpoint(), createEpochFile(), init(), migrateEpochFileV0(), and setEpoch().

                                      {
  CHECK(epochFile_);
  write(epochFile_, 0, Epoch::byte_size(), epoch_.storage_ptr());
  int32_t status = fflush(epochFile_);
  if (status != 0) {
    LOG(FATAL) << "Could not flush epoch file to disk";
  }
#ifdef __APPLE__
  status = fcntl(fileno(epochFile_), 51);
#else
  status = omnisci::fsync(fileno(epochFile_));
#endif
  if (status != 0) {
    LOG(FATAL) << "Could not sync epoch file to disk";
  }
  epochIsCheckpointed_ = true;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::writeAndSyncVersionToDisk ( const std::string &  versionFileName, const int32_t  version  )

private

Definition at line 1086 of file FileMgr.cpp.

References File_Namespace::close(), File_Namespace::create(), epochFile_, logger::FATAL, fileMgrBasePath_, omnisci::fsync(), logger::INFO, LOG, File_Namespace::open(), readVersionFromDisk(), and File_Namespace::write().

Referenced by createTopLevelMetadata(), init(), migrateEpochFileV0(), and migrateToLatestFileMgrVersion().

                                                               {
  const std::string versionFilePath(fileMgrBasePath_ + "/" + versionFileName);
  FILE* versionFile;
  if (boost::filesystem::exists(versionFilePath)) {
    int32_t oldVersion = readVersionFromDisk(versionFileName);
    LOG(INFO) << "Storage version file `" << versionFilePath
              << "` already exists, its current version is " << oldVersion;
    versionFile = open(versionFilePath);
  } else {
    versionFile = create(versionFilePath, sizeof(int32_t));
  }
  write(versionFile, 0, sizeof(int32_t), (int8_t*)&version);
  int32_t status = fflush(versionFile);
  if (status != 0) {
    LOG(FATAL) << "Could not flush version file " << versionFilePath << " to disk";
  }
#ifdef __APPLE__
  status = fcntl(fileno(epochFile_), 51);
#else
  status = omnisci::fsync(fileno(versionFile));
#endif
  if (status != 0) {
    LOG(FATAL) << "Could not sync version file " << versionFilePath << " to disk";
  }
  close(versionFile);
}

Here is the call graph for this function:

Here is the caller graph for this function:

Friends And Related Function Documentation

friend class GlobalFileMgr

friend

Definition at line 115 of file FileMgr.h.

Member Data Documentation

ChunkKeyToChunkMap File_Namespace::FileMgr::chunkIndex_

Definition at line 297 of file FileMgr.h.

Referenced by checkpoint(), createBufferUnlocked(), deleteBuffer(), deleteBuffersWithPrefix(), fetchBuffer(), freePagesBeforeEpoch(), getBuffer(), getChunkMetadataVecForKeyPrefix(), getNumUsedMetadataPages(), getNumUsedMetadataPagesForChunkKey(), init(), isBufferOnDevice(), putBuffer(), File_Namespace::GlobalFileMgr::writeFileMgrData(), and ~FileMgr().

mapd_shared_mutex File_Namespace::FileMgr::chunkIndexMutex_

mutableprivate

Definition at line 339 of file FileMgr.h.

Referenced by checkpoint(), createBuffer(), deleteBuffer(), deleteBuffersWithPrefix(), fetchBuffer(), getBuffer(), getChunkMetadataVecForKeyPrefix(), getNumUsedMetadataPages(), getNumUsedMetadataPagesForChunkKey(), isBufferOnDevice(), and putBuffer().

int32_t File_Namespace::FileMgr::db_version_

private

Definition at line 332 of file FileMgr.h.

Referenced by createTopLevelMetadata().

FILE* File_Namespace::FileMgr::DBMetaFile_ = nullptr

private

Definition at line 336 of file FileMgr.h.

Referenced by closeRemovePhysical(), getTotalFileSize(), and ~FileMgr().

size_t File_Namespace::FileMgr::defaultPageSize_

private

number of threads used when loading data

Definition at line 325 of file FileMgr.h.

Referenced by copyPage(), createBufferUnlocked(), getStorageStats(), and putBuffer().

Epoch File_Namespace::FileMgr::epoch_

private

the index of the next file id

Definition at line 327 of file FileMgr.h.

Referenced by FileMgr(), init(), migrateEpochFileV0(), openAndReadEpochFile(), rollOffOldData(), setEpoch(), and writeAndSyncEpochToDisk().

FILE* File_Namespace::FileMgr::epochFile_ = nullptr

private

Definition at line 331 of file FileMgr.h.

Referenced by closeRemovePhysical(), createEpochFile(), FileMgr(), getTotalFileSize(), openAndReadEpochFile(), writeAndSyncEpochToDisk(), writeAndSyncVersionToDisk(), and ~FileMgr().

bool File_Namespace::FileMgr::epochIsCheckpointed_ = true

private

Definition at line 328 of file FileMgr.h.

Referenced by writeAndSyncEpochToDisk().

PageSizeFileMMap File_Namespace::FileMgr::fileIndex_

private

A vector of files accessible via a file identifier.

Definition at line 323 of file FileMgr.h.

Referenced by createFile(), openExistingFile(), requestFreePage(), and requestFreePages().

std::string File_Namespace::FileMgr::fileMgrBasePath_

private

Definition at line 320 of file FileMgr.h.

Referenced by coreInit(), createEpochFile(), createFile(), FileMgr(), getStorageStats(), init(), migrateEpochFileV0(), openAndReadEpochFile(), openAndReadLegacyEpochFile(), readVersionFromDisk(), and writeAndSyncVersionToDisk().

std::pair<const int32_t, const int32_t> File_Namespace::FileMgr::fileMgrKey_

private

Global FileMgr.

Definition at line 318 of file FileMgr.h.

Referenced by checkpoint(), coreInit(), FileMgr(), and setEpoch().

int32_t File_Namespace::FileMgr::fileMgrVersion_

private

DB version from dbmeta file, should be compatible with GlobalFileMgr::omnisci_db_version_

Definition at line 334 of file FileMgr.h.

Referenced by init(), and migrateToLatestFileMgrVersion().

std::vector<FileInfo*> File_Namespace::FileMgr::files_

private

The OS file system path containing files related to this FileMgr

Definition at line 322 of file FileMgr.h.

Referenced by checkpoint(), closeRemovePhysical(), createFile(), FileMgr(), getFileForFileId(), getNumUsedPages(), getStorageStats(), getTotalFileSize(), openExistingFile(), requestFreePage(), requestFreePages(), and ~FileMgr().

mapd_shared_mutex File_Namespace::FileMgr::files_rw_mutex_

mutableprivate

Definition at line 340 of file FileMgr.h.

Referenced by checkpoint(), coreInit(), createFile(), getNumUsedPages(), getStorageStats(), and openExistingFile().

std::vector<std::pair<FileInfo*, int32_t> > File_Namespace::FileMgr::free_pages_

private

Definition at line 343 of file FileMgr.h.

Referenced by checkpoint(), free_page(), and init().

std::mutex File_Namespace::FileMgr::getPageMutex_

private

pointer to DB level metadata

Definition at line 338 of file FileMgr.h.

Referenced by requestFreePage(), and requestFreePages().

GlobalFileMgr* File_Namespace::FileMgr::gfm_

private

Definition at line 317 of file FileMgr.h.

Referenced by coreInit(), FileMgr(), getDBConvert(), getDBVersion(), and init().

bool File_Namespace::FileMgr::isFullyInitted_ {false}

private

Definition at line 344 of file FileMgr.h.

Referenced by getStorageStats(), and init().

const int32_t File_Namespace::FileMgr::latestFileMgrVersion_ {1}

private

Definition at line 335 of file FileMgr.h.

Referenced by init(), and migrateToLatestFileMgrVersion().

int32_t File_Namespace::FileMgr::maxRollbackEpochs_

private

Definition at line 319 of file FileMgr.h.

Referenced by rollOffOldData().

mapd_shared_mutex File_Namespace::FileMgr::mutex_free_page_

mutableprivate

Definition at line 342 of file FileMgr.h.

Referenced by checkpoint(), free_page(), and init().

unsigned File_Namespace::FileMgr::nextFileId_

private

Definition at line 326 of file FileMgr.h.

Referenced by createFile(), and init().

size_t File_Namespace::FileMgr::num_reader_threads_

private

Maps page sizes to FileInfo objects.

Definition at line 324 of file FileMgr.h.

Referenced by init().

---

The documentation for this class was generated from the following files:

• /home/jenkins-slave/workspace/core-os-doxygen/DataMgr/FileMgr/FileMgr.h
• /home/jenkins-slave/workspace/core-os-doxygen/DataMgr/FileMgr/FileMgr.cpp