File_Namespace::FileMgr Class Reference

#include <FileMgr.h>

Inheritance diagram for File_Namespace::FileMgr:

Collaboration diagram for File_Namespace::FileMgr:

Public Member Functions
	FileMgr (const int32_t device_id, GlobalFileMgr *gfm, const TablePair file_mgr_key, const int32_t max_rollback_epochs=-1, const size_t num_reader_threads=0, const int32_t epoch=-1)
	Constructor. More...
	FileMgr (const int32_t device_id, GlobalFileMgr *gfm, const TablePair file_mgr_key, const bool run_core_init)
	FileMgr (GlobalFileMgr *gfm, std::string basePath)
	~FileMgr () override
	Destructor. More...
StorageStats	getStorageStats () const
FileBuffer *	createBuffer (const ChunkKey &key, size_t pageSize=0, const size_t numBytes=0) override
	Creates a chunk with the specified key and page size. More...
bool	isBufferOnDevice (const ChunkKey &key) override
void	deleteBuffer (const ChunkKey &key, const bool purge=true) override
	Deletes the chunk with the specified key. More...
void	deleteBuffersWithPrefix (const ChunkKey &keyPrefix, const bool purge=true) override
FileBuffer *	getBuffer (const ChunkKey &key, const size_t numBytes=0) override
	Returns the a pointer to the chunk with the specified key. More...
void	fetchBuffer (const ChunkKey &key, AbstractBuffer *destBuffer, const size_t numBytes) override
FileBuffer *	putBuffer (const ChunkKey &key, AbstractBuffer *d, const size_t numBytes=0) override
	Puts the contents of d into the Chunk with the given key. More...
AbstractBuffer *	alloc (const size_t numBytes) override
void	free (AbstractBuffer *buffer) override
virtual Page	requestFreePage (size_t pagesize, const bool isMetadata)
MgrType	getMgrType () override
std::string	getStringMgrType () override
std::string	printSlabs () override
size_t	getMaxSize () override
size_t	getInUseSize () override
size_t	getAllocated () override
bool	isAllocationCapped () override
FileInfo *	getFileInfoForFileId (const int32_t fileId) const
FileMetadata	getMetadataForFile (const boost::filesystem::directory_iterator &fileIterator) const
void	copyPage (Page &srcPage, FileMgr *destFileMgr, Page &destPage, const size_t reservedHeaderSize, const size_t numBytes, const size_t offset)
void	requestFreePages (size_t npages, size_t pagesize, std::vector< Page > &pages, const bool isMetadata)
	Obtains free pages – creates new files if necessary – of the requested size. More...
void	getChunkMetadataVecForKeyPrefix (ChunkMetadataVector &chunkMetadataVec, const ChunkKey &keyPrefix) override
bool	hasChunkMetadataForKeyPrefix (const ChunkKey &keyPrefix)
void	checkpoint () override
	Fsyncs data files, writes out epoch and fsyncs that. More...
void	checkpoint (const int32_t db_id, const int32_t tb_id) override
virtual int32_t	epoch (int32_t db_id, int32_t tb_id) const
	Returns current value of epoch - should be one greater than recorded at last checkpoint. Because FileMgr only contains buffers from one table we can just return the FileMgr's epoch instead of finding a table-specific epoch. More...
int32_t	epochFloor () const
int32_t	incrementEpoch ()
int32_t	lastCheckpointedEpoch () const
	Returns value of epoch at last checkpoint. More...
void	resetEpochFloor ()
int32_t	maxRollbackEpochs ()
	Returns value max_rollback_epochs. More...
size_t	getNumReaderThreads ()
	Returns number of threads defined by parameter num-reader-threads which should be used during initial load and consequent read of data. More...
FILE *	getFileForFileId (const int32_t fileId)
	Returns FILE pointer associated with requested fileId. More...
size_t	getNumChunks () override
size_t	getNumUsedMetadataPagesForChunkKey (const ChunkKey &chunkKey) const
bool	getDBConvert () const
	Index for looking up chunks. More...
void	createOrMigrateTopLevelMetadata ()
std::string	getFileMgrBasePath () const
virtual void	closeRemovePhysical ()
void	removeTableRelatedDS (const int32_t db_id, const int32_t table_id) override
virtual void	free_page (std::pair< FileInfo *, int32_t > &&page)
virtual bool	hasFileMgrKey () const
const TablePair	get_fileMgrKey () const
boost::filesystem::path	getFilePath (const std::string &file_name) const
void	writePageMappingsToStatusFile (const std::vector< PageMapping > &page_mappings)
void	renameCompactionStatusFile (const char *const from_status, const char *const to_status)
void	compactFiles ()
virtual bool	updatePageIfDeleted (FileInfo *file_info, ChunkKey &chunk_key, int32_t contingent, int32_t page_epoch, int32_t page_num)
	deletes or recovers a page based on last checkpointed epoch. More...
virtual bool	failOnReadError () const
	True if a read error should cause a fatal error. More...
size_t	getPageSize () const
size_t	getMetadataPageSize () const
virtual std::string	describeSelf () const
FILE *	createFile (const std::string &full_path, const size_t requested_file_size) const
std::pair< FILE *, std::string >	createFile (const std::string &base_path, const int file_id, const size_t page_size, const size_t num_pages) const
size_t	writeFile (FILE *f, const size_t offset, const size_t size, const int8_t *buf) const

Static Public Member Functions
static void	setNumPagesPerDataFile (size_t num_pages)
static void	setNumPagesPerMetadataFile (size_t num_pages)
static void	renameAndSymlinkLegacyFiles (const std::string &table_data_dir)

Public Attributes
ChunkKeyToChunkMap	chunkIndex_

Static Public Attributes
static constexpr size_t	DEFAULT_NUM_PAGES_PER_DATA_FILE {256}
static constexpr size_t	DEFAULT_NUM_PAGES_PER_METADATA_FILE {4096}
static constexpr char const *	COPY_PAGES_STATUS {"pending_data_compaction_0"}
static constexpr char const *	UPDATE_PAGE_VISIBILITY_STATUS {"pending_data_compaction_1"}
static constexpr char const *	DELETE_EMPTY_FILES_STATUS {"pending_data_compaction_2"}
static constexpr char	LEGACY_EPOCH_FILENAME [] = "epoch"
static constexpr char	EPOCH_FILENAME [] = "epoch_metadata"
static constexpr char	DB_META_FILENAME [] = "dbmeta"
static constexpr char	FILE_MGR_VERSION_FILENAME [] = "filemgr_version"
static constexpr int32_t	INVALID_VERSION = -1
static constexpr int32_t	LATEST_FILE_MGR_VERSION = 2

Protected Member Functions
	FileMgr (const size_t defaultPageSize, const size_t defaultMetadataPageSize)
FileInfo *	createFileInfo (const size_t pageSize, const size_t numPages)
	Adds a file to the file manager repository. More...
FileInfo *	openExistingFile (const std::string &path, const int32_t fileId, const size_t pageSize, const size_t numPages, std::vector< HeaderInfo > &headerVec)
void	createEpochFile (const std::string &epochFileName)
int32_t	openAndReadLegacyEpochFile (const std::string &epochFileName)
void	openAndReadEpochFile (const std::string &epochFileName)
void	writeAndSyncEpochToDisk ()
void	setEpoch (const int32_t newEpoch)
int32_t	readVersionFromDisk (const std::string &versionFileName) const
void	writeAndSyncVersionToDisk (const std::string &versionFileName, const int32_t version)
void	processFileFutures (std::vector< std::future< std::vector< HeaderInfo >>> &file_futures, std::vector< HeaderInfo > &headerVec)
virtual FileBuffer *	createBufferUnlocked (const ChunkKey &key, size_t pageSize=0, const size_t numBytes=0)
virtual FileBuffer *	createBufferFromHeaders (const ChunkKey &key, const std::vector< HeaderInfo >::const_iterator &headerStartIt, const std::vector< HeaderInfo >::const_iterator &headerEndIt)
void	migrateToLatestFileMgrVersion ()
void	migrateEpochFileV0 ()
void	migrateLegacyFilesV1 ()
OpenFilesResult	openFiles ()
void	clearFileInfos ()
void	copySourcePageForCompaction (const Page &source_page, FileInfo *destination_file_info, std::vector< PageMapping > &page_mappings, std::set< Page > &touched_pages)
int32_t	copyPageWithoutHeaderSize (const Page &source_page, const Page &destination_page)
void	sortAndCopyFilePagesForCompaction (size_t page_size, std::vector< PageMapping > &page_mappings, std::set< Page > &touched_pages)
void	updateMappedPagesVisibility (const std::vector< PageMapping > &page_mappings)
void	deleteEmptyFiles ()
void	resumeFileCompaction (const std::string &status_file_name)
std::vector< PageMapping >	readPageMappingsFromStatusFile ()
	FileMgr (const int epoch)
void	closePhysicalUnlocked ()
void	syncFilesToDisk ()
void	freePages ()
void	initializeNumThreads (size_t num_reader_threads=0)
virtual FileBuffer *	allocateBuffer (const size_t page_size, const ChunkKey &key, const size_t num_bytes=0)
virtual FileBuffer *	allocateBuffer (const ChunkKey &key, const std::vector< HeaderInfo >::const_iterator &headerStartIt, const std::vector< HeaderInfo >::const_iterator &headerEndIt)
virtual ChunkKeyToChunkMap::iterator	deleteBufferUnlocked (const ChunkKeyToChunkMap::iterator chunk_it, const bool purge=true)
virtual FileBuffer *	getBufferUnlocked (const ChunkKey &key, const size_t numBytes=0) const

Protected Attributes
int32_t	maxRollbackEpochs_
std::string	fileMgrBasePath_
std::map< int32_t, std::unique_ptr< FileInfo > >	files_
PageSizeFileMMap	fileIndex_
size_t	num_reader_threads_
	Maps page sizes to FileInfo objects. More...
unsigned	nextFileId_
	number of threads used when loading data More...
int32_t	fileMgrVersion_
	the index of the next file id More...
FILE *	DBMetaFile_ = nullptr
std::mutex	getPageMutex_
	pointer to DB level metadata More...
heavyai::shared_mutex	chunkIndexMutex_
heavyai::shared_mutex	files_rw_mutex_
heavyai::shared_mutex	mutex_free_page_
std::vector< std::pair < FileInfo *, int32_t > >	free_pages_
bool	isFullyInitted_ {false}
const size_t	page_size_
const size_t	metadata_page_size_

Static Protected Attributes
static size_t	num_pages_per_data_file_ {DEFAULT_NUM_PAGES_PER_DATA_FILE}
static size_t	num_pages_per_metadata_file_ {DEFAULT_NUM_PAGES_PER_METADATA_FILE}

Private Member Functions
void	init (const size_t num_reader_threads, const int32_t epochOverride)
void	init (const std::string &dataPathToConvertFrom, const int32_t epochOverride)
void	rollOffOldData (const int32_t epochCeiling, const bool shouldCheckpoint)
void	freePagesBeforeEpoch (const int32_t min_epoch)
void	freePagesBeforeEpochUnlocked (const int32_t min_epoch, const ChunkKeyToChunkMap::iterator lower_bound, const ChunkKeyToChunkMap::iterator upper_bound)
FileBuffer *	getOrCreateBuffer (const ChunkKey &key)
bool	coreInit ()
	Determines file path, and if exists, runs file migration and opens and reads epoch file. More...
int32_t	epoch () const
void	writeDirtyBuffers ()
void	setDataAndMetadataFileStats (StorageStats &storage_stats) const
uint32_t	getFragmentCount () const
virtual void	readOnlyCheck (const std::string &action, const std::optional< std::string > &file_name={}) const

Private Attributes
GlobalFileMgr *	gfm_
TablePair	fileMgrKey_
	Global FileMgr. More...
Epoch	epoch_
bool	epochIsCheckpointed_ = true
FILE *	epochFile_ = nullptr

Friends
class	GlobalFileMgr

Detailed Description

Definition at line 161 of file FileMgr.h.

Constructor & Destructor Documentation

File_Namespace::FileMgr::FileMgr	(	const int32_t	device_id,
		GlobalFileMgr *	gfm,
		const TablePair	file_mgr_key,
		const int32_t	max_rollback_epochs = -1,
		const size_t	num_reader_threads = 0,
		const int32_t	epoch = -1
	)

Constructor.

Definition at line 51 of file FileMgr.cpp.

References init().

    : AbstractBufferMgr(device_id)
    , maxRollbackEpochs_(max_rollback_epochs)
    , nextFileId_(0)
    , gfm_(gfm)
    , fileMgrKey_(file_mgr_key)
    , page_size_(gfm->getPageSize())
    , metadata_page_size_(gfm->getMetadataPageSize()) {
  init(num_reader_threads, epoch);
}

Here is the call graph for this function:

File_Namespace::FileMgr::FileMgr	(	const int32_t	device_id,
		GlobalFileMgr *	gfm,
		const TablePair	file_mgr_key,
		const bool	run_core_init
	)

Definition at line 68 of file FileMgr.cpp.

References coreInit(), epochFile_, fileMgrBasePath_, fileMgrKey_, files_, File_Namespace::GlobalFileMgr::getBasePath(), gfm_, and to_string().

    : AbstractBufferMgr(device_id)
    , maxRollbackEpochs_(-1)
    , nextFileId_(0)
    , gfm_(gfm)
    , fileMgrKey_(file_mgr_key)
    , page_size_(gfm->getPageSize())
    , metadata_page_size_(gfm->getMetadataPageSize()) {
  // TODO(Misiu): Standardize prefix and delim.
  const std::string fileMgrDirPrefix("table");
  const std::string fileMgrDirDelim("_");
  fileMgrBasePath_ = (gfm_->getBasePath() + fileMgrDirPrefix + fileMgrDirDelim +
                      std::to_string(fileMgrKey_.first) +                     // db_id
                      fileMgrDirDelim + std::to_string(fileMgrKey_.second));  // tb_id
  epochFile_ = nullptr;
  files_.clear();
  if (run_core_init) {
    coreInit();
  }
}

Here is the call graph for this function:

File_Namespace::FileMgr::FileMgr	(	GlobalFileMgr *	gfm,
		std::string	basePath
	)

Definition at line 92 of file FileMgr.cpp.

References init().

    : AbstractBufferMgr(0)
    , maxRollbackEpochs_(-1)
    , fileMgrBasePath_(base_path)
    , nextFileId_(0)
    , gfm_(gfm)
    , fileMgrKey_(0, 0)
    , page_size_(gfm->getPageSize())
    , metadata_page_size_(gfm->getMetadataPageSize()) {
  init(base_path, -1);
}

Here is the call graph for this function:

File_Namespace::FileMgr::~FileMgr ( )

override

Destructor.

Definition at line 118 of file FileMgr.cpp.

References chunkIndex_, File_Namespace::close(), DBMetaFile_, epochFile_, and files_.

                  {
  // free memory used by FileInfo objects
  for (auto chunkIt = chunkIndex_.begin(); chunkIt != chunkIndex_.end(); ++chunkIt) {
    delete chunkIt->second;
  }

  files_.clear();

  if (epochFile_) {
    close(epochFile_);
    epochFile_ = nullptr;
  }

  if (DBMetaFile_) {
    close(DBMetaFile_);
    DBMetaFile_ = nullptr;
  }
}

Here is the call graph for this function:

File_Namespace::FileMgr::FileMgr ( const size_t  defaultPageSize, const size_t  defaultMetadataPageSize  )

protected

Definition at line 113 of file FileMgr.cpp.

    : AbstractBufferMgr(0)
    , page_size_(page_size)
    , metadata_page_size_(metadata_page_size) {}

File_Namespace::FileMgr::FileMgr ( const int  epoch )

protected

Definition at line 105 of file FileMgr.cpp.

References Epoch::ceiling(), and epoch_.

    : AbstractBufferMgr(-1)
    , page_size_(DEFAULT_PAGE_SIZE)
    , metadata_page_size_(DEFAULT_METADATA_PAGE_SIZE) {
  epoch_.ceiling(epoch);
}

Here is the call graph for this function:

Member Function Documentation

AbstractBuffer * File_Namespace::FileMgr::alloc ( const size_t  numBytes = 0 )

override

Definition at line 868 of file FileMgr.cpp.

References logger::FATAL, and LOG.

                                                        {
  LOG(FATAL) << "Operation not supported";
  return nullptr;  // satisfy return-type warning
}

FileBuffer * File_Namespace::FileMgr::allocateBuffer ( const size_t  page_size, const ChunkKey &  key, const size_t  num_bytes = 0  )

protectedvirtual

Reimplemented in File_Namespace::CachingFileMgr.

Definition at line 1623 of file FileMgr.cpp.

Referenced by createBufferFromHeaders(), and createBufferUnlocked().

                                                            {
  return new FileBuffer(this, page_size, key, num_bytes);
}

Here is the caller graph for this function:

FileBuffer * File_Namespace::FileMgr::allocateBuffer ( const ChunkKey &  key, const std::vector< HeaderInfo >::const_iterator &  headerStartIt, const std::vector< HeaderInfo >::const_iterator &  headerEndIt  )

protectedvirtual

Reimplemented in File_Namespace::CachingFileMgr.

Definition at line 1629 of file FileMgr.cpp.

                                                            {
  return new FileBuffer(this, key, headerStartIt, headerEndIt);
}

void File_Namespace::FileMgr::checkpoint ( )

override

Fsyncs data files, writes out epoch and fsyncs that.

Definition at line 706 of file FileMgr.cpp.

References describeSelf(), epoch(), freePages(), incrementEpoch(), rollOffOldData(), syncFilesToDisk(), VLOG, writeAndSyncEpochToDisk(), and writeDirtyBuffers().

Referenced by rollOffOldData().

                         {
  VLOG(2) << "Checkpointing " << describeSelf() << " epoch: " << epoch();
  writeDirtyBuffers();
  rollOffOldData(epoch(), false /* shouldCheckpoint */);
  syncFilesToDisk();
  writeAndSyncEpochToDisk();
  incrementEpoch();
  freePages();
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::checkpoint ( const int32_t  db_id, const int32_t  tb_id  )

inlineoverride

Definition at line 272 of file FileMgr.h.

References logger::FATAL, and LOG.

                                                                     {
    LOG(FATAL) << "Operation not supported, api checkpoint() should be used instead";
  }

void File_Namespace::FileMgr::clearFileInfos ( )

protected

Definition at line 247 of file FileMgr.cpp.

References fileIndex_, and files_.

Referenced by init().

                             {
  files_.clear();
  fileIndex_.clear();
}

Here is the caller graph for this function:

void File_Namespace::FileMgr::closePhysicalUnlocked ( )

protected

Definition at line 556 of file FileMgr.cpp.

References File_Namespace::close(), DBMetaFile_, epochFile_, and files_.

Referenced by File_Namespace::CachingFileMgr::closeRemovePhysical(), and closeRemovePhysical().

                                    {
  for (const auto& [idx, file_info] : files_) {
    if (file_info->f) {
      close(file_info->f);
      file_info->f = nullptr;
    }
  }

  if (DBMetaFile_) {
    close(DBMetaFile_);
    DBMetaFile_ = nullptr;
  }

  if (epochFile_) {
    close(epochFile_);
    epochFile_ = nullptr;
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::closeRemovePhysical ( )

virtual

Reimplemented in File_Namespace::CachingFileMgr.

Definition at line 575 of file FileMgr.cpp.

References closePhysicalUnlocked(), files_rw_mutex_, getFileMgrBasePath(), readOnlyCheck(), and File_Namespace::renameForDelete().

                                  {
  heavyai::unique_lock<heavyai::shared_mutex> write_lock(files_rw_mutex_);
  auto path = getFileMgrBasePath();
  readOnlyCheck("rename file", path);

  closePhysicalUnlocked();
  /* rename for later deletion the directory containing table related data */
  File_Namespace::renameForDelete(path);
}

Here is the call graph for this function:

void File_Namespace::FileMgr::compactFiles

(

)

Compacts metadata and data file pages and deletes resulting empty files (if any exists). Compaction occurs in 3 idempotent phases in order to enable graceful recovery if a crash/process interruption occurs in the middle data compaction.

Phase 1: Create a status file that indicates initiation of this phase. Sort metadata/data files in order of files with the lowest number of free pages to those with the highest number of free pages. Copy over used pages from files at the end of the sorted order (files with the highest number of free pages) to those at the beginning of the sorted order (files with the lowest number of free pages). Keep destination/copied to pages as free while copying. Keep track of copied source to destination page mapping. Write page mapping to the status file (to be used during crash recovery if needed).

Phase 2: Rename status file to a file name that indicates initiation of this phase. Go through page mapping and mark source/copied from pages as free while making the destination/copied to pages as used.

Phase 3: Rename status file to a file name that indicates initiation of this phase. Delete all empty files (files containing only free pages). Delete status file.

Definition at line 1276 of file FileMgr.cpp.

References CHECK, COPY_PAGES_STATUS, DELETE_EMPTY_FILES_STATUS, deleteEmptyFiles(), files_, files_rw_mutex_, getFilePath(), readOnlyCheck(), renameCompactionStatusFile(), sortAndCopyFilePagesForCompaction(), UPDATE_PAGE_VISIBILITY_STATUS, updateMappedPagesVisibility(), and writePageMappingsToStatusFile().

Referenced by resumeFileCompaction().

                           {
  heavyai::unique_lock<heavyai::shared_mutex> write_lock(files_rw_mutex_);

  readOnlyCheck("run file compaction");

  if (files_.empty()) {
    return;
  }

  auto copy_pages_status_file_path = getFilePath(COPY_PAGES_STATUS);
  CHECK(!boost::filesystem::exists(copy_pages_status_file_path));
  std::ofstream status_file(copy_pages_status_file_path.string(),
                            std::ios::out | std::ios::binary);
  status_file.close();

  std::vector<PageMapping> page_mappings;
  std::set<Page> touched_pages;
  std::set<size_t> page_sizes;
  for (const auto& [file_id, file_info] : files_) {
    page_sizes.emplace(file_info->pageSize);
  }
  for (auto page_size : page_sizes) {
    sortAndCopyFilePagesForCompaction(page_size, page_mappings, touched_pages);
  }

  writePageMappingsToStatusFile(page_mappings);
  renameCompactionStatusFile(COPY_PAGES_STATUS, UPDATE_PAGE_VISIBILITY_STATUS);

  updateMappedPagesVisibility(page_mappings);
  renameCompactionStatusFile(UPDATE_PAGE_VISIBILITY_STATUS, DELETE_EMPTY_FILES_STATUS);

  deleteEmptyFiles();
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::copyPage	(	Page &	srcPage,
		FileMgr *	destFileMgr,
		Page &	destPage,
		const size_t	reservedHeaderSize,
		const size_t	numBytes,
		const size_t	offset
	)

Definition at line 586 of file FileMgr.cpp.

References CHECK, checked_malloc(), File_Namespace::Page::fileId, free(), getFileInfoForFileId(), page_size_, File_Namespace::Page::pageNum, File_Namespace::FileInfo::read(), and File_Namespace::FileInfo::write().

Referenced by init().

                                            {
  CHECK(offset + numBytes <= page_size_);
  FileInfo* srcFileInfo = getFileInfoForFileId(srcPage.fileId);
  FileInfo* destFileInfo = destFileMgr->getFileInfoForFileId(destPage.fileId);
  int8_t* buffer = reinterpret_cast<int8_t*>(checked_malloc(numBytes));
  ScopeGuard guard = [&buffer] { ::free(buffer); };

  size_t bytesRead = srcFileInfo->read(
      srcPage.pageNum * page_size_ + offset + reservedHeaderSize, numBytes, buffer);
  CHECK(bytesRead == numBytes);
  size_t bytesWritten = destFileInfo->write(
      destPage.pageNum * page_size_ + offset + reservedHeaderSize, numBytes, buffer);
  CHECK(bytesWritten == numBytes);
}

Here is the call graph for this function:

Here is the caller graph for this function:

int32_t File_Namespace::FileMgr::copyPageWithoutHeaderSize ( const Page &  source_page, const Page &  destination_page  )

protected

Copies content of source_page to destination_page without copying over the source_page header size. The header size is instead returned by the method. Not copying over the header size enables a use case where destination_page has all the content of the source_page but is still marked as a free page.

Definition at line 1441 of file FileMgr.cpp.

References CHECK, CHECK_EQ, File_Namespace::Page::fileId, File_Namespace::FileInfo::fileId, getFileInfoForFileId(), File_Namespace::Page::pageNum, File_Namespace::FileInfo::pageSize, File_Namespace::FileInfo::read(), and File_Namespace::FileInfo::write().

Referenced by copySourcePageForCompaction().

                                                                         {
  FileInfo* source_file_info = getFileInfoForFileId(source_page.fileId);
  CHECK(source_file_info);
  CHECK_EQ(source_file_info->fileId, source_page.fileId);

  FileInfo* destination_file_info = getFileInfoForFileId(destination_page.fileId);
  CHECK(destination_file_info);
  CHECK_EQ(destination_file_info->fileId, destination_page.fileId);
  CHECK_EQ(source_file_info->pageSize, destination_file_info->pageSize);

  auto page_size = source_file_info->pageSize;
  auto buffer = std::make_unique<int8_t[]>(page_size);
  size_t bytes_read =
      source_file_info->read(source_page.pageNum * page_size, page_size, buffer.get());
  CHECK_EQ(page_size, bytes_read);

  auto header_size_offset = sizeof(int32_t);
  size_t bytes_written = destination_file_info->write(
      (destination_page.pageNum * page_size) + header_size_offset,
      page_size - header_size_offset,
      buffer.get() + header_size_offset);
  CHECK_EQ(page_size - header_size_offset, bytes_written);
  return reinterpret_cast<int32_t*>(buffer.get())[0];
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::copySourcePageForCompaction ( const Page &  source_page, FileInfo *  destination_file_info, std::vector< PageMapping > &  page_mappings, std::set< Page > &  touched_pages  )

protected

Copies a used page (indicated by the top of the source_used_pages set) from the given source file to a free page in the given destination file. Source and destination pages are recorded in the given page_mappings vector after copying is done.

Definition at line 1411 of file FileMgr.cpp.

References CHECK, CHECK_NE, copyPageWithoutHeaderSize(), File_Namespace::Page::fileId, File_Namespace::FileInfo::fileId, File_Namespace::FileInfo::getFreePage(), and File_Namespace::Page::pageNum.

Referenced by sortAndCopyFilePagesForCompaction().

                                                                       {
  size_t destination_page_num = destination_file_info->getFreePage();
  CHECK_NE(destination_page_num, static_cast<size_t>(-1));
  Page destination_page{destination_file_info->fileId, destination_page_num};

  // Assert that the same pages are not copied or overridden multiple times
  CHECK(touched_pages.find(source_page) == touched_pages.end());
  touched_pages.emplace(source_page);

  CHECK(touched_pages.find(destination_page) == touched_pages.end());
  touched_pages.emplace(destination_page);

  auto header_size = copyPageWithoutHeaderSize(source_page, destination_page);
  page_mappings.emplace_back(static_cast<size_t>(source_page.fileId),
                             source_page.pageNum,
                             header_size,
                             static_cast<size_t>(destination_page.fileId),
                             destination_page.pageNum);
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool File_Namespace::FileMgr::coreInit ( )

private

Determines file path, and if exists, runs file migration and opens and reads epoch file.

Returns

a boolean representing whether the directory path existed

Definition at line 137 of file FileMgr.cpp.

References EPOCH_FILENAME, logger::FATAL, fileMgrBasePath_, fileMgrKey_, files_rw_mutex_, File_Namespace::GlobalFileMgr::getBasePath(), gfm_, LOG, migrateToLatestFileMgrVersion(), openAndReadEpochFile(), and to_string().

Referenced by FileMgr(), and init().

                       {
  heavyai::unique_lock<heavyai::shared_mutex> write_lock(files_rw_mutex_);
  const std::string fileMgrDirPrefix("table");
  const std::string FileMgrDirDelim("_");
  fileMgrBasePath_ = (gfm_->getBasePath() + fileMgrDirPrefix + FileMgrDirDelim +
                      std::to_string(fileMgrKey_.first) +                     // db_id
                      FileMgrDirDelim + std::to_string(fileMgrKey_.second));  // tb_id
  boost::filesystem::path path(fileMgrBasePath_);
  if (boost::filesystem::exists(path)) {
    if (!boost::filesystem::is_directory(path)) {
      LOG(FATAL) << "Specified path '" << fileMgrBasePath_
                 << "' for table data is not a directory.";
    }
    migrateToLatestFileMgrVersion();
    openAndReadEpochFile(EPOCH_FILENAME);
    return true;
  }
  return false;
}

Here is the call graph for this function:

Here is the caller graph for this function:

FileBuffer * File_Namespace::FileMgr::createBuffer ( const ChunkKey &  key, size_t  pageSize = 0, const size_t  numBytes = 0  )

override

Creates a chunk with the specified key and page size.

Definition at line 716 of file FileMgr.cpp.

References CHECK, chunkIndex_, chunkIndexMutex_, createBufferUnlocked(), and show_chunk().

Referenced by init().

                                                          {
  heavyai::unique_lock<heavyai::shared_mutex> chunkIndexWriteLock(chunkIndexMutex_);
  CHECK(chunkIndex_.find(key) == chunkIndex_.end())
      << "Chunk already exists: " + show_chunk(key);
  return createBufferUnlocked(key, page_size, num_bytes);
}

Here is the call graph for this function:

Here is the caller graph for this function:

FileBuffer * File_Namespace::FileMgr::createBufferFromHeaders ( const ChunkKey &  key, const std::vector< HeaderInfo >::const_iterator &  headerStartIt, const std::vector< HeaderInfo >::const_iterator &  headerEndIt  )

protectedvirtual

Reimplemented in File_Namespace::CachingFileMgr.

Definition at line 737 of file FileMgr.cpp.

References allocateBuffer(), CHECK, chunkIndex_, chunkIndexMutex_, and show_chunk().

Referenced by init().

                                                            {
  heavyai::unique_lock<heavyai::shared_mutex> chunkIndexWriteLock(chunkIndexMutex_);
  CHECK(chunkIndex_.find(key) == chunkIndex_.end())
      << "Chunk already exists for key: " << show_chunk(key);
  chunkIndex_[key] = allocateBuffer(key, headerStartIt, headerEndIt);
  return (chunkIndex_[key]);
}

Here is the call graph for this function:

Here is the caller graph for this function:

FileBuffer * File_Namespace::FileMgr::createBufferUnlocked ( const ChunkKey &  key, size_t  pageSize = 0, const size_t  numBytes = 0  )

protectedvirtual

Reimplemented in File_Namespace::CachingFileMgr.

Definition at line 726 of file FileMgr.cpp.

References allocateBuffer(), chunkIndex_, and page_size_.

Referenced by createBuffer(), and getOrCreateBuffer().

                                                                  {
  size_t actual_page_size = page_size;
  if (actual_page_size == 0) {
    actual_page_size = page_size_;
  }
  chunkIndex_[key] = allocateBuffer(actual_page_size, key, num_bytes);
  return (chunkIndex_[key]);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::createEpochFile ( const std::string &  epochFileName )

protected

Definition at line 606 of file FileMgr.cpp.

References Epoch::byte_size(), File_Namespace::create(), epochFile_, logger::FATAL, fileMgrBasePath_, LOG, readOnlyCheck(), and writeAndSyncEpochToDisk().

Referenced by init(), and migrateEpochFileV0().

                                                            {
  std::string epochFilePath(fileMgrBasePath_ + "/" + epochFileName);
  readOnlyCheck("create file", epochFilePath);
  if (boost::filesystem::exists(epochFilePath)) {
    LOG(FATAL) << "Epoch file '" << epochFilePath << "' already exists";
  }
  epochFile_ = create(epochFilePath, sizeof(Epoch::byte_size()));
  // Write out current epoch to file - which if this
  // function is being called should be 0
  writeAndSyncEpochToDisk();
}

Here is the call graph for this function:

Here is the caller graph for this function:

FILE * File_Namespace::FileMgr::createFile	(	const std::string &	full_path,
		const size_t	requested_file_size
	)		const

Definition at line 1703 of file FileMgr.cpp.

References File_Namespace::create(), and readOnlyCheck().

                                                                  {
  readOnlyCheck("create file", full_path);
  return create(full_path, requested_file_size);
}

Here is the call graph for this function:

std::pair< FILE *, std::string > File_Namespace::FileMgr::createFile	(	const std::string &	base_path,
		const int	file_id,
		const size_t	page_size,
		const size_t	num_pages
	)		const

Definition at line 1709 of file FileMgr.cpp.

References File_Namespace::create(), File_Namespace::get_data_file_path(), and readOnlyCheck().

                                                                                {
  readOnlyCheck("create file", get_data_file_path(base_path, file_id, page_size));
  return create(base_path, file_id, page_size, num_pages);
}

Here is the call graph for this function:

FileInfo * File_Namespace::FileMgr::createFileInfo ( const size_t  pageSize, const size_t  numPages  )

protected

Adds a file to the file manager repository.

This method will create a FileInfo object for the file being added, and it will create the corresponding file on physical disk with the indicated number of pages pre-allocated.

A pointer to the FileInfo object is returned, which itself has a file pointer (FILE*) and a file identifier (int32_t fileId).

Parameters

fileName	The name given to the file in physical storage.
pageSize	The logical page size for the pages in the file.
numPages	The number of logical pages to initially allocate for the file.

Returns

FileInfo* A pointer to the FileInfo object of the added file.

Definition at line 968 of file FileMgr.cpp.

References CHECK, File_Namespace::create(), f(), logger::FATAL, fileIndex_, fileMgrBasePath_, files_, files_rw_mutex_, File_Namespace::get_data_file_path(), getFileInfoForFileId(), LOG, nextFileId_, and readOnlyCheck().

Referenced by requestFreePage(), and requestFreePages().

                                                                              {
  readOnlyCheck("create file",
                get_data_file_path(fileMgrBasePath_, nextFileId_, pageSize));
  // check arguments
  if (pageSize == 0 || numPages == 0) {
    LOG(FATAL) << "File creation failed: pageSize and numPages must be greater than 0.";
  }

  // create the new file
  auto [f, file_path] = create(fileMgrBasePath_,
                               nextFileId_,
                               pageSize,
                               numPages);  // TM: not sure if I like naming scheme here -
                                           // should be in separate namespace?
  CHECK(f);

  // instantiate a new FileInfo for the newly created file
  int32_t fileId = nextFileId_++;
  auto fInfo = std::make_unique<FileInfo>(this,
                                          fileId,
                                          f,
                                          pageSize,
                                          numPages,
                                          file_path,
                                          true);  // true means init file
  CHECK(fInfo);

  heavyai::unique_lock<heavyai::shared_mutex> write_lock(files_rw_mutex_);
  // update file manager data structures
  files_[fileId] = std::move(fInfo);
  fileIndex_.insert(std::pair<size_t, int32_t>(pageSize, fileId));

  return getFileInfoForFileId(fileId);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::createOrMigrateTopLevelMetadata

(

)

Definition at line 1056 of file FileMgr.cpp.

References DB_META_FILENAME, File_Namespace::GlobalFileMgr::db_version_, logger::FATAL, gfm_, INVALID_VERSION, LOG, readVersionFromDisk(), and writeAndSyncVersionToDisk().

                                              {
  auto file_version = readVersionFromDisk(DB_META_FILENAME);
  auto gfm_version = gfm_->db_version_;

  if (file_version > gfm_version) {
    LOG(FATAL) << "DB forward compatibility is not supported. Version of HeavyDB "
                  "software used is older than the version of DB being read: "
               << file_version;
  }
  if (file_version == INVALID_VERSION || file_version < gfm_version) {
    // new system, or we are moving forward versions
    // system wide migration would go here if required
    writeAndSyncVersionToDisk(DB_META_FILENAME, gfm_version);
    return;
  }
}

Here is the call graph for this function:

void File_Namespace::FileMgr::deleteBuffer ( const ChunkKey &  key, const bool  purge = true  )

override

Deletes the chunk with the specified key.

Definition at line 753 of file FileMgr.cpp.

References CHECK, chunkIndex_, chunkIndexMutex_, deleteBufferUnlocked(), and show_chunk().

                                                                {
  heavyai::unique_lock<heavyai::shared_mutex> chunkIndexWriteLock(chunkIndexMutex_);
  auto chunk_it = chunkIndex_.find(key);
  CHECK(chunk_it != chunkIndex_.end()) << "Chunk does not exist: " << show_chunk(key);
  deleteBufferUnlocked(chunk_it, purge);
}

Here is the call graph for this function:

void File_Namespace::FileMgr::deleteBuffersWithPrefix ( const ChunkKey &  keyPrefix, const bool  purge = true  )

override

Definition at line 770 of file FileMgr.cpp.

References chunkIndex_, chunkIndexMutex_, and deleteBufferUnlocked().

                                                                                 {
  heavyai::unique_lock<heavyai::shared_mutex> chunkIndexWriteLock(chunkIndexMutex_);
  auto chunkIt = chunkIndex_.lower_bound(keyPrefix);
  if (chunkIt == chunkIndex_.end()) {
    return;  // should we throw?
  }
  while (chunkIt != chunkIndex_.end() &&
         std::search(chunkIt->first.begin(),
                     chunkIt->first.begin() + keyPrefix.size(),
                     keyPrefix.begin(),
                     keyPrefix.end()) != chunkIt->first.begin() + keyPrefix.size()) {
    deleteBufferUnlocked(chunkIt++, purge);
  }
}

Here is the call graph for this function:

ChunkKeyToChunkMap::iterator File_Namespace::FileMgr::deleteBufferUnlocked ( const ChunkKeyToChunkMap::iterator  chunk_it, const bool  purge = true  )

protectedvirtual

Reimplemented in File_Namespace::CachingFileMgr.

Definition at line 760 of file FileMgr.cpp.

References chunkIndex_.

Referenced by deleteBuffer(), and deleteBuffersWithPrefix().

                      {
  if (purge) {
    chunk_it->second->freePages();
  }
  delete chunk_it->second;
  return chunkIndex_.erase(chunk_it);
}

Here is the caller graph for this function:

void File_Namespace::FileMgr::deleteEmptyFiles ( )

protected

Deletes files that contain only free pages. Also deletes the compaction status file.

Definition at line 1504 of file FileMgr.cpp.

References CHECK, CHECK_EQ, DELETE_EMPTY_FILES_STATUS, fileMgrBasePath_, files_, File_Namespace::get_data_file_path(), File_Namespace::get_legacy_data_file_path(), getFilePath(), and readOnlyCheck().

Referenced by compactFiles(), and resumeFileCompaction().

                               {
  for (const auto& [file_id, file_info] : files_) {
    CHECK_EQ(file_id, file_info->fileId);
    if (file_info->freePages.size() == file_info->numPages) {
      fclose(file_info->f);
      file_info->f = nullptr;
      auto file_path = get_data_file_path(fileMgrBasePath_, file_id, file_info->pageSize);
      readOnlyCheck("delete file", file_path);
      boost::filesystem::remove(get_legacy_data_file_path(file_path));
      boost::filesystem::remove(file_path);
    }
  }

  auto status_file_path = getFilePath(DELETE_EMPTY_FILES_STATUS);
  CHECK(boost::filesystem::exists(status_file_path));
  readOnlyCheck("delete file", status_file_path.string());
  boost::filesystem::remove(status_file_path);
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::string File_Namespace::FileMgr::describeSelf ( ) const

virtual

Reimplemented in File_Namespace::CachingFileMgr.

Definition at line 700 of file FileMgr.cpp.

References fileMgrKey_.

Referenced by checkpoint(), and setEpoch().

                                      {
  stringstream ss;
  ss << "table (" << fileMgrKey_.first << ", " << fileMgrKey_.second << ")";
  return ss.str();
}

Here is the caller graph for this function:

virtual int32_t File_Namespace::FileMgr::epoch ( int32_t  db_id, int32_t  tb_id  ) const

inlinevirtual

Returns current value of epoch - should be one greater than recorded at last checkpoint. Because FileMgr only contains buffers from one table we can just return the FileMgr's epoch instead of finding a table-specific epoch.

Reimplemented in File_Namespace::CachingFileMgr.

Definition at line 281 of file FileMgr.h.

References epoch().

Referenced by epoch(), File_Namespace::FileBuffer::getFileMgrEpoch(), and File_Namespace::FileInfo::openExistingFile().

{ return epoch(); }

Here is the call graph for this function:

Here is the caller graph for this function:

int32_t File_Namespace::FileMgr::epoch ( ) const

inlineprivate

Definition at line 530 of file FileMgr.h.

Referenced by checkpoint(), init(), openAndReadLegacyEpochFile(), updatePageIfDeleted(), and writeDirtyBuffers().

{ return static_cast<int32_t>(epoch_.ceiling()); }

Here is the caller graph for this function:

int32_t File_Namespace::FileMgr::epochFloor ( ) const

inline

Definition at line 283 of file FileMgr.h.

Referenced by setDataAndMetadataFileStats().

{ return static_cast<int32_t>(epoch_.floor()); }

Here is the caller graph for this function:

virtual bool File_Namespace::FileMgr::failOnReadError ( ) const

inlinevirtual

True if a read error should cause a fatal error.

Reimplemented in File_Namespace::CachingFileMgr.

Definition at line 366 of file FileMgr.h.

Referenced by File_Namespace::FileBuffer::FileBuffer().

{ return true; }

Here is the caller graph for this function:

void File_Namespace::FileMgr::fetchBuffer ( const ChunkKey &  key, AbstractBuffer *  destBuffer, const size_t  numBytes  )

override

Definition at line 797 of file FileMgr.cpp.

References CHECK, Data_Namespace::AbstractBuffer::copyTo(), logger::FATAL, getBuffer(), Data_Namespace::AbstractBuffer::isDirty(), LOG, show_chunk(), and Data_Namespace::AbstractBuffer::size().

                                                 {
  // reads chunk specified by ChunkKey into AbstractBuffer provided by
  // destBuffer
  CHECK(!destBuffer->isDirty())
      << "Aborting attempt to fetch a chunk marked dirty. Chunk inconsistency for key: "
      << show_chunk(key);
  AbstractBuffer* chunk = getBuffer(key);
  // chunk's size is either specified in function call with numBytes or we
  // just look at pageSize * numPages in FileBuffer
  if (numBytes > 0 && numBytes > chunk->size()) {
    LOG(FATAL) << "Chunk retrieved for key `" << show_chunk(key) << "` is smaller ("
               << chunk->size() << ") than number of bytes requested (" << numBytes
               << ")";
  }
  chunk->copyTo(destBuffer, numBytes);
}

Here is the call graph for this function:

void File_Namespace::FileMgr::free ( AbstractBuffer *  buffer )

override

Definition at line 873 of file FileMgr.cpp.

References logger::FATAL, and LOG.

Referenced by copyPage().

                                         {
  LOG(FATAL) << "Operation not supported";
}

Here is the caller graph for this function:

void File_Namespace::FileMgr::free_page ( std::pair< FileInfo *, int32_t > &&  page )

virtual

Reimplemented in File_Namespace::CachingFileMgr.

Definition at line 1210 of file FileMgr.cpp.

References free_pages_, and mutex_free_page_.

Referenced by File_Namespace::FileInfo::freePage(), and freePages().

                                                          {
  std::unique_lock<heavyai::shared_mutex> lock(mutex_free_page_);
  free_pages_.push_back(page);
}

Here is the caller graph for this function:

void File_Namespace::FileMgr::freePages ( )

protected

Definition at line 1615 of file FileMgr.cpp.

References free_page(), free_pages_, and mutex_free_page_.

Referenced by checkpoint(), File_Namespace::CachingFileMgr::clearForTable(), File_Namespace::CachingFileMgr::init(), init(), and sortAndCopyFilePagesForCompaction().

                        {
  heavyai::unique_lock<heavyai::shared_mutex> free_pages_write_lock(mutex_free_page_);
  for (auto& free_page : free_pages_) {
    free_page.first->freePageDeferred(free_page.second);
  }
  free_pages_.clear();
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::freePagesBeforeEpoch ( const int32_t  min_epoch )

private

Definition at line 673 of file FileMgr.cpp.

References chunkIndex_, chunkIndexMutex_, and freePagesBeforeEpochUnlocked().

Referenced by rollOffOldData().

                                                          {
  heavyai::shared_lock<heavyai::shared_mutex> chunk_index_read_lock(chunkIndexMutex_);
  freePagesBeforeEpochUnlocked(min_epoch, chunkIndex_.begin(), chunkIndex_.end());
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::freePagesBeforeEpochUnlocked ( const int32_t  min_epoch, const ChunkKeyToChunkMap::iterator  lower_bound, const ChunkKeyToChunkMap::iterator  upper_bound  )

private

Definition at line 678 of file FileMgr.cpp.

References gpu_enabled::upper_bound().

Referenced by freePagesBeforeEpoch().

                                                  {
  for (auto chunkIt = lower_bound; chunkIt != upper_bound; ++chunkIt) {
    chunkIt->second->freePagesBeforeEpoch(min_epoch);
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

const TablePair File_Namespace::FileMgr::get_fileMgrKey ( ) const

inline

Definition at line 341 of file FileMgr.h.

Referenced by updatePageIfDeleted().

{ return fileMgrKey_; }

Here is the caller graph for this function:

size_t File_Namespace::FileMgr::getAllocated ( )

inlineoverride

Definition at line 226 of file FileMgr.h.

{ return 0; }

FileBuffer * File_Namespace::FileMgr::getBuffer ( const ChunkKey &  key, const size_t  numBytes = 0  )

override

Returns the a pointer to the chunk with the specified key.

Definition at line 785 of file FileMgr.cpp.

References chunkIndexMutex_, and getBufferUnlocked().

Referenced by fetchBuffer().

                                                                          {
  heavyai::shared_lock<heavyai::shared_mutex> chunk_index_read_lock(chunkIndexMutex_);
  return getBufferUnlocked(key, num_bytes);
}

Here is the call graph for this function:

Here is the caller graph for this function:

FileBuffer * File_Namespace::FileMgr::getBufferUnlocked ( const ChunkKey &  key, const size_t  numBytes = 0  ) const

protectedvirtual

Reimplemented in File_Namespace::CachingFileMgr.

Definition at line 790 of file FileMgr.cpp.

References CHECK, chunkIndex_, and show_chunk().

Referenced by getBuffer(), and getOrCreateBuffer().

                                                                     {
  auto chunk_it = chunkIndex_.find(key);
  CHECK(chunk_it != chunkIndex_.end()) << "Chunk does not exist: " << show_chunk(key);
  return chunk_it->second;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::getChunkMetadataVecForKeyPrefix ( ChunkMetadataVector &  chunkMetadataVec, const ChunkKey &  keyPrefix  )

override

Definition at line 1021 of file FileMgr.cpp.

References chunkIndex_, and chunkIndexMutex_.

                                                                         {
  heavyai::unique_lock<heavyai::shared_mutex> chunkIndexWriteLock(chunkIndexMutex_);
  auto chunkIt = chunkIndex_.lower_bound(keyPrefix);
  if (chunkIt == chunkIndex_.end()) {
    return;  // throw?
  }
  while (chunkIt != chunkIndex_.end() &&
         std::search(chunkIt->first.begin(),
                     chunkIt->first.begin() + keyPrefix.size(),
                     keyPrefix.begin(),
                     keyPrefix.end()) != chunkIt->first.begin() + keyPrefix.size()) {
    if (chunkIt->second->hasEncoder()) {
      auto chunk_metadata = std::make_shared<ChunkMetadata>();
      chunkIt->second->encoder_->getMetadata(chunk_metadata);
      chunkMetadataVec.emplace_back(chunkIt->first, chunk_metadata);
    }
    chunkIt++;
  }
}

bool File_Namespace::FileMgr::getDBConvert

(

)

const

Index for looking up chunks.

Definition at line 1052 of file FileMgr.cpp.

References File_Namespace::GlobalFileMgr::getDBConvert(), and gfm_.

                                 {
  return gfm_->getDBConvert();
}

Here is the call graph for this function:

FILE * File_Namespace::FileMgr::getFileForFileId

(

const int32_t

fileId

)

Returns FILE pointer associated with requested fileId.

See Also

FileBuffer

Definition at line 1003 of file FileMgr.cpp.

References CHECK, and files_.

Referenced by File_Namespace::FileBuffer::readMetadata(), and File_Namespace::FileBuffer::writeMetadata().

                                                    {
  CHECK(fileId >= 0);
  CHECK(files_.find(fileId) != files_.end()) << "File does not exist for id: " << fileId;
  return files_.at(fileId)->f;
}

Here is the caller graph for this function:

FileInfo* File_Namespace::FileMgr::getFileInfoForFileId ( const int32_t  fileId ) const

inline

Definition at line 229 of file FileMgr.h.

Referenced by File_Namespace::FileBuffer::append(), File_Namespace::FileBuffer::copyPage(), copyPage(), copyPageWithoutHeaderSize(), createFileInfo(), File_Namespace::FileBuffer::freePage(), openExistingFile(), File_Namespace::readForThread(), requestFreePage(), requestFreePages(), updateMappedPagesVisibility(), File_Namespace::FileBuffer::write(), and File_Namespace::FileBuffer::writeHeader().

                                                                    {
    return files_.at(fileId).get();
  }

Here is the caller graph for this function:

std::string File_Namespace::FileMgr::getFileMgrBasePath ( ) const

inline

Definition at line 334 of file FileMgr.h.

Referenced by File_Namespace::CachingFileMgr::closeRemovePhysical(), closeRemovePhysical(), and File_Namespace::CachingFileMgr::getTableFileMgrPath().

{ return fileMgrBasePath_; }

Here is the caller graph for this function:

boost::filesystem::path File_Namespace::FileMgr::getFilePath

(

const std::string &

file_name

)

const

Definition at line 1699 of file FileMgr.cpp.

References fileMgrBasePath_.

Referenced by compactFiles(), deleteEmptyFiles(), readPageMappingsFromStatusFile(), renameCompactionStatusFile(), resumeFileCompaction(), and writePageMappingsToStatusFile().

                                                                         {
  return boost::filesystem::path(fileMgrBasePath_) / file_name;
}

Here is the caller graph for this function:

uint32_t File_Namespace::FileMgr::getFragmentCount ( ) const

private

Definition at line 407 of file FileMgr.cpp.

References CHUNK_KEY_FRAGMENT_IDX, chunkIndex_, and chunkIndexMutex_.

Referenced by getStorageStats().

                                         {
  heavyai::shared_lock<heavyai::shared_mutex> chunk_index_read_lock(chunkIndexMutex_);
  std::set<int32_t> fragment_ids;
  for (const auto& [chunk_key, file_buffer] : chunkIndex_) {
    fragment_ids.emplace(chunk_key[CHUNK_KEY_FRAGMENT_IDX]);
  }
  return static_cast<uint32_t>(fragment_ids.size());
}

Here is the caller graph for this function:

size_t File_Namespace::FileMgr::getInUseSize ( )

inlineoverride

Definition at line 225 of file FileMgr.h.

{ return 0; }

size_t File_Namespace::FileMgr::getMaxSize ( )

inlineoverride

Definition at line 224 of file FileMgr.h.

{ return 0; }

FileMetadata File_Namespace::FileMgr::getMetadataForFile

(

const boost::filesystem::directory_iterator &

fileIterator

)

const

Definition at line 157 of file FileMgr.cpp.

References CHECK_EQ, DATA_FILE_EXT, logger::FATAL, File_Namespace::FileMetadata::file_id, File_Namespace::FileMetadata::file_path, heavyai::file_size(), File_Namespace::FileMetadata::file_size, File_Namespace::FileMetadata::is_data_file, LOG, File_Namespace::FileMetadata::num_pages, and File_Namespace::FileMetadata::page_size.

Referenced by init(), openFiles(), and setDataAndMetadataFileStats().

                                                                 {
  FileMetadata fileMetadata;
  fileMetadata.is_data_file = false;
  fileMetadata.file_path = fileIterator->path().string();
  if (!boost::filesystem::is_regular_file(fileIterator->status())) {
    return fileMetadata;
  }
  // note that boost::filesystem leaves preceding dot on
  // extension - hence DATA_FILE_EXT is ".data"
  std::string extension(fileIterator->path().extension().string());
  if (extension == DATA_FILE_EXT) {
    std::string fileStem(fileIterator->path().stem().string());
    // remove trailing dot if any
    if (fileStem.size() > 0 && fileStem.back() == '.') {
      fileStem = fileStem.substr(0, fileStem.size() - 1);
    }
    size_t dotPos = fileStem.find_last_of(".");  // should only be one
    if (dotPos == std::string::npos) {
      LOG(FATAL) << "File `" << fileIterator->path()
                 << "` does not carry page size information in the filename.";
    }
    fileMetadata.is_data_file = true;
    fileMetadata.file_id = boost::lexical_cast<int>(fileStem.substr(0, dotPos));
    fileMetadata.page_size =
        boost::lexical_cast<size_t>(fileStem.substr(dotPos + 1, fileStem.size()));

    fileMetadata.file_size = boost::filesystem::file_size(fileMetadata.file_path);
    CHECK_EQ(fileMetadata.file_size % fileMetadata.page_size,
             size_t(0));  // should be no partial pages
    fileMetadata.num_pages = fileMetadata.file_size / fileMetadata.page_size;
  }
  return fileMetadata;
}

Here is the call graph for this function:

Here is the caller graph for this function:

size_t File_Namespace::FileMgr::getMetadataPageSize ( ) const

inline

Definition at line 369 of file FileMgr.h.

{ return metadata_page_size_; }

MgrType File_Namespace::FileMgr::getMgrType ( )

inlineoverride

Definition at line 221 of file FileMgr.h.

{ return FILE_MGR; };

size_t File_Namespace::FileMgr::getNumChunks ( )

override

Definition at line 1694 of file FileMgr.cpp.

References chunkIndex_, and chunkIndexMutex_.

                             {
  heavyai::shared_lock<heavyai::shared_mutex> read_lock(chunkIndexMutex_);
  return chunkIndex_.size();
}

size_t File_Namespace::FileMgr::getNumReaderThreads ( )

inline

Returns number of threads defined by parameter num-reader-threads which should be used during initial load and consequent read of data.

Definition at line 316 of file FileMgr.h.

Referenced by File_Namespace::FileBuffer::read().

{ return num_reader_threads_; }

Here is the caller graph for this function:

size_t File_Namespace::FileMgr::getNumUsedMetadataPagesForChunkKey

(

const ChunkKey &

chunkKey

)

const

Definition at line 1042 of file FileMgr.cpp.

References chunkIndex_, and chunkIndexMutex_.

                                                                                 {
  heavyai::shared_lock<heavyai::shared_mutex> read_lock(chunkIndexMutex_);
  const auto& chunkIt = chunkIndex_.find(chunkKey);
  if (chunkIt != chunkIndex_.end()) {
    return chunkIt->second->numMetadataPages();
  } else {
    throw std::runtime_error("Chunk was not found.");
  }
}

FileBuffer * File_Namespace::FileMgr::getOrCreateBuffer ( const ChunkKey &  key )

private

Definition at line 1672 of file FileMgr.cpp.

References chunkIndex_, chunkIndexMutex_, createBufferUnlocked(), and getBufferUnlocked().

Referenced by putBuffer().

                                                          {
  FileBuffer* buf;
  heavyai::unique_lock<heavyai::shared_mutex> chunkIndexWriteLock(chunkIndexMutex_);
  auto chunk_it = chunkIndex_.find(key);
  if (chunk_it == chunkIndex_.end()) {
    buf = createBufferUnlocked(key);
  } else {
    buf = getBufferUnlocked(key);
  }
  return buf;
}

Here is the call graph for this function:

Here is the caller graph for this function:

size_t File_Namespace::FileMgr::getPageSize ( ) const

inline

Definition at line 368 of file FileMgr.h.

{ return page_size_; }

StorageStats File_Namespace::FileMgr::getStorageStats

(

)

const

Definition at line 334 of file FileMgr.cpp.

References File_Namespace::StorageStats::fragment_count, getFragmentCount(), isFullyInitted_, and setDataAndMetadataFileStats().

                                            {
  StorageStats storage_stats;
  setDataAndMetadataFileStats(storage_stats);
  if (isFullyInitted_) {
    storage_stats.fragment_count = getFragmentCount();
  }
  return storage_stats;
}

Here is the call graph for this function:

std::string File_Namespace::FileMgr::getStringMgrType ( )

inlineoverride

Definition at line 222 of file FileMgr.h.

{ return ToString(FILE_MGR); }

bool File_Namespace::FileMgr::hasChunkMetadataForKeyPrefix

(

const ChunkKey &

keyPrefix

)

Definition at line 1009 of file FileMgr.cpp.

References chunkIndex_, and chunkIndexMutex_.

                                                                     {
  heavyai::shared_lock<heavyai::shared_mutex> chunk_index_read_lock(chunkIndexMutex_);
  auto chunk_it = chunkIndex_.lower_bound(key_prefix);
  if (chunk_it == chunkIndex_.end()) {
    return false;
  } else {
    auto it_pair =
        std::mismatch(key_prefix.begin(), key_prefix.end(), chunk_it->first.begin());
    return it_pair.first == key_prefix.end();
  }
}

virtual bool File_Namespace::FileMgr::hasFileMgrKey ( ) const

inlinevirtual

Reimplemented in File_Namespace::CachingFileMgr.

Definition at line 340 of file FileMgr.h.

{ return true; }

int32_t File_Namespace::FileMgr::incrementEpoch ( )

inline

Definition at line 285 of file FileMgr.h.

References logger::FATAL, LOG, and Epoch::max_allowable_epoch().

Referenced by checkpoint(), and init().

                                  {
    int32_t newEpoch = epoch_.increment();
    epochIsCheckpointed_ = false;
    // We test for error here instead of in Epoch::increment so we can log FileMgr
    // metadata
    if (newEpoch > Epoch::max_allowable_epoch()) {
      LOG(FATAL) << "Epoch for table (" << fileMgrKey_.first << ", " << fileMgrKey_.second
                 << ") greater than maximum allowed value of "
                 << Epoch::max_allowable_epoch() << ".";
    }
    return newEpoch;
  }

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::init ( const size_t  num_reader_threads, const int32_t  epochOverride  )

private

Definition at line 252 of file FileMgr.cpp.

References Epoch::ceiling(), CHECK, clearFileInfos(), coreInit(), createBufferFromHeaders(), createEpochFile(), epoch(), epoch_, EPOCH_FILENAME, logger::FATAL, FILE_MGR_VERSION_FILENAME, fileMgrBasePath_, fileMgrVersion_, Epoch::floor(), freePages(), incrementEpoch(), initializeNumThreads(), isFullyInitted_, LATEST_FILE_MGR_VERSION, LOG, nextFileId_, openFiles(), readOnlyCheck(), resumeFileCompaction(), rollOffOldData(), setEpoch(), gpu_enabled::sort(), VLOG, and writeAndSyncVersionToDisk().

Referenced by FileMgr().

                                                                               {
  // if epochCeiling = -1 this means open from epoch file

  const bool dataExists = coreInit();
  if (dataExists) {
    if (epochOverride != -1) {  // if opening at specified epoch
      setEpoch(epochOverride);
    }

    auto open_files_result = openFiles();
    if (!open_files_result.compaction_status_file_name.empty()) {
      resumeFileCompaction(open_files_result.compaction_status_file_name);
      clearFileInfos();
      open_files_result = openFiles();
      CHECK(open_files_result.compaction_status_file_name.empty());
    }

    /* Sort headerVec so that all HeaderInfos
     * from a chunk will be grouped together
     * and in order of increasing PageId
     * - Version Epoch */
    auto& header_vec = open_files_result.header_infos;
    std::sort(header_vec.begin(), header_vec.end());

    /* Goal of next section is to find sequences in the
     * sorted headerVec of the same ChunkId, which we
     * can then initiate a FileBuffer with */

    VLOG(3) << "Number of Headers in Vector: " << header_vec.size();
    if (header_vec.size() > 0) {
      ChunkKey lastChunkKey = header_vec.begin()->chunkKey;
      auto startIt = header_vec.begin();

      for (auto headerIt = header_vec.begin() + 1; headerIt != header_vec.end();
           ++headerIt) {
        if (headerIt->chunkKey != lastChunkKey) {
          createBufferFromHeaders(lastChunkKey, startIt, headerIt);
          lastChunkKey = headerIt->chunkKey;
          startIt = headerIt;
        }
      }
      // now need to insert last Chunk
      createBufferFromHeaders(lastChunkKey, startIt, header_vec.end());
    }
    nextFileId_ = open_files_result.max_file_id + 1;
    rollOffOldData(epoch(), true /* only checkpoint if data is rolled off */);
    incrementEpoch();
    freePages();
  } else {
    boost::filesystem::path path(fileMgrBasePath_);
    readOnlyCheck("create file", path.string());
    if (!boost::filesystem::create_directory(path)) {
      LOG(FATAL) << "Could not create data directory: " << path;
    }
    fileMgrVersion_ = LATEST_FILE_MGR_VERSION;
    if (epochOverride != -1) {
      epoch_.floor(epochOverride);
      epoch_.ceiling(epochOverride);
    } else {
      // These are default constructor values for epoch_, but resetting here for clarity
      epoch_.floor(0);
      epoch_.ceiling(0);
    }
    createEpochFile(EPOCH_FILENAME);
    writeAndSyncVersionToDisk(FILE_MGR_VERSION_FILENAME, fileMgrVersion_);
    incrementEpoch();
  }

  initializeNumThreads(num_reader_threads);
  isFullyInitted_ = true;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::init ( const std::string &  dataPathToConvertFrom, const int32_t  epochOverride  )

private

Definition at line 430 of file FileMgr.cpp.

References threading_serial::async(), CHECK, copyPage(), createBuffer(), createBufferFromHeaders(), EPOCH_FILENAME, logger::FATAL, File_Namespace::FileMetadata::file_id, File_Namespace::FileMetadata::file_path, File_Namespace::GlobalFileMgr::getFileMgr(), getMetadataForFile(), gfm_, File_Namespace::FileMetadata::is_data_file, isFullyInitted_, LOG, nextFileId_, File_Namespace::FileMetadata::num_pages, openAndReadEpochFile(), openExistingFile(), File_Namespace::FileMetadata::page_size, processFileFutures(), File_Namespace::MultiPage::push(), readOnlyCheck(), requestFreePage(), setEpoch(), gpu_enabled::sort(), and Data_Namespace::AbstractBuffer::syncEncoder().

                                                {
  int32_t converted_data_epoch = 0;
  boost::filesystem::path path(dataPathToConvertFrom);
  if (boost::filesystem::exists(path)) {
    if (!boost::filesystem::is_directory(path)) {
      LOG(FATAL) << "Specified path `" << path << "` is not a directory.";
    }
    openAndReadEpochFile(EPOCH_FILENAME);

    if (epochOverride != -1) {  // if opening at previous epoch
      setEpoch(epochOverride);
    }

    boost::filesystem::directory_iterator
        endItr;  // default construction yields past-the-end
    int32_t maxFileId = -1;
    int32_t fileCount = 0;
    int32_t threadCount = std::thread::hardware_concurrency();
    std::vector<HeaderInfo> headerVec;
    std::vector<std::future<std::vector<HeaderInfo>>> file_futures;
    for (boost::filesystem::directory_iterator fileIt(path); fileIt != endItr; ++fileIt) {
      FileMetadata fileMetadata = getMetadataForFile(fileIt);
      if (fileMetadata.is_data_file) {
        maxFileId = std::max(maxFileId, fileMetadata.file_id);
        file_futures.emplace_back(std::async(std::launch::async, [fileMetadata, this] {
          std::vector<HeaderInfo> tempHeaderVec;
          openExistingFile(fileMetadata.file_path,
                           fileMetadata.file_id,
                           fileMetadata.page_size,
                           fileMetadata.num_pages,
                           tempHeaderVec);
          return tempHeaderVec;
        }));
        fileCount++;
        if (fileCount % threadCount) {
          processFileFutures(file_futures, headerVec);
        }
      }
    }

    if (file_futures.size() > 0) {
      processFileFutures(file_futures, headerVec);
    }

    /* Sort headerVec so that all HeaderInfos
     * from a chunk will be grouped together
     * and in order of increasing PageId
     * - Version Epoch */

    std::sort(headerVec.begin(), headerVec.end());

    /* Goal of next section is to find sequences in the
     * sorted headerVec of the same ChunkId, which we
     * can then initiate a FileBuffer with */

    if (headerVec.size() > 0) {
      ChunkKey lastChunkKey = headerVec.begin()->chunkKey;
      auto startIt = headerVec.begin();

      for (auto headerIt = headerVec.begin() + 1; headerIt != headerVec.end();
           ++headerIt) {
        if (headerIt->chunkKey != lastChunkKey) {
          FileMgr* c_fm_ =
              dynamic_cast<File_Namespace::FileMgr*>(gfm_->getFileMgr(lastChunkKey));
          CHECK(c_fm_);
          auto srcBuf = createBufferFromHeaders(lastChunkKey, startIt, headerIt);
          auto destBuf = c_fm_->createBuffer(lastChunkKey, srcBuf->pageSize());
          destBuf->syncEncoder(srcBuf);
          destBuf->setSize(srcBuf->size());
          destBuf->setDirty();  // this needs to be set to force writing out metadata
                                // files from "checkpoint()" call

          size_t totalNumPages = srcBuf->getMultiPage().size();
          for (size_t pageNum = 0; pageNum < totalNumPages; pageNum++) {
            Page srcPage = srcBuf->getMultiPage()[pageNum].current().page;
            Page destPage = c_fm_->requestFreePage(
                srcBuf->pageSize(),
                false);  // may modify and use api "FileBuffer::addNewMultiPage" instead
            MultiPage multiPage(srcBuf->pageSize());
            multiPage.push(destPage, converted_data_epoch);
            destBuf->multiPages_.push_back(multiPage);
            size_t reservedHeaderSize = srcBuf->reservedHeaderSize();
            copyPage(
                srcPage, c_fm_, destPage, reservedHeaderSize, srcBuf->pageDataSize(), 0);
            destBuf->writeHeader(destPage, pageNum, converted_data_epoch, false);
          }
          lastChunkKey = headerIt->chunkKey;
          startIt = headerIt;
        }
      }

      // now need to insert last Chunk
      FileMgr* c_fm_ =
          dynamic_cast<File_Namespace::FileMgr*>(gfm_->getFileMgr(lastChunkKey));
      auto srcBuf = createBufferFromHeaders(lastChunkKey, startIt, headerVec.end());
      auto destBuf = c_fm_->createBuffer(lastChunkKey, srcBuf->pageSize());
      destBuf->syncEncoder(srcBuf);
      destBuf->setSize(srcBuf->size());
      destBuf->setDirty();  // this needs to be set to write out metadata file from the
                            // "checkpoint()" call

      size_t totalNumPages = srcBuf->getMultiPage().size();
      for (size_t pageNum = 0; pageNum < totalNumPages; pageNum++) {
        Page srcPage = srcBuf->getMultiPage()[pageNum].current().page;
        Page destPage = c_fm_->requestFreePage(
            srcBuf->pageSize(),
            false);  // may modify and use api "FileBuffer::addNewMultiPage" instead
        MultiPage multiPage(srcBuf->pageSize());
        multiPage.push(destPage, converted_data_epoch);
        destBuf->multiPages_.push_back(multiPage);
        size_t reservedHeaderSize = srcBuf->reservedHeaderSize();
        copyPage(srcPage, c_fm_, destPage, reservedHeaderSize, srcBuf->pageDataSize(), 0);
        destBuf->writeHeader(destPage, pageNum, converted_data_epoch, false);
      }
    }
    nextFileId_ = maxFileId + 1;
  } else {
    readOnlyCheck("create file", path.string());
    if (!boost::filesystem::create_directory(path)) {
      LOG(FATAL) << "Specified path does not exist: " << path;
    }
  }
  isFullyInitted_ = true;
}

Here is the call graph for this function:

void File_Namespace::FileMgr::initializeNumThreads ( size_t  num_reader_threads = 0 )

protected

Definition at line 1604 of file FileMgr.cpp.

References num_reader_threads_.

Referenced by File_Namespace::CachingFileMgr::init(), and init().

                                                            {
  // # of threads is based on # of cores on the host
  size_t num_hardware_based_threads = std::thread::hardware_concurrency();
  if (num_reader_threads == 0 || num_reader_threads > num_hardware_based_threads) {
    // # of threads has not been defined by user
    num_reader_threads_ = num_hardware_based_threads;
  } else {
    num_reader_threads_ = num_reader_threads;
  }
}

Here is the caller graph for this function:

bool File_Namespace::FileMgr::isAllocationCapped ( )

inlineoverride

Definition at line 227 of file FileMgr.h.

{ return false; }

bool File_Namespace::FileMgr::isBufferOnDevice ( const ChunkKey &  key )

override

Definition at line 748 of file FileMgr.cpp.

References chunkIndex_, and chunkIndexMutex_.

                                                  {
  heavyai::shared_lock<heavyai::shared_mutex> chunkIndexReadLock(chunkIndexMutex_);
  return chunkIndex_.find(key) != chunkIndex_.end();
}

int32_t File_Namespace::FileMgr::lastCheckpointedEpoch ( ) const

inline

Returns value of epoch at last checkpoint.

Definition at line 301 of file FileMgr.h.

Referenced by File_Namespace::GlobalFileMgr::existsDiffBetweenFileMgrParamsAndFileMgr(), and setDataAndMetadataFileStats().

                                               {
    return epoch() - (epochIsCheckpointed_ ? 0 : 1);
  }

Here is the caller graph for this function:

int32_t File_Namespace::FileMgr::maxRollbackEpochs ( )

inline

Returns value max_rollback_epochs.

Definition at line 310 of file FileMgr.h.

Referenced by File_Namespace::GlobalFileMgr::existsDiffBetweenFileMgrParamsAndFileMgr().

{ return maxRollbackEpochs_; }

Here is the caller graph for this function:

void File_Namespace::FileMgr::migrateEpochFileV0 ( )

protected

Definition at line 1116 of file FileMgr.cpp.

References Epoch::ceiling(), createEpochFile(), epoch_, EPOCH_FILENAME, FILE_MGR_VERSION_FILENAME, fileMgrBasePath_, Epoch::floor(), logger::INFO, LEGACY_EPOCH_FILENAME, LOG, Epoch::min_allowable_epoch(), openAndReadLegacyEpochFile(), readOnlyCheck(), writeAndSyncEpochToDisk(), and writeAndSyncVersionToDisk().

Referenced by migrateToLatestFileMgrVersion().

                                 {
  const std::string versionFilePath(fileMgrBasePath_ + "/" + FILE_MGR_VERSION_FILENAME);
  LOG(INFO) << "Migrating file format version from 0 to 1 for  `" << versionFilePath;
  readOnlyCheck("migrate epoch file", versionFilePath);
  epoch_.floor(Epoch::min_allowable_epoch());
  epoch_.ceiling(openAndReadLegacyEpochFile(LEGACY_EPOCH_FILENAME));
  createEpochFile(EPOCH_FILENAME);
  writeAndSyncEpochToDisk();
  int32_t migrationCompleteVersion = 1;
  writeAndSyncVersionToDisk(FILE_MGR_VERSION_FILENAME, migrationCompleteVersion);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::migrateLegacyFilesV1 ( )

protected

Definition at line 1128 of file FileMgr.cpp.

References FILE_MGR_VERSION_FILENAME, fileMgrBasePath_, logger::INFO, LOG, readOnlyCheck(), renameAndSymlinkLegacyFiles(), and writeAndSyncVersionToDisk().

Referenced by migrateToLatestFileMgrVersion().

                                   {
  LOG(INFO) << "Migrating file format version from 1 to 2";
  readOnlyCheck("migrate file format version");
  renameAndSymlinkLegacyFiles(fileMgrBasePath_);
  constexpr int32_t migration_complete_version{2};
  writeAndSyncVersionToDisk(FILE_MGR_VERSION_FILENAME, migration_complete_version);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::migrateToLatestFileMgrVersion ( )

protected

Definition at line 1157 of file FileMgr.cpp.

References logger::FATAL, FILE_MGR_VERSION_FILENAME, fileMgrVersion_, INVALID_VERSION, LATEST_FILE_MGR_VERSION, LOG, migrateEpochFileV0(), migrateLegacyFilesV1(), readVersionFromDisk(), UNREACHABLE, and writeAndSyncVersionToDisk().

Referenced by coreInit().

                                            {
  fileMgrVersion_ = readVersionFromDisk(FILE_MGR_VERSION_FILENAME);
  if (fileMgrVersion_ == INVALID_VERSION) {
    fileMgrVersion_ = 0;
    writeAndSyncVersionToDisk(FILE_MGR_VERSION_FILENAME, fileMgrVersion_);
  } else if (fileMgrVersion_ > LATEST_FILE_MGR_VERSION) {
    LOG(FATAL)
        << "Table storage forward compatibility is not supported. Version of HeavyDB "
           "software used is older than the version of table being read: "
        << fileMgrVersion_;
  }

  while (fileMgrVersion_ < LATEST_FILE_MGR_VERSION) {
    switch (fileMgrVersion_) {
      case 0: {
        migrateEpochFileV0();
        break;
      }
      case 1: {
        migrateLegacyFilesV1();
        break;
      }
      default: {
        UNREACHABLE();
      }
    }
    fileMgrVersion_++;
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::openAndReadEpochFile ( const std::string &  epochFileName )

protected

Definition at line 639 of file FileMgr.cpp.

References Epoch::byte_size(), epoch_, epochFile_, logger::FATAL, heavyai::file_size(), fileMgrBasePath_, LOG, File_Namespace::open(), File_Namespace::read(), and Epoch::storage_ptr().

Referenced by coreInit(), and init().

                                                                 {
  if (!epochFile_) {  // Check to see if already open
    std::string epochFilePath(fileMgrBasePath_ + "/" + epochFileName);
    if (!boost::filesystem::exists(epochFilePath)) {
      LOG(FATAL) << "Epoch file `" << epochFilePath << "` does not exist";
    }
    if (!boost::filesystem::is_regular_file(epochFilePath)) {
      LOG(FATAL) << "Epoch file `" << epochFilePath << "` is not a regular file";
    }
    if (boost::filesystem::file_size(epochFilePath) != Epoch::byte_size()) {
      LOG(FATAL) << "Epoch file `" << epochFilePath
                 << "` is not sized properly (current size: "
                 << boost::filesystem::file_size(epochFilePath)
                 << ", expected size: " << Epoch::byte_size() << ")";
    }
    epochFile_ = open(epochFilePath);
  }
  read(epochFile_, 0, Epoch::byte_size(), epoch_.storage_ptr(), epochFileName);
}

Here is the call graph for this function:

Here is the caller graph for this function:

int32_t File_Namespace::FileMgr::openAndReadLegacyEpochFile ( const std::string &  epochFileName )

protected

Definition at line 618 of file FileMgr.cpp.

References File_Namespace::close(), epoch(), logger::FATAL, heavyai::file_size(), fileMgrBasePath_, LOG, File_Namespace::open(), and File_Namespace::read().

Referenced by migrateEpochFileV0().

                                                                          {
  std::string epochFilePath(fileMgrBasePath_ + "/" + epochFileName);
  if (!boost::filesystem::exists(epochFilePath)) {
    return 0;
  }

  if (!boost::filesystem::is_regular_file(epochFilePath)) {
    LOG(FATAL) << "Epoch file `" << epochFilePath << "` is not a regular file";
  }
  if (boost::filesystem::file_size(epochFilePath) < 4) {
    LOG(FATAL) << "Epoch file `" << epochFilePath
               << "` is not sized properly (current size: "
               << boost::filesystem::file_size(epochFilePath) << ", expected size: 4)";
  }
  FILE* legacyEpochFile = open(epochFilePath);
  int32_t epoch;
  read(legacyEpochFile, 0, sizeof(int32_t), (int8_t*)&epoch, epochFilePath);
  close(legacyEpochFile);
  return epoch;
}

Here is the call graph for this function:

Here is the caller graph for this function:

FileInfo * File_Namespace::FileMgr::openExistingFile ( const std::string &  path, const int32_t  fileId, const size_t  pageSize, const size_t  numPages, std::vector< HeaderInfo > &  headerVec  )

protected

Definition at line 946 of file FileMgr.cpp.

References CHECK, f(), fileIndex_, files_, files_rw_mutex_, getFileInfoForFileId(), and File_Namespace::open().

Referenced by init(), and openFiles().

                                                                      {
  FILE* f = open(path);
  auto file_info = std::make_unique<FileInfo>(this,
                                              fileId,
                                              f,
                                              pageSize,
                                              numPages,
                                              path,
                                              false);  // false means don't init file

  file_info->openExistingFile(headerVec);
  heavyai::unique_lock<heavyai::shared_mutex> write_lock(files_rw_mutex_);
  CHECK(files_.find(fileId) == files_.end()) << "Attempting to re-open file";
  files_.emplace(fileId, std::move(file_info));
  fileIndex_.insert(std::pair<size_t, int32_t>(pageSize, fileId));
  return getFileInfoForFileId(fileId);
}

Here is the call graph for this function:

Here is the caller graph for this function:

OpenFilesResult File_Namespace::FileMgr::openFiles ( )

protected

Definition at line 200 of file FileMgr.cpp.

References threading_serial::async(), Epoch::ceiling(), CHECK, File_Namespace::OpenFilesResult::compaction_status_file_name, epoch_, File_Namespace::FileMetadata::file_id, File_Namespace::FileMetadata::file_path, fileMgrBasePath_, getMetadataForFile(), File_Namespace::OpenFilesResult::header_infos, logger::INFO, File_Namespace::anonymous_namespace{FileMgr.cpp}::is_compaction_status_file(), File_Namespace::FileMetadata::is_data_file, LOG, File_Namespace::OpenFilesResult::max_file_id, File_Namespace::FileMetadata::num_pages, openExistingFile(), File_Namespace::FileMetadata::page_size, processFileFutures(), run_benchmark_import::result, timer_start(), and timer_stop().

Referenced by File_Namespace::CachingFileMgr::init(), and init().

                                   {
  auto clock_begin = timer_start();
  boost::filesystem::directory_iterator
      end_itr;  // default construction yields past-the-end
  OpenFilesResult result;
  result.max_file_id = -1;
  int32_t file_count = 0;
  int32_t thread_count = std::thread::hardware_concurrency();
  std::vector<std::future<std::vector<HeaderInfo>>> file_futures;
  boost::filesystem::path path(fileMgrBasePath_);
  for (boost::filesystem::directory_iterator file_it(path); file_it != end_itr;
       ++file_it) {
    FileMetadata file_metadata = getMetadataForFile(file_it);
    if (file_metadata.is_data_file) {
      result.max_file_id = std::max(result.max_file_id, file_metadata.file_id);
      file_futures.emplace_back(std::async(std::launch::async, [file_metadata, this] {
        std::vector<HeaderInfo> temp_header_vec;
        openExistingFile(file_metadata.file_path,
                         file_metadata.file_id,
                         file_metadata.page_size,
                         file_metadata.num_pages,
                         temp_header_vec);
        return temp_header_vec;
      }));
      file_count++;
      if (file_count % thread_count == 0) {
        processFileFutures(file_futures, result.header_infos);
      }
    }

    if (is_compaction_status_file(file_it->path().filename().string())) {
      CHECK(result.compaction_status_file_name.empty());
      result.compaction_status_file_name = file_it->path().filename().string();
    }
  }

  if (file_futures.size() > 0) {
    processFileFutures(file_futures, result.header_infos);
  }

  int64_t queue_time_ms = timer_stop(clock_begin);
  LOG(INFO) << "Completed Reading table's file metadata, Elapsed time : " << queue_time_ms
            << "ms Epoch: " << epoch_.ceiling() << " files read: " << file_count
            << " table location: '" << fileMgrBasePath_ << "'";
  return result;
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::string File_Namespace::FileMgr::printSlabs ( )

inlineoverride

Definition at line 223 of file FileMgr.h.

{ return "Not Implemented"; }

void File_Namespace::FileMgr::processFileFutures ( std::vector< std::future< std::vector< HeaderInfo >>> &  file_futures, std::vector< HeaderInfo > &  headerVec  )

protected

Definition at line 416 of file FileMgr.cpp.

Referenced by init(), and openFiles().

                                      {
  for (auto& file_future : file_futures) {
    file_future.wait();
  }
  // concatenate the vectors after thread completes
  for (auto& file_future : file_futures) {
    auto tempHeaderVec = file_future.get();
    headerVec.insert(headerVec.end(), tempHeaderVec.begin(), tempHeaderVec.end());
  }
  file_futures.clear();
}

Here is the caller graph for this function:

FileBuffer * File_Namespace::FileMgr::putBuffer ( const ChunkKey &  key, AbstractBuffer *  d, const size_t  numBytes = 0  )

override

Puts the contents of d into the Chunk with the given key.

Parameters

key	- Unique identifier for a Chunk.
d	- An object representing the source data for the Chunk.

Returns

AbstractBuffer*

Definition at line 816 of file FileMgr.cpp.

References CHECK, CHECK_LT, Data_Namespace::AbstractBuffer::clearDirtyBits(), logger::FATAL, Data_Namespace::AbstractBuffer::getDeviceId(), Data_Namespace::AbstractBuffer::getMemoryPtr(), getOrCreateBuffer(), Data_Namespace::AbstractBuffer::getType(), Data_Namespace::AbstractBuffer::isAppended(), Data_Namespace::AbstractBuffer::isDirty(), Data_Namespace::AbstractBuffer::isUpdated(), LOG, show_chunk(), and Data_Namespace::AbstractBuffer::size().

                                                      {
  auto chunk = getOrCreateBuffer(key);
  size_t oldChunkSize = chunk->size();
  // write the buffer's data to the Chunk
  size_t newChunkSize = (numBytes == 0) ? srcBuffer->size() : numBytes;
  if (chunk->isDirty()) {
    // multiple appends are allowed,
    // but only single update is allowed
    if (srcBuffer->isUpdated() && chunk->isUpdated()) {
      LOG(FATAL) << "Aborting attempt to write a chunk marked dirty. Chunk inconsistency "
                    "for key: "
                 << show_chunk(key);
    }
  }
  CHECK(srcBuffer->isDirty()) << "putBuffer expects a dirty buffer";
  if (srcBuffer->isUpdated()) {
    // chunk size is not changed when fixed rows are updated or are marked as deleted.
    // but when rows are vacuumed or varlen rows are updated (new rows are appended),
    // chunk size will change. For vacuum, checkpoint should sync size from cpu to disk.
    // For varlen update, it takes another route via fragmenter using disk-level buffer.
    if (0 == numBytes && !chunk->isDirty()) {
      chunk->setSize(newChunkSize);
    }
    //@todo use dirty flags to only flush pages of chunk that need to
    // be flushed
    chunk->write((int8_t*)srcBuffer->getMemoryPtr(),
                 newChunkSize,
                 0,
                 srcBuffer->getType(),
                 srcBuffer->getDeviceId());
  } else if (srcBuffer->isAppended()) {
    CHECK_LT(oldChunkSize, newChunkSize);
    chunk->append((int8_t*)srcBuffer->getMemoryPtr() + oldChunkSize,
                  newChunkSize - oldChunkSize,
                  srcBuffer->getType(),
                  srcBuffer->getDeviceId());
  } else {
    // If dirty buffer comes in unmarked, it must be empty.
    // Encoder sync is still required to flush the metadata.
    CHECK(numBytes == 0)
        << "Dirty buffer with size > 0 must be marked as isAppended() or isUpdated()";
  }
  // chunk->clearDirtyBits(); // Hack: because write and append will set dirty bits
  //@todo commenting out line above will make sure this metadata is set
  // but will trigger error on fetch chunk
  srcBuffer->clearDirtyBits();
  chunk->syncEncoder(srcBuffer);
  return chunk;
}

Here is the call graph for this function:

void File_Namespace::FileMgr::readOnlyCheck ( const std::string &  action, const std::optional< std::string > &  file_name = {}  ) const

privatevirtual

Reimplemented in File_Namespace::CachingFileMgr.

Definition at line 1725 of file FileMgr.cpp.

References CHECK, and g_read_only.

Referenced by closeRemovePhysical(), compactFiles(), createEpochFile(), createFile(), createFileInfo(), deleteEmptyFiles(), init(), migrateEpochFileV0(), migrateLegacyFilesV1(), renameCompactionStatusFile(), resumeFileCompaction(), writeAndSyncVersionToDisk(), writeFile(), and writePageMappingsToStatusFile().

                                                                           {
  CHECK(!g_read_only) << "Error trying to " << action
                      << (file_name.has_value() ? (": '" + file_name.value() + "'") : "")
                      << ".  Not allowed in read only mode.";
}

Here is the caller graph for this function:

std::vector< PageMapping > File_Namespace::FileMgr::readPageMappingsFromStatusFile ( )

protected

Deserializes a page mapping vector from expected status file.

Definition at line 1547 of file FileMgr.cpp.

References CHECK, CHECK_EQ, CHECK_GE, heavyai::file_size(), getFilePath(), and UPDATE_PAGE_VISIBILITY_STATUS.

Referenced by resumeFileCompaction().

                                                               {
  auto file_path = getFilePath(UPDATE_PAGE_VISIBILITY_STATUS);
  CHECK(boost::filesystem::exists(file_path));
  std::ifstream status_file{file_path.string(),
                            std::ios::in | std::ios::binary | std::ios::ate};
  CHECK(status_file.is_open());
  size_t file_size = status_file.tellg();
  status_file.seekg(0, std::ios::beg);
  CHECK_GE(file_size, sizeof(int64_t));

  int64_t page_mappings_count;
  status_file.read(reinterpret_cast<char*>(&page_mappings_count), sizeof(int64_t));
  auto page_mappings_byte_size = file_size - sizeof(int64_t);
  CHECK_EQ(page_mappings_byte_size % sizeof(PageMapping), static_cast<size_t>(0));
  CHECK_EQ(static_cast<size_t>(page_mappings_count),
           page_mappings_byte_size / sizeof(PageMapping));

  std::vector<PageMapping> page_mappings(page_mappings_count);
  status_file.read(reinterpret_cast<char*>(page_mappings.data()),
                   page_mappings_byte_size);
  status_file.close();
  return page_mappings;
}

Here is the call graph for this function:

Here is the caller graph for this function:

int32_t File_Namespace::FileMgr::readVersionFromDisk ( const std::string &  versionFileName ) const

protected

Definition at line 1073 of file FileMgr.cpp.

References File_Namespace::close(), heavyai::file_size(), fileMgrBasePath_, INVALID_VERSION, File_Namespace::open(), File_Namespace::read(), and setup::version.

Referenced by createOrMigrateTopLevelMetadata(), migrateToLatestFileMgrVersion(), and writeAndSyncVersionToDisk().

                                                                           {
  const std::string versionFilePath(fileMgrBasePath_ + "/" + versionFileName);
  if (!boost::filesystem::exists(versionFilePath) ||
      !boost::filesystem::is_regular_file(versionFilePath) ||
      boost::filesystem::file_size(versionFilePath) < 4) {
    return INVALID_VERSION;
  }
  FILE* versionFile = open(versionFilePath);
  int32_t version;
  read(versionFile, 0, sizeof(int32_t), (int8_t*)&version, versionFilePath);
  close(versionFile);
  return version;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::removeTableRelatedDS ( const int32_t  db_id, const int32_t  table_id  )

override

Definition at line 1215 of file FileMgr.cpp.

References UNREACHABLE.

                                                                              {
  UNREACHABLE();
}

void File_Namespace::FileMgr::renameAndSymlinkLegacyFiles ( const std::string &  table_data_dir )

static

Definition at line 1136 of file FileMgr.cpp.

References DATA_FILE_EXT, logger::INFO, File_Namespace::kLegacyDataFileExtension, and LOG.

Referenced by migrateLegacyFilesV1(), and anonymous_namespace{TableArchiver.cpp}::rename_table_directories().

                                                                         {
  std::map<boost::filesystem::path, boost::filesystem::path> old_to_new_paths;
  for (boost::filesystem::directory_iterator it(table_data_dir), end_it; it != end_it;
       it++) {
    const auto old_path = boost::filesystem::canonical(it->path());
    if (boost::filesystem::is_regular_file(it->status()) &&
        old_path.extension().string() == kLegacyDataFileExtension) {
      auto new_path = old_path;
      new_path.replace_extension(DATA_FILE_EXT);
      old_to_new_paths[old_path] = new_path;
    }
  }
  for (const auto& [old_path, new_path] : old_to_new_paths) {
    boost::filesystem::rename(old_path, new_path);
    LOG(INFO) << "Rebrand migration: Renamed " << old_path << " to " << new_path;
    boost::filesystem::create_symlink(new_path.filename(), old_path);
    LOG(INFO) << "Rebrand migration: Added symlink from " << old_path << " to "
              << new_path.filename();
  }
}

Here is the caller graph for this function:

void File_Namespace::FileMgr::renameCompactionStatusFile	(	const char *const	from_status,
		const char *const	to_status
	)

Renames a given status file name to a new given file name.

Definition at line 1574 of file FileMgr.cpp.

References CHECK, getFilePath(), and readOnlyCheck().

Referenced by compactFiles(), and resumeFileCompaction().

                                                                      {
  auto from_status_file_path = getFilePath(from_status);
  auto to_status_file_path = getFilePath(to_status);

  readOnlyCheck("write file", from_status_file_path.string());

  CHECK(boost::filesystem::exists(from_status_file_path));
  CHECK(!boost::filesystem::exists(to_status_file_path));
  boost::filesystem::rename(from_status_file_path, to_status_file_path);
}

Here is the call graph for this function:

Here is the caller graph for this function:

Page File_Namespace::FileMgr::requestFreePage ( size_t  pagesize, const bool  isMetadata  )

virtual

Reimplemented in File_Namespace::CachingFileMgr.

Definition at line 877 of file FileMgr.cpp.

References CHECK, createFileInfo(), File_Namespace::FileInfo::fileId, fileIndex_, getFileInfoForFileId(), File_Namespace::FileInfo::getFreePage(), getPageMutex_, num_pages_per_data_file_, and num_pages_per_metadata_file_.

Referenced by File_Namespace::FileBuffer::addNewMultiPage(), init(), File_Namespace::FileBuffer::write(), and File_Namespace::FileBuffer::writeMetadata().

                                                                    {
  std::lock_guard<std::mutex> lock(getPageMutex_);

  auto candidateFiles = fileIndex_.equal_range(pageSize);
  int32_t pageNum = -1;
  for (auto fileIt = candidateFiles.first; fileIt != candidateFiles.second; ++fileIt) {
    FileInfo* fileInfo = getFileInfoForFileId(fileIt->second);
    pageNum = fileInfo->getFreePage();
    if (pageNum != -1) {
      return (Page(fileInfo->fileId, pageNum));
    }
  }
  // if here then we need to add a file
  FileInfo* fileInfo;
  if (isMetadata) {
    fileInfo = createFileInfo(pageSize, num_pages_per_metadata_file_);
  } else {
    fileInfo = createFileInfo(pageSize, num_pages_per_data_file_);
  }
  pageNum = fileInfo->getFreePage();
  CHECK(pageNum != -1);
  return (Page(fileInfo->fileId, pageNum));
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::requestFreePages	(	size_t	npages,
		size_t	pagesize,
		std::vector< Page > &	pages,
		const bool	isMetadata
	)

Obtains free pages – creates new files if necessary – of the requested size.

Given a page size and number of pages, this method updates the vector "pages" to include free pages of the requested size. These pages are immediately removed from the free list of the affected file(s). If there are not enough pages available among current files, new files are created and their pages are included in the vector.

Parameters

npages	The number of free pages requested
pagesize	The size of each requested page
pages	A vector containing the free pages obtained by this method

Definition at line 901 of file FileMgr.cpp.

References CHECK, createFileInfo(), File_Namespace::FileInfo::fileId, fileIndex_, getFileInfoForFileId(), File_Namespace::FileInfo::getFreePage(), getPageMutex_, num_pages_per_data_file_, and num_pages_per_metadata_file_.

                                                      {
  // not used currently
  // @todo add method to FileInfo to get more than one page
  std::lock_guard<std::mutex> lock(getPageMutex_);
  auto candidateFiles = fileIndex_.equal_range(pageSize);
  size_t numPagesNeeded = numPagesRequested;
  for (auto fileIt = candidateFiles.first; fileIt != candidateFiles.second; ++fileIt) {
    FileInfo* fileInfo = getFileInfoForFileId(fileIt->second);
    int32_t pageNum;
    do {
      pageNum = fileInfo->getFreePage();
      if (pageNum != -1) {
        pages.emplace_back(fileInfo->fileId, pageNum);
        numPagesNeeded--;
      }
    } while (pageNum != -1 && numPagesNeeded > 0);
    if (numPagesNeeded == 0) {
      break;
    }
  }
  while (numPagesNeeded > 0) {
    FileInfo* fileInfo;
    if (isMetadata) {
      fileInfo = createFileInfo(pageSize, num_pages_per_metadata_file_);
    } else {
      fileInfo = createFileInfo(pageSize, num_pages_per_data_file_);
    }
    int32_t pageNum;
    do {
      pageNum = fileInfo->getFreePage();
      if (pageNum != -1) {
        pages.emplace_back(fileInfo->fileId, pageNum);
        numPagesNeeded--;
      }
    } while (pageNum != -1 && numPagesNeeded > 0);
    if (numPagesNeeded == 0) {
      break;
    }
  }
  CHECK(pages.size() == numPagesRequested);
}

Here is the call graph for this function:

void File_Namespace::FileMgr::resetEpochFloor ( )

inline

Definition at line 305 of file FileMgr.h.

{ epoch_.floor(epoch_.ceiling()); }

void File_Namespace::FileMgr::resumeFileCompaction ( const std::string &  status_file_name )

protected

Resumes an interrupted file compaction process. This method would normally only be called when re-initializing the file manager after a crash occurred in the middle of file compaction.

Definition at line 1224 of file FileMgr.cpp.

References CHECK, compactFiles(), COPY_PAGES_STATUS, DELETE_EMPTY_FILES_STATUS, deleteEmptyFiles(), files_rw_mutex_, getFilePath(), readOnlyCheck(), readPageMappingsFromStatusFile(), renameCompactionStatusFile(), UNREACHABLE, UPDATE_PAGE_VISIBILITY_STATUS, and updateMappedPagesVisibility().

Referenced by init().

                                                                    {
  readOnlyCheck("resume file compaction", status_file_name);

  if (status_file_name == COPY_PAGES_STATUS) {
    // Delete status file and restart data compaction process
    auto file_path = getFilePath(status_file_name);
    CHECK(boost::filesystem::exists(file_path));
    boost::filesystem::remove(file_path);
    compactFiles();
  } else if (status_file_name == UPDATE_PAGE_VISIBILITY_STATUS) {
    // Execute second and third phases of data compaction
    heavyai::unique_lock<heavyai::shared_mutex> write_lock(files_rw_mutex_);
    auto page_mappings = readPageMappingsFromStatusFile();
    updateMappedPagesVisibility(page_mappings);
    renameCompactionStatusFile(UPDATE_PAGE_VISIBILITY_STATUS, DELETE_EMPTY_FILES_STATUS);
    deleteEmptyFiles();
  } else if (status_file_name == DELETE_EMPTY_FILES_STATUS) {
    // Execute last phase of data compaction
    heavyai::unique_lock<heavyai::shared_mutex> write_lock(files_rw_mutex_);
    deleteEmptyFiles();
  } else {
    UNREACHABLE() << "Unexpected status file name: " << status_file_name;
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::rollOffOldData ( const int32_t  epochCeiling, const bool  shouldCheckpoint  )

private

Definition at line 687 of file FileMgr.cpp.

References checkpoint(), epoch_, Epoch::floor(), freePagesBeforeEpoch(), and maxRollbackEpochs_.

Referenced by checkpoint(), and init().

                                                                                      {
  if (maxRollbackEpochs_ >= 0) {
    auto min_epoch = std::max(epoch_ceiling - maxRollbackEpochs_, epoch_.floor());
    if (min_epoch > epoch_.floor()) {
      freePagesBeforeEpoch(min_epoch);
      epoch_.floor(min_epoch);
      if (should_checkpoint) {
        checkpoint();
      }
    }
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::setDataAndMetadataFileStats ( StorageStats &  storage_stats ) const

private

Definition at line 343 of file FileMgr.cpp.

References CHECK, File_Namespace::StorageStats::data_file_count, File_Namespace::StorageStats::epoch, File_Namespace::StorageStats::epoch_floor, epochFloor(), logger::FATAL, File_Namespace::FileMetadata::file_size, fileMgrBasePath_, files_, files_rw_mutex_, getMetadataForFile(), File_Namespace::FileMetadata::is_data_file, File_Namespace::anonymous_namespace{FileMgr.cpp}::is_metadata_file(), isFullyInitted_, lastCheckpointedEpoch(), LOG, File_Namespace::StorageStats::metadata_file_count, metadata_page_size_, File_Namespace::FileMetadata::num_pages, num_pages_per_metadata_file_, File_Namespace::FileMetadata::page_size, File_Namespace::StorageStats::total_data_file_size, File_Namespace::StorageStats::total_data_page_count, File_Namespace::StorageStats::total_free_data_page_count, File_Namespace::StorageStats::total_free_metadata_page_count, File_Namespace::StorageStats::total_metadata_file_size, and File_Namespace::StorageStats::total_metadata_page_count.

Referenced by getStorageStats().

                                                                           {
  heavyai::shared_lock<heavyai::shared_mutex> read_lock(files_rw_mutex_);
  if (!isFullyInitted_) {
    CHECK(!fileMgrBasePath_.empty());
    boost::filesystem::path path(fileMgrBasePath_);
    if (boost::filesystem::exists(path)) {
      if (!boost::filesystem::is_directory(path)) {
        LOG(FATAL) << "getStorageStats: Specified path '" << fileMgrBasePath_
                   << "' for table data is not a directory.";
      }

      storage_stats.epoch = lastCheckpointedEpoch();
      storage_stats.epoch_floor = epochFloor();
      boost::filesystem::directory_iterator
          endItr;  // default construction yields past-the-end
      for (boost::filesystem::directory_iterator fileIt(path); fileIt != endItr;
           ++fileIt) {
        FileMetadata file_metadata = getMetadataForFile(fileIt);
        if (file_metadata.is_data_file) {
          if (is_metadata_file(file_metadata.file_size,
                               file_metadata.page_size,
                               metadata_page_size_,
                               num_pages_per_metadata_file_)) {
            storage_stats.metadata_file_count++;
            storage_stats.total_metadata_file_size += file_metadata.file_size;
            storage_stats.total_metadata_page_count += file_metadata.num_pages;
          } else {
            storage_stats.data_file_count++;
            storage_stats.total_data_file_size += file_metadata.file_size;
            storage_stats.total_data_page_count += file_metadata.num_pages;
          }
        }
      }
    }
  } else {
    storage_stats.epoch = lastCheckpointedEpoch();
    storage_stats.epoch_floor = epochFloor();
    storage_stats.total_free_metadata_page_count = 0;
    storage_stats.total_free_data_page_count = 0;

    // We already initialized this table so take the faster path of walking through the
    // FileInfo objects and getting metadata from there
    for (const auto& file_info_entry : files_) {
      const auto file_info = file_info_entry.second.get();
      if (is_metadata_file(file_info->size(),
                           file_info->pageSize,
                           metadata_page_size_,
                           num_pages_per_metadata_file_)) {
        storage_stats.metadata_file_count++;
        storage_stats.total_metadata_file_size +=
            file_info->pageSize * file_info->numPages;
        storage_stats.total_metadata_page_count += file_info->numPages;
        storage_stats.total_free_metadata_page_count.value() +=
            file_info->freePages.size();
      } else {
        storage_stats.data_file_count++;
        storage_stats.total_data_file_size += file_info->pageSize * file_info->numPages;
        storage_stats.total_data_page_count += file_info->numPages;
        storage_stats.total_free_data_page_count.value() += file_info->freePages.size();
      }
    }
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::setEpoch ( const int32_t  newEpoch )

protected

Definition at line 1199 of file FileMgr.cpp.

References Epoch::ceiling(), describeSelf(), epoch_, Epoch::floor(), and writeAndSyncEpochToDisk().

Referenced by init().

                                             {
  if (newEpoch < epoch_.floor()) {
    std::stringstream error_message;
    error_message << "Cannot set epoch for " << describeSelf()
                  << " lower than the minimum rollback epoch (" << epoch_.floor() << ").";
    throw std::runtime_error(error_message.str());
  }
  epoch_.ceiling(newEpoch);
  writeAndSyncEpochToDisk();
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::setNumPagesPerDataFile ( size_t  num_pages )

static

Definition at line 1588 of file FileMgr.cpp.

References num_pages_per_data_file_.

                                                     {
  num_pages_per_data_file_ = num_pages;
}

void File_Namespace::FileMgr::setNumPagesPerMetadataFile ( size_t  num_pages )

static

Definition at line 1592 of file FileMgr.cpp.

References num_pages_per_metadata_file_.

                                                         {
  num_pages_per_metadata_file_ = num_pages;
}

void File_Namespace::FileMgr::sortAndCopyFilePagesForCompaction ( size_t  page_size, std::vector< PageMapping > &  page_mappings, std::set< Page > &  touched_pages  )

protected

Sorts all files with the given page size in ascending order of number of free pages. Then copy over pages from files with more free pages to those with less free pages. Leave destination/copied to pages as free when copying. Record copied source and destination pages in page mapping.

Definition at line 1316 of file FileMgr.cpp.

References CHECK, copySourcePageForCompaction(), File_Namespace::Page::fileId, fileIndex_, files_, File_Namespace::FileInfo::freePages, freePages(), and gpu_enabled::sort().

Referenced by compactFiles().

                                                                             {
  std::vector<FileInfo*> sorted_file_infos;
  auto range = fileIndex_.equal_range(page_size);
  for (auto it = range.first; it != range.second; it++) {
    sorted_file_infos.emplace_back(files_.at(it->second).get());
  }
  if (sorted_file_infos.empty()) {
    return;
  }

  // Sort file infos in ascending order of free pages count i.e. from files with
  // the least number of free pages to those with the highest number of free pages.
  std::sort(sorted_file_infos.begin(),
            sorted_file_infos.end(),
            [](const FileInfo* file_1, const FileInfo* file_2) {
              return file_1->freePages.size() < file_2->freePages.size();
            });

  size_t destination_index = 0, source_index = sorted_file_infos.size() - 1;

  // For page copy destinations, skip files without free pages.
  while (destination_index < source_index &&
         sorted_file_infos[destination_index]->freePages.empty()) {
    destination_index++;
  }

  // For page copy sources, skip files with only free pages.
  while (destination_index < source_index &&
         sorted_file_infos[source_index]->freePages.size() ==
             sorted_file_infos[source_index]->numPages) {
    source_index--;
  }

  std::set<size_t> source_used_pages;
  CHECK(destination_index <= source_index);

  // Get the total number of free pages available for compaction
  int64_t total_free_pages{0};
  for (size_t i = destination_index; i <= source_index; i++) {
    total_free_pages += sorted_file_infos[i]->numFreePages();
  }

  while (destination_index < source_index) {
    if (source_used_pages.empty()) {
      // Populate source_used_pages with only used pages in the source file.
      auto source_file_info = sorted_file_infos[source_index];
      auto& free_pages = source_file_info->freePages;
      for (size_t page_num = 0; page_num < source_file_info->numPages; page_num++) {
        if (free_pages.find(page_num) == free_pages.end()) {
          source_used_pages.emplace(page_num);
        }
      }

      // Free pages of current source file will not be copy destinations
      total_free_pages -= source_file_info->numFreePages();
    }

    // Exit early if there are not enough free pages to empty the next file
    if (total_free_pages - static_cast<int64_t>(source_used_pages.size()) < 0) {
      return;
    }

    // Copy pages from source files to destination files
    auto dest_file_info = sorted_file_infos[destination_index];
    while (!source_used_pages.empty() && !dest_file_info->freePages.empty()) {
      // Get next page to copy
      size_t source_page_num = *source_used_pages.begin();
      source_used_pages.erase(source_page_num);

      Page source_page{sorted_file_infos[source_index]->fileId, source_page_num};
      copySourcePageForCompaction(source_page,
                                  sorted_file_infos[destination_index],
                                  page_mappings,
                                  touched_pages);
      total_free_pages--;
    }

    if (source_used_pages.empty()) {
      source_index--;
    }

    if (dest_file_info->freePages.empty()) {
      destination_index++;
    }
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::syncFilesToDisk ( )

protected

Definition at line 1596 of file FileMgr.cpp.

References CHECK, files_, and files_rw_mutex_.

Referenced by checkpoint().

                              {
  heavyai::shared_lock<heavyai::shared_mutex> files_read_lock(files_rw_mutex_);
  for (const auto& file_info_entry : files_) {
    int32_t status = file_info_entry.second->syncToDisk();
    CHECK(status == 0) << "Could not sync file to disk";
  }
}

Here is the caller graph for this function:

void File_Namespace::FileMgr::updateMappedPagesVisibility ( const std::vector< PageMapping > &  page_mappings )

protected

Goes through the given page mapping and marks source/copied from pages as free while marking destination/copied to pages as used (by setting the header size).

Definition at line 1471 of file FileMgr.cpp.

References CHECK_GT, logger::FATAL, files_, getFileInfoForFileId(), and LOG.

Referenced by compactFiles(), and resumeFileCompaction().

                                                                                     {
  for (const auto& page_mapping : page_mappings) {
    auto destination_file = getFileInfoForFileId(page_mapping.destination_file_id);

    // Set destination page header size
    auto header_size = page_mapping.source_page_header_size;
    CHECK_GT(header_size, 0);
    destination_file->write(
        page_mapping.destination_page_num * destination_file->pageSize,
        sizeof(PageHeaderSizeType),
        reinterpret_cast<int8_t*>(&header_size));
    auto source_file = getFileInfoForFileId(page_mapping.source_file_id);

    // Free source page
    PageHeaderSizeType free_page_header_size{0};
    source_file->write(page_mapping.source_page_num * source_file->pageSize,
                       sizeof(PageHeaderSizeType),
                       reinterpret_cast<int8_t*>(&free_page_header_size));
    source_file->freePageDeferred(page_mapping.source_page_num);
  }

  for (const auto& file_info_entry : files_) {
    int32_t status = file_info_entry.second->syncToDisk();
    if (status != 0) {
      LOG(FATAL) << "Could not sync file to disk";
    }
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool File_Namespace::FileMgr::updatePageIfDeleted ( FileInfo *  file_info, ChunkKey &  chunk_key, int32_t  contingent, int32_t  page_epoch, int32_t  page_num  )

virtual

deletes or recovers a page based on last checkpointed epoch.

Reimplemented in File_Namespace::CachingFileMgr.

Definition at line 1637 of file FileMgr.cpp.

References CHUNK_KEY_DB_IDX, CHUNK_KEY_TABLE_IDX, epoch(), File_Namespace::FileInfo::freePageImmediate(), g_multi_instance, g_read_only, get_fileMgrKey(), File_Namespace::is_page_deleted_with_checkpoint(), File_Namespace::is_page_deleted_without_checkpoint(), and File_Namespace::FileInfo::recoverPage().

Referenced by File_Namespace::FileInfo::openExistingFile().

                                                    {
  // If the parent FileMgr has a fileMgrKey, then all keys are locked to one table and
  // can be set from the manager.
  auto [db_id, tb_id] = get_fileMgrKey();
  chunk_key[CHUNK_KEY_DB_IDX] = db_id;
  chunk_key[CHUNK_KEY_TABLE_IDX] = tb_id;

  auto table_epoch = epoch(db_id, tb_id);

  if (is_page_deleted_with_checkpoint(table_epoch, page_epoch, contingent)) {
    if (!g_read_only && !g_multi_instance) {
      // Read-only mode can find pages like this if the server was previously run in
      // write-mode but is not allowed to free them.
      // TODO(sy): Confirm that proper locking is held before writing here.
      file_info->freePageImmediate(page_num);
    }
    return true;
  }

  // Recover page if it was deleted but not checkpointed.
  if (is_page_deleted_without_checkpoint(table_epoch, page_epoch, contingent)) {
    if (!g_read_only && !g_multi_instance) {
      // Read-only mode can find pages like this if the server was previously run in
      // write-mode but is not allowed to free them.
      // TODO(sy): Confirm that proper locking is held before writing here.
      file_info->recoverPage(chunk_key, page_num);
    }
  }
  return false;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::writeAndSyncEpochToDisk ( )

protected

Definition at line 659 of file FileMgr.cpp.

References Epoch::byte_size(), CHECK, epoch_, epochFile_, epochIsCheckpointed_, heavyai::fsync(), Epoch::storage_ptr(), and writeFile().

Referenced by checkpoint(), createEpochFile(), migrateEpochFileV0(), and setEpoch().

                                      {
  CHECK(epochFile_);
  writeFile(epochFile_, 0, Epoch::byte_size(), epoch_.storage_ptr());
  int32_t status = fflush(epochFile_);
  CHECK(status == 0) << "Could not flush epoch file to disk";
#ifdef __APPLE__
  status = fcntl(fileno(epochFile_), 51);
#else
  status = heavyai::fsync(fileno(epochFile_));
#endif
  CHECK(status == 0) << "Could not sync epoch file to disk";
  epochIsCheckpointed_ = true;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::writeAndSyncVersionToDisk ( const std::string &  versionFileName, const int32_t  version  )

protected

Definition at line 1087 of file FileMgr.cpp.

References File_Namespace::close(), File_Namespace::create(), epochFile_, logger::FATAL, fileMgrBasePath_, heavyai::fsync(), logger::INFO, LOG, File_Namespace::open(), readOnlyCheck(), readVersionFromDisk(), and File_Namespace::write().

Referenced by createOrMigrateTopLevelMetadata(), init(), migrateEpochFileV0(), migrateLegacyFilesV1(), and migrateToLatestFileMgrVersion().

                                                               {
  const std::string versionFilePath(fileMgrBasePath_ + "/" + versionFileName);
  readOnlyCheck("write file", versionFilePath);
  FILE* versionFile;
  if (boost::filesystem::exists(versionFilePath)) {
    int32_t oldVersion = readVersionFromDisk(versionFileName);
    LOG(INFO) << "Storage version file `" << versionFilePath
              << "` already exists, its current version is " << oldVersion;
    versionFile = open(versionFilePath);
  } else {
    versionFile = create(versionFilePath, sizeof(int32_t));
  }
  write(versionFile, 0, sizeof(int32_t), (int8_t*)&version);
  int32_t status = fflush(versionFile);
  if (status != 0) {
    LOG(FATAL) << "Could not flush version file " << versionFilePath << " to disk";
  }
#ifdef __APPLE__
  status = fcntl(fileno(epochFile_), 51);
#else
  status = heavyai::fsync(fileno(versionFile));
#endif
  if (status != 0) {
    LOG(FATAL) << "Could not sync version file " << versionFilePath << " to disk";
  }
  close(versionFile);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::writeDirtyBuffers ( )

private

Definition at line 1684 of file FileMgr.cpp.

References chunkIndex_, chunkIndexMutex_, and epoch().

Referenced by checkpoint().

                                {
  heavyai::unique_lock<heavyai::shared_mutex> chunk_index_write_lock(chunkIndexMutex_);
  for (auto [key, buf] : chunkIndex_) {
    if (buf->isDirty()) {
      buf->writeMetadata(epoch());
      buf->clearDirtyBits();
    }
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

size_t File_Namespace::FileMgr::writeFile	(	FILE *	f,
		const size_t	offset,
		const size_t	size,
		const int8_t *	buf
	)		const

Definition at line 1717 of file FileMgr.cpp.

References readOnlyCheck(), and File_Namespace::write().

Referenced by File_Namespace::FileInfo::freePage(), File_Namespace::FileInfo::freePageImmediate(), File_Namespace::FileInfo::initNewFile(), File_Namespace::FileInfo::recoverPage(), File_Namespace::FileInfo::write(), and writeAndSyncEpochToDisk().

                                                   {
  readOnlyCheck("write file");
  return write(f, offset, size, buf);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::FileMgr::writePageMappingsToStatusFile

(

const std::vector< PageMapping > &

page_mappings

)

Serializes a page mapping vector to expected status file. Page mapping vector is serialized in the following format: [{page mapping vector size}, {page mapping vector data bytes ...}]

Definition at line 1528 of file FileMgr.cpp.

References CHECK, COPY_PAGES_STATUS, getFilePath(), and readOnlyCheck().

Referenced by compactFiles().

                                                 {
  auto file_path = getFilePath(COPY_PAGES_STATUS);

  readOnlyCheck("write file", file_path.string());

  CHECK(boost::filesystem::exists(file_path));
  CHECK(boost::filesystem::is_empty(file_path));
  std::ofstream status_file{file_path.string(), std::ios::out | std::ios::binary};
  int64_t page_mappings_count = page_mappings.size();
  status_file.write(reinterpret_cast<const char*>(&page_mappings_count), sizeof(int64_t));
  status_file.write(reinterpret_cast<const char*>(page_mappings.data()),
                    page_mappings_count * sizeof(PageMapping));
  status_file.close();
}

Here is the call graph for this function:

Here is the caller graph for this function:

Friends And Related Function Documentation

friend class GlobalFileMgr

friend

Definition at line 162 of file FileMgr.h.

Member Data Documentation

ChunkKeyToChunkMap File_Namespace::FileMgr::chunkIndex_

Definition at line 330 of file FileMgr.h.

Referenced by createBuffer(), createBufferFromHeaders(), createBufferUnlocked(), deleteBuffer(), deleteBuffersWithPrefix(), deleteBufferUnlocked(), File_Namespace::CachingFileMgr::dump(), freePagesBeforeEpoch(), getBufferUnlocked(), getChunkMetadataVecForKeyPrefix(), File_Namespace::CachingFileMgr::getChunkSpaceReservedByTable(), getFragmentCount(), File_Namespace::CachingFileMgr::getMetadataSpaceReservedByTable(), getNumChunks(), getNumUsedMetadataPagesForChunkKey(), getOrCreateBuffer(), hasChunkMetadataForKeyPrefix(), isBufferOnDevice(), writeDirtyBuffers(), File_Namespace::GlobalFileMgr::writeFileMgrData(), and ~FileMgr().

heavyai::shared_mutex File_Namespace::FileMgr::chunkIndexMutex_

mutableprotected

Definition at line 420 of file FileMgr.h.

Referenced by createBuffer(), createBufferFromHeaders(), deleteBuffer(), deleteBuffersWithPrefix(), freePagesBeforeEpoch(), getBuffer(), getChunkMetadataVecForKeyPrefix(), File_Namespace::CachingFileMgr::getChunkSpaceReservedByTable(), getFragmentCount(), File_Namespace::CachingFileMgr::getMetadataSpaceReservedByTable(), getNumChunks(), getNumUsedMetadataPagesForChunkKey(), getOrCreateBuffer(), hasChunkMetadataForKeyPrefix(), isBufferOnDevice(), and writeDirtyBuffers().

constexpr char const* File_Namespace::FileMgr::COPY_PAGES_STATUS {"pending_data_compaction_0"}

static

Definition at line 388 of file FileMgr.h.

Referenced by compactFiles(), File_Namespace::anonymous_namespace{FileMgr.cpp}::is_compaction_status_file(), resumeFileCompaction(), and writePageMappingsToStatusFile().

constexpr char File_Namespace::FileMgr::DB_META_FILENAME[] = "dbmeta"

static

Definition at line 401 of file FileMgr.h.

Referenced by createOrMigrateTopLevelMetadata().

FILE* File_Namespace::FileMgr::DBMetaFile_ = nullptr

protected

Definition at line 418 of file FileMgr.h.

Referenced by closePhysicalUnlocked(), and ~FileMgr().

constexpr size_t File_Namespace::FileMgr::DEFAULT_NUM_PAGES_PER_DATA_FILE {256}

static

Definition at line 384 of file FileMgr.h.

constexpr size_t File_Namespace::FileMgr::DEFAULT_NUM_PAGES_PER_METADATA_FILE {4096}

static

Definition at line 385 of file FileMgr.h.

Referenced by File_Namespace::CachingFileMgr::getMinimumSize().

constexpr char const* File_Namespace::FileMgr::DELETE_EMPTY_FILES_STATUS {"pending_data_compaction_2"}

static

Definition at line 390 of file FileMgr.h.

Referenced by compactFiles(), deleteEmptyFiles(), File_Namespace::anonymous_namespace{FileMgr.cpp}::is_compaction_status_file(), and resumeFileCompaction().

Epoch File_Namespace::FileMgr::epoch_

private

Definition at line 542 of file FileMgr.h.

Referenced by FileMgr(), init(), migrateEpochFileV0(), openAndReadEpochFile(), openFiles(), rollOffOldData(), setEpoch(), and writeAndSyncEpochToDisk().

constexpr char File_Namespace::FileMgr::EPOCH_FILENAME[] = "epoch_metadata"

static

Definition at line 400 of file FileMgr.h.

Referenced by coreInit(), init(), and migrateEpochFileV0().

FILE* File_Namespace::FileMgr::epochFile_ = nullptr

private

Definition at line 544 of file FileMgr.h.

Referenced by closePhysicalUnlocked(), createEpochFile(), FileMgr(), openAndReadEpochFile(), writeAndSyncEpochToDisk(), writeAndSyncVersionToDisk(), and ~FileMgr().

bool File_Namespace::FileMgr::epochIsCheckpointed_ = true

private

Definition at line 543 of file FileMgr.h.

Referenced by writeAndSyncEpochToDisk().

constexpr char File_Namespace::FileMgr::FILE_MGR_VERSION_FILENAME[] = "filemgr_version"

static

Definition at line 402 of file FileMgr.h.

Referenced by init(), migrateEpochFileV0(), migrateLegacyFilesV1(), and migrateToLatestFileMgrVersion().

PageSizeFileMMap File_Namespace::FileMgr::fileIndex_

protected

Definition at line 414 of file FileMgr.h.

Referenced by clearFileInfos(), createFileInfo(), openExistingFile(), requestFreePage(), requestFreePages(), and sortAndCopyFilePagesForCompaction().

std::string File_Namespace::FileMgr::fileMgrBasePath_

protected

Definition at line 411 of file FileMgr.h.

Referenced by File_Namespace::CachingFileMgr::CachingFileMgr(), coreInit(), createEpochFile(), createFileInfo(), deleteEmptyFiles(), FileMgr(), getFilePath(), init(), migrateEpochFileV0(), migrateLegacyFilesV1(), openAndReadEpochFile(), openAndReadLegacyEpochFile(), openFiles(), File_Namespace::CachingFileMgr::readTableFileMgrs(), readVersionFromDisk(), setDataAndMetadataFileStats(), and writeAndSyncVersionToDisk().

TablePair File_Namespace::FileMgr::fileMgrKey_

private

Global FileMgr.

Definition at line 540 of file FileMgr.h.

Referenced by coreInit(), describeSelf(), and FileMgr().

int32_t File_Namespace::FileMgr::fileMgrVersion_

protected

the index of the next file id

Definition at line 417 of file FileMgr.h.

Referenced by init(), and migrateToLatestFileMgrVersion().

std::map<int32_t, std::unique_ptr<FileInfo> > File_Namespace::FileMgr::files_

protected

The OS file system path containing files related to this FileMgr

Definition at line 413 of file FileMgr.h.

Referenced by clearFileInfos(), closePhysicalUnlocked(), compactFiles(), createFileInfo(), deleteEmptyFiles(), FileMgr(), getFileForFileId(), openExistingFile(), setDataAndMetadataFileStats(), sortAndCopyFilePagesForCompaction(), syncFilesToDisk(), updateMappedPagesVisibility(), and ~FileMgr().

heavyai::shared_mutex File_Namespace::FileMgr::files_rw_mutex_

mutableprotected

Definition at line 421 of file FileMgr.h.

Referenced by File_Namespace::CachingFileMgr::closeRemovePhysical(), closeRemovePhysical(), compactFiles(), coreInit(), createFileInfo(), openExistingFile(), resumeFileCompaction(), setDataAndMetadataFileStats(), and syncFilesToDisk().

std::vector<std::pair<FileInfo*, int32_t> > File_Namespace::FileMgr::free_pages_

protected

Definition at line 424 of file FileMgr.h.

Referenced by free_page(), and freePages().

std::mutex File_Namespace::FileMgr::getPageMutex_

protected

pointer to DB level metadata

Definition at line 419 of file FileMgr.h.

Referenced by requestFreePage(), and requestFreePages().

GlobalFileMgr* File_Namespace::FileMgr::gfm_

private

Definition at line 539 of file FileMgr.h.

Referenced by coreInit(), createOrMigrateTopLevelMetadata(), FileMgr(), getDBConvert(), and init().

constexpr int32_t File_Namespace::FileMgr::INVALID_VERSION = -1

static

Definition at line 403 of file FileMgr.h.

Referenced by createOrMigrateTopLevelMetadata(), migrateToLatestFileMgrVersion(), and readVersionFromDisk().

bool File_Namespace::FileMgr::isFullyInitted_ {false}

protected

Definition at line 425 of file FileMgr.h.

Referenced by getStorageStats(), File_Namespace::CachingFileMgr::init(), init(), and setDataAndMetadataFileStats().

constexpr int32_t File_Namespace::FileMgr::LATEST_FILE_MGR_VERSION = 2

static

Definition at line 404 of file FileMgr.h.

Referenced by init(), and migrateToLatestFileMgrVersion().

constexpr char File_Namespace::FileMgr::LEGACY_EPOCH_FILENAME[] = "epoch"

static

Definition at line 399 of file FileMgr.h.

Referenced by migrateEpochFileV0().

int32_t File_Namespace::FileMgr::maxRollbackEpochs_

protected

Definition at line 410 of file FileMgr.h.

Referenced by File_Namespace::CachingFileMgr::CachingFileMgr(), and rollOffOldData().

const size_t File_Namespace::FileMgr::metadata_page_size_

protected

Definition at line 552 of file FileMgr.h.

Referenced by File_Namespace::CachingFileMgr::getMetadataFileSize(), File_Namespace::CachingFileMgr::getMetadataSpaceReservedByTable(), and setDataAndMetadataFileStats().

heavyai::shared_mutex File_Namespace::FileMgr::mutex_free_page_

mutableprotected

Definition at line 423 of file FileMgr.h.

Referenced by free_page(), and freePages().

unsigned File_Namespace::FileMgr::nextFileId_

protected

number of threads used when loading data

Definition at line 416 of file FileMgr.h.

Referenced by File_Namespace::CachingFileMgr::CachingFileMgr(), createFileInfo(), File_Namespace::CachingFileMgr::init(), and init().

size_t File_Namespace::FileMgr::num_pages_per_data_file_ {DEFAULT_NUM_PAGES_PER_DATA_FILE}

staticprotected

Definition at line 427 of file FileMgr.h.

Referenced by File_Namespace::CachingFileMgr::getDataFileSize(), requestFreePage(), requestFreePages(), and setNumPagesPerDataFile().

size_t File_Namespace::FileMgr::num_pages_per_metadata_file_ {DEFAULT_NUM_PAGES_PER_METADATA_FILE}

staticprotected

Definition at line 428 of file FileMgr.h.

Referenced by File_Namespace::CachingFileMgr::getMetadataFileSize(), requestFreePage(), requestFreePages(), setDataAndMetadataFileStats(), and setNumPagesPerMetadataFile().

size_t File_Namespace::FileMgr::num_reader_threads_

protected

Maps page sizes to FileInfo objects.

Definition at line 415 of file FileMgr.h.

Referenced by initializeNumThreads().

const size_t File_Namespace::FileMgr::page_size_

protected

Definition at line 551 of file FileMgr.h.

Referenced by copyPage(), createBufferUnlocked(), File_Namespace::CachingFileMgr::getChunkSpaceReservedByTable(), File_Namespace::CachingFileMgr::getDataFileSize(), and File_Namespace::CachingFileMgr::getPageSize().

constexpr char const* File_Namespace::FileMgr::UPDATE_PAGE_VISIBILITY_STATUS {"pending_data_compaction_1"}

static

Definition at line 389 of file FileMgr.h.

Referenced by compactFiles(), File_Namespace::anonymous_namespace{FileMgr.cpp}::is_compaction_status_file(), readPageMappingsFromStatusFile(), and resumeFileCompaction().

---

The documentation for this class was generated from the following files:

• /home/jenkins-slave/workspace/core-os-doxygen/DataMgr/FileMgr/FileMgr.h
• /home/jenkins-slave/workspace/core-os-doxygen/DataMgr/FileMgr/FileMgr.cpp