#include <GlobalFileMgr.h>

Inheritance diagram for File_Namespace::GlobalFileMgr:

Collaboration diagram for File_Namespace::GlobalFileMgr:

Public Member Functions
	GlobalFileMgr (const int deviceId, std::string basePath=".", const size_t num_reader_threads=0, const size_t defaultPageSize=2097152)
	Constructor. More...

	~GlobalFileMgr () override
	Destructor. More...

AbstractBuffer *	createBuffer (const ChunkKey &key, size_t pageSize=0, const size_t numBytes=0) override
	Creates a chunk with the specified key and page size. More...

bool	isBufferOnDevice (const ChunkKey &key) override

void	deleteBuffer (const ChunkKey &key, const bool purge=true) override
	Deletes the chunk with the specified key. More...

void	deleteBuffersWithPrefix (const ChunkKey &keyPrefix, const bool purge=true) override

AbstractBuffer *	getBuffer (const ChunkKey &key, const size_t numBytes=0) override
	Returns the a pointer to the chunk with the specified key. More...

void	fetchBuffer (const ChunkKey &key, AbstractBuffer *destBuffer, const size_t numBytes) override

AbstractBuffer *	putBuffer (const ChunkKey &key, AbstractBuffer *d, const size_t numBytes=0) override
	Puts the contents of d into the Chunk with the given key. More...

AbstractBuffer *	alloc (const size_t numBytes) override

void	free (AbstractBuffer *buffer) override

MgrType	getMgrType () override

std::string	getStringMgrType () override

std::string	printSlabs () override

void	clearSlabs () override

size_t	getMaxSize () override

size_t	getInUseSize () override

size_t	getAllocated () override

bool	isAllocationCapped () override

void	init ()

void	getChunkMetadataVec (std::vector< std::pair< ChunkKey, ChunkMetadata >> &chunkMetadataVec) override

void	getChunkMetadataVecForKeyPrefix (std::vector< std::pair< ChunkKey, ChunkMetadata >> &chunkMetadataVec, const ChunkKey &keyPrefix) override

void	checkpoint () override
	Fsyncs data files, writes out epoch and fsyncs that. More...

void	checkpoint (const int db_id, const int tb_id) override

size_t	getNumReaderThreads ()
	Returns number of threads defined by parameter num-reader-threads which should be used during initial load and consequent read of data. More...

size_t	getNumChunks () override

FileMgr *	findFileMgr (const int db_id, const int tb_id, const bool removeFromMap=false)

FileMgr *	getFileMgr (const int db_id, const int tb_id)

FileMgr *	getFileMgr (const ChunkKey &key)

std::string	getBasePath () const

size_t	getDefaultPageSize () const

void	writeFileMgrData (FileMgr *fileMgr=0)

int	getDBVersion () const

bool	getDBConvert () const

void	setDBConvert (bool val)

void	removeTableRelatedDS (const int db_id, const int tb_id)

void	setTableEpoch (const int db_id, const int tb_id, const int start_epoch)

size_t	getTableEpoch (const int db_id, const int tb_id)

Private Attributes
std::string	basePath_

size_t	num_reader_threads_
	The OS file system path containing the files. More...

int	epoch_
	number of threads used when loading data More...

size_t	defaultPageSize_

int	mapd_db_version_
	default page size, used to set FileMgr defaultPageSize_ More...

bool	dbConvert_

std::map< std::pair< int, int > , FileMgr * >	fileMgrs_

mapd_shared_mutex	fileMgrs_mutex_

Detailed Description

Definition at line 47 of file GlobalFileMgr.h.

Constructor & Destructor Documentation

File_Namespace::GlobalFileMgr::GlobalFileMgr	(	const int	deviceId,
		std::string	basePath = `"."`,
		const size_t	num_reader_threads = `0`,
		const size_t	defaultPageSize = `2097152`
	)

Constructor.

Definition at line 40 of file GlobalFileMgr.cpp.

References dbConvert_, init(), and mapd_db_version_.

     : AbstractBufferMgr(deviceId)
     , basePath_(basePath)
     , num_reader_threads_(num_reader_threads)
     , epoch_(-1)
     ,  // set the default epoch for all tables corresponding to the time of
        // last checkpoint
     defaultPageSize_(defaultPageSize) {
   mapd_db_version_ =
       1;  // DS changes triggered by individual FileMgr per table project (release 2.1.0)
   dbConvert_ = false;
   init();
 }

Here is the call graph for this function:

File_Namespace::GlobalFileMgr::~GlobalFileMgr ( )

override

Destructor.

Definition at line 57 of file GlobalFileMgr.cpp.

References fileMgrs_, and fileMgrs_mutex_.

                               {
   mapd_lock_guard<mapd_shared_mutex> fileMgrsMutex(fileMgrs_mutex_);
   for (auto fileMgrsIt = fileMgrs_.begin(); fileMgrsIt != fileMgrs_.end(); ++fileMgrsIt) {
     delete fileMgrsIt->second;
   }
 }

Member Function Documentation

AbstractBuffer* File_Namespace::GlobalFileMgr::alloc ( const size_t numBytes )

inlineoverride

Definition at line 105 of file GlobalFileMgr.h.

References logger::FATAL, and LOG.

                                                         {
     LOG(FATAL) << "Operation not supported";
     return nullptr;  // satisfy return-type warning
   }

void File_Namespace::GlobalFileMgr::checkpoint ( )

override

Fsyncs data files, writes out epoch and fsyncs that.

Definition at line 81 of file GlobalFileMgr.cpp.

References fileMgrs_, and fileMgrs_mutex_.

                                {
   mapd_lock_guard<mapd_shared_mutex> fileMgrsMutex(fileMgrs_mutex_);
   for (auto fileMgrsIt = fileMgrs_.begin(); fileMgrsIt != fileMgrs_.end(); ++fileMgrsIt) {
     fileMgrsIt->second->checkpoint();
   }
 }

void File_Namespace::GlobalFileMgr::checkpoint	(	const int	db_id,
		const int	tb_id
	)

override

Definition at line 88 of file GlobalFileMgr.cpp.

References File_Namespace::FileMgr::checkpoint(), and getFileMgr().

                                                                {
   getFileMgr(db_id, tb_id)->checkpoint();
 }

Here is the call graph for this function:

void File_Namespace::GlobalFileMgr::clearSlabs ( )

inlineoverride

Definition at line 115 of file GlobalFileMgr.h.

115 { /* noop */

116 }

AbstractBuffer* File_Namespace::GlobalFileMgr::createBuffer	(	const ChunkKey &	key,
		size_t	pageSize = `0`,
		const size_t	numBytes = `0`
	)

inlineoverride

Creates a chunk with the specified key and page size.

Definition at line 60 of file GlobalFileMgr.h.

                                                                    {
     return getFileMgr(key)->createBuffer(key, pageSize, numBytes);
   }

void File_Namespace::GlobalFileMgr::deleteBuffer	(	const ChunkKey &	key,
		const bool	purge = `true`
	)

inlineoverride

Deletes the chunk with the specified key.

Definition at line 74 of file GlobalFileMgr.h.

                                                                            {
     return getFileMgr(key)->deleteBuffer(key, purge);
   }

void File_Namespace::GlobalFileMgr::deleteBuffersWithPrefix	(	const ChunkKey &	keyPrefix,
		const bool	purge = `true`
	)

override

Definition at line 105 of file GlobalFileMgr.cpp.

References File_Namespace::FileMgr::deleteBuffersWithPrefix(), and getFileMgr().

                                                                                        {
   /* keyPrefix[0] can be -1 only for gpu or cpu buffers but not for FileMgr.
    * There is no assert here, as GlobalFileMgr is being called with -1 value as well in
    * the same loop with other buffers. So the case of -1 will just be ignored, as nothing
    * needs to be done.
    */
   if (keyPrefix[0] != -1) {
     return getFileMgr(keyPrefix)->deleteBuffersWithPrefix(keyPrefix, purge);
   }
 }

Here is the call graph for this function:

void File_Namespace::GlobalFileMgr::fetchBuffer	(	const ChunkKey &	key,
		AbstractBuffer *	destBuffer,
		const size_t	numBytes
	)

inlineoverride

Definition at line 86 of file GlobalFileMgr.h.

                                                    {
     return getFileMgr(key)->fetchBuffer(key, destBuffer, numBytes);
   }

FileMgr * File_Namespace::GlobalFileMgr::findFileMgr	(	const int	db_id,
		const int	tb_id,
		const bool	removeFromMap = `false`
	)

Definition at line 130 of file GlobalFileMgr.cpp.

References fileMgrs_, and fileMgrs_mutex_.

Referenced by getFileMgr(), removeTableRelatedDS(), and setTableEpoch().

                                                               {
   FileMgr* fm = nullptr;
   const auto file_mgr_key = std::make_pair(db_id, tb_id);
   {
     mapd_lock_guard<mapd_shared_mutex> read_lock(fileMgrs_mutex_);
     auto it = fileMgrs_.find(file_mgr_key);
     if (it != fileMgrs_.end()) {
       fm = it->second;
       if (removeFromMap) {
         fileMgrs_.erase(it);
       }
     }
   }
   return fm;
 }

Here is the caller graph for this function:

void File_Namespace::GlobalFileMgr::free ( AbstractBuffer * buffer )

inlineoverride

Definition at line 110 of file GlobalFileMgr.h.

References logger::FATAL, and LOG.

110 { LOG(FATAL) << "Operation not supported"; }

LOG

#define LOG(tag)

Definition: Logger.h:185

logger::FATAL

Definition: Logger.h:81

size_t File_Namespace::GlobalFileMgr::getAllocated ( )

inlineoverride

Definition at line 119 of file GlobalFileMgr.h.

119 { return 0; }

std::string File_Namespace::GlobalFileMgr::getBasePath ( ) const

inline

Definition at line 153 of file GlobalFileMgr.h.

Referenced by Catalog_Namespace::abs_path(), File_Namespace::FileMgr::FileMgr(), and File_Namespace::FileMgr::init().

153 { return basePath_; }

File_Namespace::GlobalFileMgr::basePath_

std::string basePath_

Definition: GlobalFileMgr.h:167

Here is the caller graph for this function:

AbstractBuffer* File_Namespace::GlobalFileMgr::getBuffer	(	const ChunkKey &	key,
		const size_t	numBytes = `0`
	)

inlineoverride

Returns the a pointer to the chunk with the specified key.

Definition at line 82 of file GlobalFileMgr.h.

                                                                                      {
     return getFileMgr(key)->getBuffer(key, numBytes);
   }

void File_Namespace::GlobalFileMgr::getChunkMetadataVec ( std::vector< std::pair< ChunkKey, ChunkMetadata >> & chunkMetadataVec )

override

Definition at line 116 of file GlobalFileMgr.cpp.

References fileMgrs_, and fileMgrs_mutex_.

                                                                    {
   mapd_shared_lock<mapd_shared_mutex> fileMgrsMutex(fileMgrs_mutex_);
   std::vector<std::pair<ChunkKey, ChunkMetadata>> chunkMetadataVecForFileMgr;
   for (auto fileMgrsIt = fileMgrs_.begin(); fileMgrsIt != fileMgrs_.end(); ++fileMgrsIt) {
     fileMgrsIt->second->getChunkMetadataVec(chunkMetadataVecForFileMgr);
     while (!chunkMetadataVecForFileMgr.empty()) {
       // norair - order of elements is reversed, consider optimising this later if needed
       chunkMetadataVec.push_back(chunkMetadataVecForFileMgr.back());
       chunkMetadataVecForFileMgr.pop_back();
     }
   }
 }

void File_Namespace::GlobalFileMgr::getChunkMetadataVecForKeyPrefix	(	std::vector< std::pair< ChunkKey, ChunkMetadata >> &	chunkMetadataVec,
		const ChunkKey &	keyPrefix
	)

inlineoverride

Definition at line 126 of file GlobalFileMgr.h.

                                           {
     return getFileMgr(keyPrefix)->getChunkMetadataVecForKeyPrefix(chunkMetadataVec,
                                                                   keyPrefix);
   }

bool File_Namespace::GlobalFileMgr::getDBConvert ( ) const

inline

Definition at line 159 of file GlobalFileMgr.h.

Referenced by File_Namespace::FileMgr::getDBConvert().

159 { return dbConvert_; }

File_Namespace::GlobalFileMgr::dbConvert_

bool dbConvert_

Definition: GlobalFileMgr.h:181

Here is the caller graph for this function:

int File_Namespace::GlobalFileMgr::getDBVersion ( ) const

inline

Definition at line 158 of file GlobalFileMgr.h.

Referenced by File_Namespace::FileMgr::getDBVersion().

158 { return mapd_db_version_; }

File_Namespace::GlobalFileMgr::mapd_db_version_

int mapd_db_version_

default page size, used to set FileMgr defaultPageSize_

Definition: GlobalFileMgr.h:175

Here is the caller graph for this function:

size_t File_Namespace::GlobalFileMgr::getDefaultPageSize ( ) const

inline

Definition at line 154 of file GlobalFileMgr.h.

Referenced by Data_Namespace::DataMgr::convertDB().

154 { return defaultPageSize_; }

File_Namespace::GlobalFileMgr::defaultPageSize_

size_t defaultPageSize_

Definition: GlobalFileMgr.h:173

Here is the caller graph for this function:

FileMgr * File_Namespace::GlobalFileMgr::getFileMgr	(	const int	db_id,
		const int	tb_id
	)

Definition at line 148 of file GlobalFileMgr.cpp.

References CHECK(), defaultPageSize_, epoch_, fileMgrs_, fileMgrs_mutex_, findFileMgr(), and num_reader_threads_.

Referenced by checkpoint(), Data_Namespace::DataMgr::createTopLevelMetadata(), deleteBuffersWithPrefix(), Catalog_Namespace::Catalog::getTableDataDirectories(), getTableEpoch(), and File_Namespace::FileMgr::init().

                                                                    {
   { /* check if FileMgr already exists for (db_id, tb_id) */
     FileMgr* fm = findFileMgr(db_id, tb_id);
     if (fm != nullptr) {
       return fm;
     }
   }
 
   { /* create new FileMgr for (db_id, tb_id) */
     const auto file_mgr_key = std::make_pair(db_id, tb_id);
     mapd_lock_guard<mapd_shared_mutex> write_lock(fileMgrs_mutex_);
     auto it = fileMgrs_.find(file_mgr_key);
     if (it != fileMgrs_.end()) {
       return it->second;
     }
     FileMgr* fm =
         new FileMgr(0, this, file_mgr_key, num_reader_threads_, epoch_, defaultPageSize_);
     auto it_ok = fileMgrs_.insert(std::make_pair(file_mgr_key, fm));
     CHECK(it_ok.second);
 
     return fm;
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

FileMgr* File_Namespace::GlobalFileMgr::getFileMgr ( const ChunkKey & key )

inline

Definition at line 152 of file GlobalFileMgr.h.

References getFileMgr().

Referenced by getFileMgr().

152 { return getFileMgr(key[0], key[1]); }

File_Namespace::GlobalFileMgr::getFileMgr

FileMgr * getFileMgr(const int db_id, const int tb_id)

Definition: GlobalFileMgr.cpp:148

Here is the call graph for this function:

Here is the caller graph for this function:

size_t File_Namespace::GlobalFileMgr::getInUseSize ( )

inlineoverride

Definition at line 118 of file GlobalFileMgr.h.

118 { return 0; }

size_t File_Namespace::GlobalFileMgr::getMaxSize ( )

inlineoverride

Definition at line 117 of file GlobalFileMgr.h.

117 { return 0; }

MgrType File_Namespace::GlobalFileMgr::getMgrType ( )

inlineoverride

Definition at line 112 of file GlobalFileMgr.h.

112 { return GLOBAL_FILE_MGR; };

size_t File_Namespace::GlobalFileMgr::getNumChunks ( )

override

Definition at line 92 of file GlobalFileMgr.cpp.

References fileMgrs_, and fileMgrs_mutex_.

                                    {
   {
     mapd_shared_lock<mapd_shared_mutex> fileMgrsMutex(fileMgrs_mutex_);
     size_t num_chunks = 0;
     for (auto fileMgrsIt = fileMgrs_.begin(); fileMgrsIt != fileMgrs_.end();
          ++fileMgrsIt) {
       num_chunks += fileMgrsIt->second->getNumChunks();
     }
 
     return num_chunks;
   }
 }

size_t File_Namespace::GlobalFileMgr::getNumReaderThreads ( )

inline

Returns number of threads defined by parameter num-reader-threads which should be used during initial load and consequent read of data.

Definition at line 144 of file GlobalFileMgr.h.

144 { return num_reader_threads_; }

File_Namespace::GlobalFileMgr::num_reader_threads_

size_t num_reader_threads_

The OS file system path containing the files.

Definition: GlobalFileMgr.h:168

std::string File_Namespace::GlobalFileMgr::getStringMgrType ( )

inlineoverride

Definition at line 113 of file GlobalFileMgr.h.

113 { return ToString(GLOBAL_FILE_MGR); }

size_t File_Namespace::GlobalFileMgr::getTableEpoch	(	const int	db_id,
		const int	tb_id
	)

Definition at line 224 of file GlobalFileMgr.cpp.

References File_Namespace::FileMgr::epoch_, and getFileMgr().

                                                                     {
   FileMgr* fm = getFileMgr(db_id, tb_id);
 
   return fm->epoch_;
 }

Here is the call graph for this function:

void File_Namespace::GlobalFileMgr::init ( )

Definition at line 64 of file GlobalFileMgr.cpp.

References basePath_, logger::FATAL, and LOG.

Referenced by GlobalFileMgr().

                          {
   // check if basePath_ already exists, and if not create one
   boost::filesystem::path path(basePath_);
   if (basePath_.size() > 0 && basePath_[basePath_.size() - 1] != '/') {
     basePath_.push_back('/');
   }
   if (boost::filesystem::exists(path)) {
     if (!boost::filesystem::is_directory(path)) {
       LOG(FATAL) << "Specified path is not a directory.";
     }
   } else {  // data directory does not exist
     if (!boost::filesystem::create_directory(path)) {
       LOG(FATAL) << "Could not create data directory";
     }
   }
 }

Here is the caller graph for this function:

bool File_Namespace::GlobalFileMgr::isAllocationCapped ( )

inlineoverride

Definition at line 120 of file GlobalFileMgr.h.

120 { return false; }

bool File_Namespace::GlobalFileMgr::isBufferOnDevice ( const ChunkKey & key )

inlineoverride

Definition at line 66 of file GlobalFileMgr.h.

                                                       {
     return getFileMgr(key)->isBufferOnDevice(key);
   }

std::string File_Namespace::GlobalFileMgr::printSlabs ( )

inlineoverride

Definition at line 114 of file GlobalFileMgr.h.

114 { return "Not Implemented"; }

AbstractBuffer* File_Namespace::GlobalFileMgr::putBuffer	(	const ChunkKey &	key,
		AbstractBuffer *	d,
		const size_t	numBytes = `0`
	)

inlineoverride

Puts the contents of d into the Chunk with the given key.

Parameters

key	- Unique identifier for a Chunk.
d	- An object representing the source data for the Chunk.

Returns: AbstractBuffer*

Definition at line 98 of file GlobalFileMgr.h.

                                                                 {
     return getFileMgr(key)->putBuffer(key, d, numBytes);
   }

void File_Namespace::GlobalFileMgr::removeTableRelatedDS	(	const int	db_id,
		const int	tb_id
	)

Definition at line 188 of file GlobalFileMgr.cpp.

References File_Namespace::FileMgr::closeRemovePhysical(), and findFileMgr().

                                                                          {
   FileMgr* fm = findFileMgr(db_id, tb_id, true);
   if (fm == nullptr) {
     // fileMgr has not been initialized so there is no need to
     // spend the time initializing
     // inmitialize just enough to have to rename
     const auto file_mgr_key = std::make_pair(db_id, tb_id);
     fm = new FileMgr(0, this, file_mgr_key, true);
   }
   fm->closeRemovePhysical();
   /* remove table related in-memory DS only if directory was removed successfully */
 
   delete fm;
 }

Here is the call graph for this function:

void File_Namespace::GlobalFileMgr::setDBConvert ( bool val )

inline

Definition at line 160 of file GlobalFileMgr.h.

160 { dbConvert_ = val; }

File_Namespace::GlobalFileMgr::dbConvert_

bool dbConvert_

Definition: GlobalFileMgr.h:181

void File_Namespace::GlobalFileMgr::setTableEpoch	(	const int	db_id,
		const int	tb_id,
		const int	start_epoch
	)

Definition at line 203 of file GlobalFileMgr.cpp.

References defaultPageSize_, findFileMgr(), logger::INFO, LOG, num_reader_threads_, and File_Namespace::FileMgr::setEpoch().

                                                          {
   const auto file_mgr_key = std::make_pair(db_id, tb_id);
   // this is where the real rollback of any data ahead of the currently set epoch is
   // performed
   FileMgr* fm = new FileMgr(
       0, this, file_mgr_key, num_reader_threads_, start_epoch, defaultPageSize_);
   fm->setEpoch(start_epoch - 1);
   // remove the dummy one we built
   delete fm;
 
   // see if one exists currently, and remove it
   fm = findFileMgr(db_id, tb_id, true);
 
   if (fm != nullptr) {
     LOG(INFO) << "found and removed fm";
     delete fm;
   }
 }

Here is the call graph for this function:

void File_Namespace::GlobalFileMgr::writeFileMgrData ( FileMgr * fileMgr = 0 )

Definition at line 172 of file GlobalFileMgr.cpp.

References File_Namespace::FileMgr::chunkIndex_, and fileMgrs_.

                       {  // this function is not used, keep it for now for future needs
   for (auto fileMgrIt = fileMgrs_.begin(); fileMgrIt != fileMgrs_.end(); fileMgrIt++) {
     FileMgr* fm = fileMgrIt->second;
     if ((fileMgr != 0) && (fileMgr != fm)) {
       continue;
     }
     for (auto chunkIt = fm->chunkIndex_.begin(); chunkIt != fm->chunkIndex_.end();
          chunkIt++) {
       chunkIt->second->write((int8_t*)chunkIt->second, chunkIt->second->size(), 0);
       // chunkIt->second->write((int8_t*)chunkIt->second, chunkIt->second->size(), 0,
       // CPU_LEVEL, -1);
     }
   }
 }

Member Data Documentation

std::string File_Namespace::GlobalFileMgr::basePath_

private

Definition at line 167 of file GlobalFileMgr.h.

Referenced by init().

bool File_Namespace::GlobalFileMgr::dbConvert_

private

DB version for DataMgr DS and corresponding file buffer read/write code

Definition at line 181 of file GlobalFileMgr.h.

Referenced by GlobalFileMgr().

size_t File_Namespace::GlobalFileMgr::defaultPageSize_

private

Definition at line 173 of file GlobalFileMgr.h.

Referenced by getFileMgr(), and setTableEpoch().

int File_Namespace::GlobalFileMgr::epoch_

private

number of threads used when loading data

Definition at line 169 of file GlobalFileMgr.h.

Referenced by getFileMgr().

std::map<std::pair<int, int>, FileMgr*> File_Namespace::GlobalFileMgr::fileMgrs_

private

true if conversion should be done between different "mapd_db_version_"

Definition at line 183 of file GlobalFileMgr.h.

Referenced by checkpoint(), findFileMgr(), getChunkMetadataVec(), getFileMgr(), getNumChunks(), writeFileMgrData(), and ~GlobalFileMgr().

mapd_shared_mutex File_Namespace::GlobalFileMgr::fileMgrs_mutex_

private

Definition at line 184 of file GlobalFileMgr.h.

Referenced by checkpoint(), findFileMgr(), getChunkMetadataVec(), getFileMgr(), getNumChunks(), and ~GlobalFileMgr().

int File_Namespace::GlobalFileMgr::mapd_db_version_

private

default page size, used to set FileMgr defaultPageSize_

Definition at line 175 of file GlobalFileMgr.h.

Referenced by GlobalFileMgr().

size_t File_Namespace::GlobalFileMgr::num_reader_threads_

private

The OS file system path containing the files.

Definition at line 168 of file GlobalFileMgr.h.

Referenced by getFileMgr(), and setTableEpoch().

The documentation for this class was generated from the following files:

/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/FileMgr/GlobalFileMgr.h
/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/FileMgr/GlobalFileMgr.cpp

Public Member Functions

Private Attributes

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation