#include <FileInfo.h>

Collaboration diagram for File_Namespace::FileInfo:

Public Member Functions
	FileInfo (FileMgr fileMgr, const int32_t fileId, FILE f, const size_t pageSize, const size_t numPages, const bool init=false)
	Constructor. More...

	~FileInfo ()
	Destructor. More...

void	initNewFile ()
	Adds all pages to freePages and zeroes first four bytes of header. More...

void	freePageDeferred (int32_t pageId)

void	freePage (int32_t pageId, const bool isRolloff)

int32_t	getFreePage ()

size_t	write (const size_t offset, const size_t size, int8_t *buf)

size_t	read (const size_t offset, const size_t size, int8_t *buf)

void	openExistingFile (std::vector< HeaderInfo > &headerVec, const int32_t fileMgrEpoch)

void	print (bool pagesummary)
	Prints a summary of the file to stdout. More...

size_t	size () const
	Returns the number of bytes used by the file. More...

int32_t	syncToDisk ()

size_t	available ()
	Returns the number of free bytes available. More...

size_t	numFreePages ()
	Returns the number of free pages available. More...

size_t	used ()
	Returns the amount of used bytes; size() - available() More...

Public Attributes
FileMgr *	fileMgr

int32_t	fileId

FILE *	f
	unique file identifier (i.e., used for a file name) More...

size_t	pageSize
	file stream object for the represented file More...

size_t	numPages
	the fixed size of each page in the file More...

bool	isDirty
	the number of pages in the file More...

std::set< size_t >	freePages

std::mutex	freePagesMutex_
	set of page numbers of free pages More...

std::mutex	readWriteMutex_

Detailed Description

Definition at line 55 of file FileInfo.h.

Constructor & Destructor Documentation

◆ FileInfo()

File_Namespace::FileInfo::FileInfo	(	FileMgr *	fileMgr,
		const int32_t	fileId,
		FILE *	f,
		const size_t	pageSize,
		const size_t	numPages,
		const bool	init = `false`
	)

Constructor.

Definition at line 28 of file FileInfo.cpp.

References initNewFile().

     : fileMgr(fileMgr), fileId(fileId), f(f), pageSize(pageSize), numPages(numPages) {
   if (init) {
     initNewFile();
   }
 }

Here is the call graph for this function:

◆ ~FileInfo()

File_Namespace::FileInfo::~FileInfo ( )

Destructor.

Definition at line 40 of file FileInfo.cpp.

References File_Namespace::close(), and f.

                     {
   // close file, if applicable
   if (f) {
     close(f);
   }
 }

Here is the call graph for this function:

Member Function Documentation

◆ available()

size_t File_Namespace::FileInfo::available ( )

inline

Returns the number of free bytes available.

Definition at line 99 of file FileInfo.h.

References pageSize.

Referenced by print(), and used().

99 { return freePages.size() * pageSize; }

File_Namespace::FileInfo::freePages

std::set< size_t > freePages

Definition: FileInfo.h:62

File_Namespace::FileInfo::pageSize

size_t pageSize

file stream object for the represented file

Definition: FileInfo.h:59

Here is the caller graph for this function:

◆ freePage()

void File_Namespace::FileInfo::freePage	(	int32_t	pageId,
		const bool	isRolloff
	)

Definition at line 199 of file FileInfo.cpp.

References CHECK, File_Namespace::DELETE_CONTINGENT, File_Namespace::FileMgr::epoch(), f, fileMgr, File_Namespace::FileMgr::free_page(), freePages, freePagesMutex_, isDirty, pageSize, readWriteMutex_, File_Namespace::ROLLOFF_CONTINGENT, and File_Namespace::write().

Referenced by File_Namespace::FileBuffer::freePage().

                                                         {
   std::lock_guard<std::mutex> lock(readWriteMutex_);
 #define RESILIENT_PAGE_HEADER
 #ifdef RESILIENT_PAGE_HEADER
   int epoch_freed_page[2] = {DELETE_CONTINGENT, fileMgr->epoch()};
   if (isRolloff) {
     epoch_freed_page[0] = ROLLOFF_CONTINGENT;
   }
   File_Namespace::write(f,
                         pageId * pageSize + sizeof(int32_t),
                         sizeof(epoch_freed_page),
                         (int8_t*)epoch_freed_page);
   fileMgr->free_page(std::make_pair(this, pageId));
 #else
   int32_t zeroVal = 0;
   int8_t* zeroAddr = reinterpret_cast<int8_t*>(&zeroVal);
   File_Namespace::write(f, pageId * pageSize, sizeof(int32_t), zeroAddr);
   std::lock_guard<std::mutex> lock(freePagesMutex_);
   freePages.insert(pageId);
 #endif  // RESILIENT_PAGE_HEADER
   isDirty = true;
 
 #ifdef ENABLE_CRASH_CORRUPTION_TEST
   signal(SIGUSR2, sighandler);
   if (goto_crash)
     CHECK(pageId % 8 != 4);
 #endif
 }

Here is the call graph for this function:

Here is the caller graph for this function:

◆ freePageDeferred()

void File_Namespace::FileInfo::freePageDeferred ( int32_t pageId )

Definition at line 184 of file FileInfo.cpp.

References freePages, and freePagesMutex_.

                                               {
   std::lock_guard<std::mutex> lock(freePagesMutex_);
   freePages.insert(pageId);
 }

◆ getFreePage()

int32_t File_Namespace::FileInfo::getFreePage ( )

Definition at line 228 of file FileInfo.cpp.

References freePages, and freePagesMutex_.

Referenced by File_Namespace::FileMgr::requestFreePage(), and File_Namespace::FileMgr::requestFreePages().

                               {
   // returns -1 if there is no free page
   std::lock_guard<std::mutex> lock(freePagesMutex_);
   if (freePages.size() == 0) {
     return -1;
   }
   auto pageIt = freePages.begin();
   int32_t pageNum = *pageIt;
   freePages.erase(pageIt);
   return pageNum;
 }

Here is the caller graph for this function:

◆ initNewFile()

void File_Namespace::FileInfo::initNewFile ( )

Adds all pages to freePages and zeroes first four bytes of header.

Definition at line 47 of file FileInfo.cpp.

References f, freePages, isDirty, numPages, pageSize, and File_Namespace::write().

Referenced by FileInfo().

                            {
   // initialize pages and free page list
   // Also zeroes out first four bytes of every header
 
   int32_t headerSize = 0;
   int8_t* headerSizePtr = (int8_t*)(&headerSize);
   for (size_t pageId = 0; pageId < numPages; ++pageId) {
     File_Namespace::write(f, pageId * pageSize, sizeof(int32_t), headerSizePtr);
     freePages.insert(pageId);
   }
   isDirty = true;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

◆ numFreePages()

size_t File_Namespace::FileInfo::numFreePages ( )

inline

Returns the number of free pages available.

Definition at line 102 of file FileInfo.h.

                                {
     std::lock_guard<std::mutex> lock(freePagesMutex_);
     return freePages.size();
   }

◆ openExistingFile()

void File_Namespace::FileInfo::openExistingFile	(	std::vector< HeaderInfo > &	headerVec,
		const int32_t	fileMgrEpoch
	)

Definition at line 71 of file FileInfo.cpp.

References CHECK_EQ, CHECK_GE, File_Namespace::DELETE_CONTINGENT, f, fileId, fileMgr, freePages, File_Namespace::FileMgr::get_fileMgrKey(), LOG, numPages, pageSize, File_Namespace::ROLLOFF_CONTINGENT, show_chunk(), VLOG, logger::WARNING, and File_Namespace::write().

Referenced by File_Namespace::FileMgr::openExistingFile().

                                                             {
   // HeaderInfo is defined in Page.h
 
   // Oct 2020: Changing semantics such that fileMgrEpoch should be last checkpointed
   // epoch, not incremented epoch. This changes some of the gt/gte/lt/lte comparison below
   ChunkKey oldChunkKey(4);
   int32_t oldPageId = -99;
   int32_t oldVersionEpoch = -99;
   int32_t skipped = 0;
   for (size_t pageNum = 0; pageNum < numPages; ++pageNum) {
     int32_t headerSize;
 
     constexpr size_t MAX_INTS_TO_READ{10};  // currently use 1+6 ints
     int32_t ints[MAX_INTS_TO_READ];
     CHECK_EQ(fseek(f, pageNum * pageSize, SEEK_SET), 0);
     CHECK_EQ(fread(ints, sizeof(int32_t), MAX_INTS_TO_READ, f), MAX_INTS_TO_READ);
 
     headerSize = ints[0];
     const bool should_delete_deleted =
         DELETE_CONTINGENT == ints[1] && fileMgrEpoch >= ints[2];
     const bool should_delete_rolled_off =
         ROLLOFF_CONTINGENT == ints[1] && fileMgrEpoch >= ints[2];
     if (should_delete_deleted || should_delete_rolled_off) {
       int32_t zero{0};
       File_Namespace::write(f, pageNum * pageSize, sizeof(int32_t), (int8_t*)&zero);
       headerSize = 0;
     }
 
     if (headerSize != 0) {
       // headerSize doesn't include headerSize itself
       // We're tying ourself to headers of ints here
       size_t numHeaderElems = headerSize / sizeof(int32_t);
       CHECK_GE(numHeaderElems, size_t(2));
       // We don't want to read headerSize in our header - so start
       // reading 4 bytes past it
 
       // always derive dbid/tbid from FileMgr
       ChunkKey chunkKey(&ints[1], &ints[1 + numHeaderElems - 2]);
       chunkKey[0] = fileMgr->get_fileMgrKey().first;
       chunkKey[1] = fileMgr->get_fileMgrKey().second;
       // recover page in case a crash failed deletion of this page
       if (ints[1] == DELETE_CONTINGENT || ints[1] == ROLLOFF_CONTINGENT) {
         File_Namespace::write(f,
                               pageNum * pageSize + sizeof(int32_t),
                               2 * sizeof(int32_t),
                               (int8_t*)&chunkKey[0]);
       }
 
       // Last two elements of header are always PageId and Version
       // epoch - these are not in the chunk key so seperate them
       int32_t pageId = ints[1 + numHeaderElems - 2];
       int32_t versionEpoch = ints[1 + numHeaderElems - 1];
       if (chunkKey != oldChunkKey || oldPageId != pageId - (1 + skipped)) {
         if (skipped > 0) {
           VLOG(4) << "FId.PSz: " << fileId << "." << pageSize
                   << " Chunk key: " << show_chunk(oldChunkKey)
                   << " Page id from : " << oldPageId << " to : " << oldPageId + skipped
                   << " Epoch: " << oldVersionEpoch;
         } else if (oldPageId != -99) {
           VLOG(4) << "FId.PSz: " << fileId << "." << pageSize
                   << " Chunk key: " << show_chunk(oldChunkKey)
                   << " Page id: " << oldPageId << " Epoch: " << oldVersionEpoch;
         }
         oldPageId = pageId;
         oldVersionEpoch = versionEpoch;
         oldChunkKey = chunkKey;
         skipped = 0;
       } else {
         skipped++;
       }
 
       /* Check if version epoch is equal to
        * or greater (note: should never be greater)
        * than FileMgr epoch_ - this means that this
        * page wasn't checkpointed and thus we should
        * not use it
        */
       if (versionEpoch > fileMgrEpoch) {
         // First write 0 to first four bytes of
         // header to mark as free
         headerSize = 0;
         File_Namespace::write(
             f, pageNum * pageSize, sizeof(int32_t), (int8_t*)&headerSize);
         // Now add page to free list
         freePages.insert(pageNum);
         LOG(WARNING) << "Was not checkpointed: Chunk key: " << show_chunk(chunkKey)
                      << " Page id: " << pageId << " Epoch: " << versionEpoch
                      << " FileMgrEpoch " << fileMgrEpoch << endl;
 
       } else {  // page was checkpointed properly
         Page page(fileId, pageNum);
         headerVec.emplace_back(chunkKey, pageId, versionEpoch, page);
       }
     } else {  // no header for this page - insert into free list
       freePages.insert(pageNum);
     }
   }
   // printlast
   if (oldPageId != -99) {
     if (skipped > 0) {
       VLOG(4) << "FId.PSz: " << fileId << "." << pageSize
               << " Chunk key: " << show_chunk(oldChunkKey)
               << " Page id from : " << oldPageId << " to : " << oldPageId + skipped
               << " Epoch: " << oldVersionEpoch;
     } else {
       VLOG(4) << "FId.PSz: " << fileId << "." << pageSize
               << " Chunk key: " << show_chunk(oldChunkKey) << " Page id: " << oldPageId
               << " Epoch: " << oldVersionEpoch;
     }
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

◆ print()

void File_Namespace::FileInfo::print ( bool pagesummary )

Prints a summary of the file to stdout.

Definition at line 240 of file FileInfo.cpp.

References available(), fileId, size(), and used().

                                      {
   std::cout << "File: " << fileId << std::endl;
   std::cout << "Size: " << size() << std::endl;
   std::cout << "Used: " << used() << std::endl;
   std::cout << "Free: " << available() << std::endl;
   if (!pagesummary) {
     return;
   }
 }

Here is the call graph for this function:

◆ read()

size_t File_Namespace::FileInfo::read	(	const size_t	offset,
		const size_t	size,
		int8_t *	buf
	)

Definition at line 66 of file FileInfo.cpp.

References f, File_Namespace::read(), and readWriteMutex_.

Referenced by File_Namespace::FileBuffer::copyPage(), File_Namespace::FileMgr::copyPage(), and File_Namespace::readForThread().

                                                                          {
   std::lock_guard<std::mutex> lock(readWriteMutex_);
   return File_Namespace::read(f, offset, size, buf);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

◆ size()

size_t File_Namespace::FileInfo::size ( ) const

inline

Returns the number of bytes used by the file.

Definition at line 92 of file FileInfo.h.

References numPages, and syncToDisk().

Referenced by print(), and used().

92 { return pageSize * numPages; }

File_Namespace::FileInfo::pageSize

size_t pageSize

file stream object for the represented file

Definition: FileInfo.h:59

File_Namespace::FileInfo::numPages

size_t numPages

the fixed size of each page in the file

Definition: FileInfo.h:60

Here is the call graph for this function:

Here is the caller graph for this function:

◆ syncToDisk()

int32_t File_Namespace::FileInfo::syncToDisk ( )

Syncs file to disk via a buffer flush and then a sync (fflush and fsync on posix systems)

Definition at line 249 of file FileInfo.cpp.

References f, logger::FATAL, omnisci::fsync(), isDirty, LOG, and readWriteMutex_.

Referenced by size().

                              {
   std::lock_guard<std::mutex> lock(readWriteMutex_);
   if (isDirty) {
     if (fflush(f) != 0) {
       LOG(FATAL) << "Error trying to flush changes to disk, the error was: "
                  << std::strerror(errno);
     }
 #ifdef __APPLE__
     const int32_t sync_result = fcntl(fileno(f), 51);
 #else
     const int32_t sync_result = omnisci::fsync(fileno(f));
 #endif
     if (sync_result == 0) {
       isDirty = false;
     }
     return sync_result;
   }
   return 0;  // if file was not dirty and no syncing was needed
 }

Here is the call graph for this function:

Here is the caller graph for this function:

◆ used()

size_t File_Namespace::FileInfo::used ( )

inline

Returns the amount of used bytes; size() - available()

Definition at line 108 of file FileInfo.h.

References available(), and size().

Referenced by print().

108 { return size() - available(); }

File_Namespace::FileInfo::size

size_t size() const

Returns the number of bytes used by the file.

Definition: FileInfo.h:92

File_Namespace::FileInfo::available

size_t available()

Returns the number of free bytes available.

Definition: FileInfo.h:99

Here is the call graph for this function:

Here is the caller graph for this function:

◆ write()

size_t File_Namespace::FileInfo::write	(	const size_t	offset,
		const size_t	size,
		int8_t *	buf
	)

Definition at line 60 of file FileInfo.cpp.

References f, isDirty, readWriteMutex_, and File_Namespace::write().

Referenced by File_Namespace::FileBuffer::append(), File_Namespace::FileBuffer::copyPage(), File_Namespace::FileMgr::copyPage(), File_Namespace::FileBuffer::write(), and File_Namespace::FileBuffer::writeHeader().

                                                                           {
   std::lock_guard<std::mutex> lock(readWriteMutex_);
   isDirty = true;
   return File_Namespace::write(f, offset, size, buf);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

◆ f

FILE* File_Namespace::FileInfo::f

unique file identifier (i.e., used for a file name)

Definition at line 58 of file FileInfo.h.

Referenced by freePage(), initNewFile(), openExistingFile(), read(), syncToDisk(), write(), and ~FileInfo().

◆ fileId

int32_t File_Namespace::FileInfo::fileId

Definition at line 57 of file FileInfo.h.

Referenced by openExistingFile(), print(), File_Namespace::FileMgr::requestFreePage(), and File_Namespace::FileMgr::requestFreePages().

◆ fileMgr

FileMgr* File_Namespace::FileInfo::fileMgr

Definition at line 56 of file FileInfo.h.

Referenced by freePage(), and openExistingFile().

◆ freePages

std::set<size_t> File_Namespace::FileInfo::freePages

Definition at line 62 of file FileInfo.h.

Referenced by freePage(), freePageDeferred(), getFreePage(), initNewFile(), and openExistingFile().

◆ freePagesMutex_

std::mutex File_Namespace::FileInfo::freePagesMutex_

set of page numbers of free pages

Definition at line 63 of file FileInfo.h.

Referenced by freePage(), freePageDeferred(), and getFreePage().

◆ isDirty

bool File_Namespace::FileInfo::isDirty

the number of pages in the file

Definition at line 61 of file FileInfo.h.

Referenced by freePage(), initNewFile(), syncToDisk(), and write().

◆ numPages

size_t File_Namespace::FileInfo::numPages

the fixed size of each page in the file

Definition at line 60 of file FileInfo.h.

Referenced by initNewFile(), openExistingFile(), and size().

◆ pageSize

size_t File_Namespace::FileInfo::pageSize

file stream object for the represented file

Definition at line 59 of file FileInfo.h.

Referenced by available(), freePage(), initNewFile(), and openExistingFile().

◆ readWriteMutex_

std::mutex File_Namespace::FileInfo::readWriteMutex_

Definition at line 64 of file FileInfo.h.

Referenced by freePage(), read(), syncToDisk(), and write().

The documentation for this struct was generated from the following files:

/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/FileMgr/FileInfo.h
/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/FileMgr/FileInfo.cpp

Public Member Functions

Public Attributes

Detailed Description

Constructor & Destructor Documentation

◆ FileInfo()

◆ ~FileInfo()

Member Function Documentation

◆ available()

◆ freePage()

◆ freePageDeferred()

◆ getFreePage()

◆ initNewFile()

◆ numFreePages()

◆ openExistingFile()

◆ print()

◆ read()

◆ size()

◆ syncToDisk()

◆ used()

◆ write()

Member Data Documentation

◆ f

◆ fileId

◆ fileMgr

◆ freePages

◆ freePagesMutex_

◆ isDirty

◆ numPages

◆ pageSize

◆ readWriteMutex_