GlobalFileMgr.h

Go to the documentation of this file.

/*
 * Copyright 2017 MapD Technologies, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef DATAMGR_MEMORY_FILE_GLOBAL_FILEMGR_H
#define DATAMGR_MEMORY_FILE_GLOBAL_FILEMGR_H

#include <iostream>
#include <map>
#include <mutex>
#include <set>
#include "../Shared/mapd_shared_mutex.h"

#include "../AbstractBuffer.h"
#include "../AbstractBufferMgr.h"
#include "FileMgr.h"

using namespace Data_Namespace;

namespace File_Namespace {

class GlobalFileMgr : public AbstractBufferMgr {  // implements

 public:
  GlobalFileMgr(const int deviceId,
                std::string basePath = ".",
                const size_t num_reader_threads = 0,
                const size_t defaultPageSize = 2097152);

  ~GlobalFileMgr() override;

  AbstractBuffer* createBuffer(const ChunkKey& key,
                               size_t pageSize = 0,
                               const size_t numBytes = 0) override {
    return getFileMgr(key)->createBuffer(key, pageSize, numBytes);
  }

  bool isBufferOnDevice(const ChunkKey& key) override {
    return getFileMgr(key)->isBufferOnDevice(key);
  }

  // Purge == true means delete the data chunks -
  // can't undelete and revert to previous
  // state - reclaims disk space for chunk
  void deleteBuffer(const ChunkKey& key, const bool purge = true) override {
    return getFileMgr(key)->deleteBuffer(key, purge);
  }

  void deleteBuffersWithPrefix(const ChunkKey& keyPrefix,
                               const bool purge = true) override;

  AbstractBuffer* getBuffer(const ChunkKey& key, const size_t numBytes = 0) override {
    return getFileMgr(key)->getBuffer(key, numBytes);
  }

  void fetchBuffer(const ChunkKey& key,
                   AbstractBuffer* destBuffer,
                   const size_t numBytes) override {
    return getFileMgr(key)->fetchBuffer(key, destBuffer, numBytes);
  }

  AbstractBuffer* putBuffer(const ChunkKey& key,
                            AbstractBuffer* d,
                            const size_t numBytes = 0) override {
    return getFileMgr(key)->putBuffer(key, d, numBytes);
  }

  // Buffer API
  AbstractBuffer* alloc(const size_t numBytes) override {
    LOG(FATAL) << "Operation not supported";
    return nullptr;  // satisfy return-type warning
  }

  void free(AbstractBuffer* buffer) override { LOG(FATAL) << "Operation not supported"; }

  inline MgrType getMgrType() override { return GLOBAL_FILE_MGR; };
  inline std::string getStringMgrType() override { return ToString(GLOBAL_FILE_MGR); }
  inline std::string printSlabs() override { return "Not Implemented"; }
  inline void clearSlabs() override { /* noop */
  }
  inline size_t getMaxSize() override { return 0; }
  inline size_t getInUseSize() override { return 0; }
  inline size_t getAllocated() override { return 0; }
  inline bool isAllocationCapped() override { return false; }

  void init();

  void getChunkMetadataVec(
      std::vector<std::pair<ChunkKey, ChunkMetadata>>& chunkMetadataVec) override;
  void getChunkMetadataVecForKeyPrefix(
      std::vector<std::pair<ChunkKey, ChunkMetadata>>& chunkMetadataVec,
      const ChunkKey& keyPrefix) override {
    return getFileMgr(keyPrefix)->getChunkMetadataVecForKeyPrefix(chunkMetadataVec,
                                                                  keyPrefix);
  }

  void checkpoint() override;
  void checkpoint(const int db_id, const int tb_id) override;

  inline size_t getNumReaderThreads() { return num_reader_threads_; }

  size_t getNumChunks() override;

  FileMgr* findFileMgr(const int db_id,
                       const int tb_id,
                       const bool removeFromMap = false);
  FileMgr* getFileMgr(const int db_id, const int tb_id);
  FileMgr* getFileMgr(const ChunkKey& key) { return getFileMgr(key[0], key[1]); }
  std::string getBasePath() const { return basePath_; }
  size_t getDefaultPageSize() const { return defaultPageSize_; }

  void writeFileMgrData(FileMgr* fileMgr = 0);

  inline int getDBVersion() const { return mapd_db_version_; }
  inline bool getDBConvert() const { return dbConvert_; }
  inline void setDBConvert(bool val) { dbConvert_ = val; }

  void removeTableRelatedDS(const int db_id, const int tb_id);
  void setTableEpoch(const int db_id, const int tb_id, const int start_epoch);
  size_t getTableEpoch(const int db_id, const int tb_id);

 private:
  std::string basePath_;       
  size_t num_reader_threads_;  
  int epoch_; /* the current epoch (time of last checkpoint) will be used for all
               * tables except of the one for which the value of the epoch has been reset
               * using --start-epoch option at start up to rollback this table's updates.
               */
  size_t defaultPageSize_;  
  // bool isDirty_;               /// true if metadata changed since last writeState()
  int mapd_db_version_;  
  /* In future mapd_db_version_ may be added to AbstractBufferMgr class.
   * This will allow support of different dbVersions for different tables, so
   * original tables can be generated by different versions of mapd software.
   */
  bool dbConvert_;  
  std::map<std::pair<int, int>, FileMgr*> fileMgrs_;
  mapd_shared_mutex fileMgrs_mutex_;
};

}  // namespace File_Namespace

#endif  // DATAMGR_MEMORY_FILE_GLOBAL_FILEMGR_H