24 #include <boost/filesystem.hpp>
28 namespace foreign_storage {
33 template <
typename Func,
typename T>
38 upper_prefix.push_back(std::numeric_limits<int>::max());
39 auto end_it = chunk_collection.upper_bound(static_cast<const ChunkKey>(upper_prefix));
40 for (
auto chunk_it = chunk_collection.lower_bound(chunk_prefix); chunk_it != end_it;
66 const size_t num_bytes) {
80 if ((*buf)->hasDataPages()) {
92 auto index_chunk_key = chunk_key;
95 if (index_buffer && (*index_buffer)->hasDataPages() && (*buf)->size() == 0) {
107 return (*buf)->hasEncoder();
114 if (metadata_vec.empty()) {
117 auto first_chunk_key = metadata_vec.begin()->first;
118 for (
auto& [chunk_key, metadata] : metadata_vec) {
132 bool chunk_in_cache =
false;
136 if (!index_chunk_key.empty()) {
144 if (!index_chunk_key.empty()) {
153 const std::shared_ptr<ChunkMetadata> buf_metadata =
154 std::make_shared<ChunkMetadata>();
156 chunk_in_cache = *metadata.get() == *buf_metadata;
160 if (!chunk_in_cache) {
164 if (!index_chunk_key.empty()) {
177 const ChunkKey& chunk_prefix)
const {
182 const ChunkKey& chunk_prefix)
const {
203 const ChunkKey& chunk_prefix)
const {
225 boost::filesystem::path path(base_path);
226 if (boost::filesystem::exists(path)) {
227 if (!boost::filesystem::is_directory(path)) {
228 throw std::runtime_error{
229 "cache path \"" + base_path +
230 "\" is not a directory. Please specify a valid directory "
231 "with --disk_cache_path=<path>, or use the default location."};
234 if (!boost::filesystem::create_directory(path)) {
235 throw std::runtime_error{
236 "could not create directory at cache path \"" + base_path +
237 "\". Please specify a valid directory location "
238 "with --disk_cache_path=<path> or use the default location."};
244 const std::set<ChunkKey>& keys)
const {
246 for (
const auto& key : keys) {
251 CHECK(!file_buf->hasDataPages());
254 file_buf->resetToEmpty();
256 return chunk_buffer_map;
261 bool is_new_buffer) {
262 if (!is_new_buffer) {
278 int32_t table_id)
const {
std::vector< int > ChunkKey
bool isMetadataCached(const ChunkKey &) const
bool is_table_key(const ChunkKey &key)
heavyai::shared_lock< heavyai::shared_mutex > read_lock
bool is_varlen_data_key(const ChunkKey &key)
void clearForTablePrefix(const ChunkKey &)
void storeDataWrapper(const std::string &doc, int32_t db_id, int32_t tb_id)
#define CHUNK_KEY_FRAGMENT_IDX
std::map< ChunkKey, AbstractBuffer * > ChunkToBufferMap
heavyai::unique_lock< heavyai::shared_mutex > write_lock
This file includes the class specification for the cache used by the Foreign Storage Interface (FSI)...
Represents/provides access to contiguous data stored in the file system.
void initEncoder(const SQLTypeInfo &tmp_sql_type)
void setNumElems(const size_t num_elems)
virtual bool resetChunkStats(const ChunkStats &)
: Reset chunk level stats (min, max, nulls) using new values from the argument.
virtual void getMetadata(const std::shared_ptr< ChunkMetadata > &chunkMetadata)
std::shared_lock< T > shared_lock
void getCachedMetadataVecForKeyPrefix(ChunkMetadataVector &, const ChunkKey &) const
bool hasStoredDataWrapperMetadata(int32_t db_id, int32_t table_id) const
std::unique_lock< T > unique_lock
#define CHUNK_KEY_TABLE_IDX
void iterate_over_matching_prefix(Func func, T &chunk_collection, const ChunkKey &chunk_prefix)
An AbstractBuffer is a unit of data management for a data manager.
std::string dumpEvictionQueue() const
ChunkToBufferMap getChunkBuffersForCaching(const std::set< ChunkKey > &chunk_keys) const
void putBuffer(const ChunkKey &, AbstractBuffer *, const size_t numBytes=0)
void cacheMetadataVec(const ChunkMetadataVector &)
std::set< ChunkKey >::iterator eraseChunk(const std::set< ChunkKey >::iterator &)
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
std::string dumpCachedMetadataEntries() const
void deleteBufferIfExists(const ChunkKey &chunk_key)
void set_metadata_for_buffer(AbstractBuffer *buffer, ChunkMetadata *meta)
void validatePath(const std::string &) const
#define CHUNK_KEY_VARLEN_IDX
std::string dumpCachedChunkEntries() const
std::vector< ChunkKey > getCachedChunksForKeyPrefix(const ChunkKey &) const
AbstractBuffer * getChunkBufferForPrecaching(const ChunkKey &chunk_key, bool is_new_buffer)
void setSize(const size_t size)
void checkpoint(const int32_t db_id, const int32_t tb_id)
Encoder * getEncoder() const
#define DEBUG_TIMER(name)
File_Namespace::FileBuffer * getCachedChunkIfExists(const ChunkKey &)
bool hasCachedMetadataForKeyPrefix(const ChunkKey &) const
ForeignStorageCache(const File_Namespace::DiskCacheConfig &config)
#define CHUNK_KEY_COLUMN_IDX
bool in_same_table(const ChunkKey &left_key, const ChunkKey &right_key)
A selection of helper methods for File I/O.
bool is_varlen_key(const ChunkKey &key)