27 namespace foreign_storage {
28 using read_lock = mapd_shared_lock<mapd_shared_mutex>;
32 template <
typename Func,
typename T>
37 upper_prefix.push_back(std::numeric_limits<int>::max());
38 auto end_it = chunk_collection.upper_bound(static_cast<const ChunkKey>(upper_prefix));
39 for (
auto chunk_it = chunk_collection.lower_bound(chunk_prefix); chunk_it != end_it;
55 : num_chunks_added_(0), num_metadata_added_(0) {
88 CHECK(!chunk_keys.empty());
92 const ChunkKey table_key{db_id, table_id};
94 for (
const auto& chunk_key : chunk_keys) {
124 CHECK(meta_vec.size() == 0);
128 for (
auto& [chunk_key, metadata] : meta_vec) {
132 if (
const auto& buf =
caching_file_mgr_->getBuffer(chunk_key); buf->pageCount() > 0) {
146 buf->pageCount() > 0) {
151 return (meta_vec.size() > 0);
168 for (
auto& [chunk_key, metadata] : metadata_vec) {
182 bool chunk_in_cache =
false;
186 if (!index_chunk_key.empty()) {
194 if (!index_chunk_key.empty()) {
203 const std::shared_ptr<ChunkMetadata> buf_metadata =
204 std::make_shared<ChunkMetadata>();
206 chunk_in_cache = *metadata.get() == *buf_metadata;
210 if (!chunk_in_cache) {
214 if (!index_chunk_key.empty()) {
227 const ChunkKey& chunk_prefix)
const {
231 [&metadata_vec,
this](
auto chunk) {
232 std::shared_ptr<ChunkMetadata> buf_metadata = std::make_shared<ChunkMetadata>();
234 metadata_vec.push_back(std::make_pair(chunk, buf_metadata));
241 const ChunkKey& chunk_prefix)
const {
245 ChunkKey upper_prefix(chunk_prefix);
246 upper_prefix.push_back(std::numeric_limits<int>::max());
247 auto end_it =
cached_metadata_.upper_bound(static_cast<const ChunkKey>(upper_prefix));
248 for (
auto meta_it =
cached_metadata_.lower_bound(chunk_prefix); meta_it != end_it;
258 ChunkKey upper_prefix(chunk_prefix);
259 upper_prefix.push_back(std::numeric_limits<int>::max());
263 auto end_it =
cached_chunks_.upper_bound(static_cast<const ChunkKey>(upper_prefix));
264 for (
auto chunk_it =
cached_chunks_.lower_bound(chunk_prefix); chunk_it != end_it;) {
271 auto end_it =
cached_metadata_.upper_bound(static_cast<const ChunkKey>(upper_prefix));
272 for (
auto meta_it =
cached_metadata_.lower_bound(chunk_prefix); meta_it != end_it;) {
281 std::set<ChunkKey> table_keys;
291 table_keys.emplace(
ChunkKey{(*meta_it)[0], (*meta_it)[1]});
300 caching_file_mgr_->getFileMgrBasePath(), caching_file_mgr_->getNumReaderThreads());
304 const ChunkKey& chunk_prefix)
const {
306 std::vector<ChunkKey> ret_vec;
308 [&ret_vec](
auto chunk) { ret_vec.push_back(chunk); },
cached_chunks_, chunk_prefix);
313 const std::vector<ChunkKey>& chunk_keys)
const {
316 for (
const auto& chunk_key : chunk_keys) {
320 CHECK(dynamic_cast<File_Namespace::FileBuffer*>(chunk_buffer_map[chunk_key]));
321 CHECK_EQ(chunk_buffer_map[chunk_key]->pageCount(), static_cast<size_t>(0));
324 chunk_buffer_map[chunk_key]->resetToEmpty();
326 return chunk_buffer_map;
342 const std::set<ChunkKey>::iterator& chunk_it) {
350 std::string ret_string =
"Cached chunks:\n";
352 ret_string +=
" " +
show_chunk(chunk_key) +
"\n";
358 std::string ret_string =
"Cached ChunkMetadata:\n";
360 ret_string +=
" " +
show_chunk(meta_key) +
"\n";
367 boost::filesystem::path path(base_path);
368 if (boost::filesystem::exists(path)) {
369 if (!boost::filesystem::is_directory(path)) {
370 throw std::runtime_error{
371 "cache path \"" + base_path +
372 "\" is not a directory. Please specify a valid directory "
373 "with --disk_cache_path=<path>, or use the default location."};
376 if (!boost::filesystem::create_directory(path)) {
377 throw std::runtime_error{
378 "could not create directory at cache path \"" + base_path +
379 "\". Please specify a valid directory location "
380 "with --disk_cache_path=<path> or use the default location."};
390 for (
const auto& chunk_metadata : metadata_vec) {
392 new_metadata_vec.push_back(chunk_metadata);
400 bool is_new_buffer) {
401 if (!is_new_buffer) {
std::vector< int > ChunkKey
bool isMetadataCached(const ChunkKey &) const
std::set< ChunkKey > cached_chunks_
std::set< ChunkKey >::iterator evictChunkByIterator(const std::set< ChunkKey >::iterator &chunk_it)
bool is_table_key(const ChunkKey &key)
bool is_varlen_data_key(const ChunkKey &key)
void clearForTablePrefix(const ChunkKey &)
#define CHUNK_KEY_FRAGMENT_IDX
std::map< ChunkKey, AbstractBuffer * > ChunkToBufferMap
void cacheTableChunks(const std::vector< ChunkKey > &chunk_keys)
Represents/provides access to contiguous data stored in the file system.
void initEncoder(const SQLTypeInfo &tmp_sql_type)
void setNumElems(const size_t num_elems)
ChunkKey get_table_key(const ChunkKey &key)
std::string show_chunk(const ChunkKey &key)
virtual bool resetChunkStats(const ChunkStats &)
: Reset chunk level stats (min, max, nulls) using new values from the argument.
virtual void getMetadata(const std::shared_ptr< ChunkMetadata > &chunkMetadata)
ForeignStorageCache(const DiskCacheConfig &config)
AbstractBuffer * getCachedChunkIfExists(const ChunkKey &)
void getCachedMetadataVecForKeyPrefix(ChunkMetadataVector &, const ChunkKey &) const
#define CHUNK_KEY_TABLE_IDX
void iterate_over_matching_prefix(Func func, T &chunk_collection, const ChunkKey &chunk_prefix)
An AbstractBuffer is a unit of data management for a data manager.
void cacheMetadataWithFragIdGreaterOrEqualTo(const ChunkMetadataVector &metadata_vec, const int frag_id)
bool recoverCacheForTable(ChunkMetadataVector &, const ChunkKey &)
void cacheMetadataVec(const ChunkMetadataVector &)
std::set< ChunkKey >::iterator eraseChunk(const std::set< ChunkKey >::iterator &)
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
mapd_shared_mutex metadata_mutex_
void evictThenEraseChunkUnlocked(const ChunkKey &)
std::string dumpCachedMetadataEntries() const
void deleteBufferIfExists(const ChunkKey &chunk_key)
void set_metadata_for_buffer(AbstractBuffer *buffer, ChunkMetadata *meta)
void evictThenEraseChunk(const ChunkKey &)
void validatePath(const std::string &) const
std::string dumpCachedChunkEntries() const
std::vector< ChunkKey > getCachedChunksForKeyPrefix(const ChunkKey &) const
AbstractBuffer * getChunkBufferForPrecaching(const ChunkKey &chunk_key, bool is_new_buffer)
void setSize(const size_t size)
size_t num_reader_threads
mapd_shared_lock< mapd_shared_mutex > read_lock
mapd_shared_mutex chunks_mutex_
Encoder * getEncoder() const
#define DEBUG_TIMER(name)
bool hasCachedMetadataForKeyPrefix(const ChunkKey &) const
ChunkToBufferMap getChunkBuffersForCaching(const std::vector< ChunkKey > &chunk_keys) const
mapd_unique_lock< mapd_shared_mutex > write_lock
#define CHUNK_KEY_COLUMN_IDX
std::set< ChunkKey > cached_metadata_
size_t num_metadata_added_
A selection of helper methods for File I/O.
void cacheChunk(const ChunkKey &, AbstractBuffer *)
bool is_varlen_key(const ChunkKey &key)