OmniSciDB  bf83d84833
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
foreign_storage::anonymous_namespace{CsvDataWrapper.cpp} Namespace Reference

Functions

std::set< const
ColumnDescriptor * > 
get_columns (const std::map< ChunkKey, AbstractBuffer * > &buffers, std::shared_ptr< Catalog_Namespace::Catalog > catalog, const int32_t table_id, const int fragment_id)
 
bool skip_metadata_scan (const ColumnDescriptor *column)
 
foreign_storage::ForeignStorageCacheget_cache_if_enabled (std::shared_ptr< Catalog_Namespace::Catalog > &catalog)
 
void add_placeholder_metadata (const ColumnDescriptor *column, const ForeignTable *foreign_table, const int db_id, const size_t start_row, const size_t total_num_rows, std::map< ChunkKey, std::shared_ptr< ChunkMetadata >> &chunk_metadata_map)
 

Function Documentation

void foreign_storage::anonymous_namespace{CsvDataWrapper.cpp}::add_placeholder_metadata ( const ColumnDescriptor column,
const ForeignTable *  foreign_table,
const int  db_id,
const size_t  start_row,
const size_t  total_num_rows,
std::map< ChunkKey, std::shared_ptr< ChunkMetadata >> &  chunk_metadata_map 
)

Definition at line 911 of file CsvDataWrapper.cpp.

References CHUNK_KEY_FRAGMENT_IDX, ColumnDescriptor::columnId, ColumnDescriptor::columnType, Data_Namespace::AbstractBuffer::initEncoder(), SQLTypeInfo::is_varlen_indeed(), TableDescriptor::maxFragRows, and TableDescriptor::tableId.

Referenced by foreign_storage::CsvDataWrapper::populateChunkMetadata().

917  {
918  ChunkKey chunk_key = {db_id, foreign_table->tableId, column->columnId, 0};
919  if (column->columnType.is_varlen_indeed()) {
920  chunk_key.emplace_back(1);
921  }
922 
923  // Create placeholder metadata for every fragment touched by this scan
924  int start_fragment = start_row / foreign_table->maxFragRows;
925  int end_fragment = total_num_rows / foreign_table->maxFragRows;
926  for (int fragment_id = start_fragment; fragment_id <= end_fragment; fragment_id++) {
927  size_t num_elements = (static_cast<size_t>(foreign_table->maxFragRows *
928  (fragment_id + 1)) > total_num_rows)
929  ? total_num_rows % foreign_table->maxFragRows
930  : foreign_table->maxFragRows;
931 
932  ForeignStorageBuffer empty_buffer;
933  // Use default encoder metadata as in parquet wrapper
934  empty_buffer.initEncoder(column->columnType);
935  auto chunk_metadata = empty_buffer.getEncoder()->getMetadata(column->columnType);
936  chunk_metadata->numElements = num_elements;
937  // signal to query engine populate, not set by default for arrays
938  chunk_metadata->chunkStats.min.intval = std::numeric_limits<int32_t>::max();
939  chunk_metadata->chunkStats.max.intval = std::numeric_limits<int32_t>::lowest();
940 
941  chunk_key[CHUNK_KEY_FRAGMENT_IDX] = fragment_id;
942  chunk_metadata_map[chunk_key] = chunk_metadata;
943  }
944 }
std::vector< int > ChunkKey
Definition: types.h:37
#define CHUNK_KEY_FRAGMENT_IDX
Definition: types.h:42
void initEncoder(const SQLTypeInfo &tmp_sql_type)
SQLTypeInfo columnType
bool is_varlen_indeed() const
Definition: sqltypes.h:506

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

foreign_storage::ForeignStorageCache* foreign_storage::anonymous_namespace{CsvDataWrapper.cpp}::get_cache_if_enabled ( std::shared_ptr< Catalog_Namespace::Catalog > &  catalog)

Definition at line 585 of file CsvDataWrapper.cpp.

Referenced by foreign_storage::cache_blocks(), and foreign_storage::CsvDataWrapper::populateChunkMetadata().

586  {
587  if (catalog->getDataMgr()
588  .getPersistentStorageMgr()
589  ->getDiskCacheConfig()
590  .isEnabledForFSI()) {
591  return catalog->getDataMgr().getPersistentStorageMgr()->getDiskCache();
592  } else {
593  return nullptr;
594  }
595 }

+ Here is the caller graph for this function:

std::set<const ColumnDescriptor*> foreign_storage::anonymous_namespace{CsvDataWrapper.cpp}::get_columns ( const std::map< ChunkKey, AbstractBuffer * > &  buffers,
std::shared_ptr< Catalog_Namespace::Catalog catalog,
const int32_t  table_id,
const int  fragment_id 
)

Definition at line 145 of file CsvDataWrapper.cpp.

References CHECK, CHECK_EQ, CHUNK_KEY_COLUMN_IDX, and CHUNK_KEY_FRAGMENT_IDX.

Referenced by foreign_storage::CsvDataWrapper::populateChunkBuffers().

149  {
150  CHECK(!buffers.empty());
151  std::set<const ColumnDescriptor*> columns;
152  for (const auto& entry : buffers) {
153  CHECK_EQ(fragment_id, entry.first[CHUNK_KEY_FRAGMENT_IDX]);
154  const auto column_id = entry.first[CHUNK_KEY_COLUMN_IDX];
155  const auto column = catalog->getMetadataForColumnUnlocked(table_id, column_id);
156  columns.emplace(column);
157  }
158  return columns;
159 }
#define CHECK_EQ(x, y)
Definition: Logger.h:205
#define CHUNK_KEY_FRAGMENT_IDX
Definition: types.h:42
#define CHECK(condition)
Definition: Logger.h:197
#define CHUNK_KEY_COLUMN_IDX
Definition: types.h:41

+ Here is the caller graph for this function:

bool foreign_storage::anonymous_namespace{CsvDataWrapper.cpp}::skip_metadata_scan ( const ColumnDescriptor column)

Definition at line 163 of file CsvDataWrapper.cpp.

References ColumnDescriptor::columnType, and SQLTypeInfo::is_dict_encoded_type().

Referenced by foreign_storage::CsvDataWrapper::populateChunkMetadata(), and foreign_storage::CsvDataWrapper::updateMetadata().

163  {
164  return column->columnType.is_dict_encoded_type();
165 }
bool is_dict_encoded_type() const
Definition: sqltypes.h:516
SQLTypeInfo columnType

+ Here is the call graph for this function:

+ Here is the caller graph for this function: