OmniSciDB  bf83d84833
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
foreign_storage::anonymous_namespace{ParquetDataWrapper.cpp} Namespace Reference

Functions

template<typename T >
std::pair< typename std::map
< ChunkKey, T >::iterator,
typename std::map< ChunkKey, T >
::iterator > 
prefix_range (std::map< ChunkKey, T > &map, const ChunkKey &chunk_key_prefix)
 
void reduce_metadata (std::shared_ptr< ChunkMetadata > reduce_to, std::shared_ptr< ChunkMetadata > reduce_from)
 

Function Documentation

template<typename T >
std::pair<typename std::map<ChunkKey, T>::iterator, typename std::map<ChunkKey, T>::iterator> foreign_storage::anonymous_namespace{ParquetDataWrapper.cpp}::prefix_range ( std::map< ChunkKey, T > &  map,
const ChunkKey chunk_key_prefix 
)

Definition at line 37 of file ParquetDataWrapper.cpp.

37  {
38  ChunkKey chunk_key_prefix_sentinel = chunk_key_prefix;
39  chunk_key_prefix_sentinel.push_back(std::numeric_limits<int>::max());
40  auto begin = map.lower_bound(chunk_key_prefix);
41  auto end = map.upper_bound(chunk_key_prefix_sentinel);
42  return std::make_pair(begin, end);
43 }
std::vector< int > ChunkKey
Definition: types.h:37
void foreign_storage::anonymous_namespace{ParquetDataWrapper.cpp}::reduce_metadata ( std::shared_ptr< ChunkMetadata reduce_to,
std::shared_ptr< ChunkMetadata reduce_from 
)

Definition at line 45 of file ParquetDataWrapper.cpp.

References CHECK, Data_Namespace::AbstractBuffer::getEncoder(), Data_Namespace::AbstractBuffer::initEncoder(), and Encoder::resetChunkStats().

Referenced by foreign_storage::ParquetDataWrapper::metadataScanFiles().

46  {
47  CHECK(reduce_to->sqlType == reduce_from->sqlType);
48  reduce_to->numBytes += reduce_from->numBytes;
49  reduce_to->numElements += reduce_from->numElements;
50  reduce_to->chunkStats.has_nulls |= reduce_from->chunkStats.has_nulls;
51 
52  auto column_type = reduce_to->sqlType;
53  column_type = column_type.is_array() ? column_type.get_elem_type() : column_type;
54 
55  // metadata reducution is done at metadata scan time, both string & geometry
56  // columns have no valid stats to reduce beyond `has_nulls`
57  if (column_type.is_string() || column_type.is_geometry()) {
58  return;
59  }
60 
61  ForeignStorageBuffer buffer_to;
62  buffer_to.initEncoder(column_type);
63  auto encoder_to = buffer_to.getEncoder();
64  encoder_to->resetChunkStats(reduce_to->chunkStats);
65 
66  ForeignStorageBuffer buffer_from;
67  buffer_from.initEncoder(column_type);
68  auto encoder_from = buffer_from.getEncoder();
69  encoder_from->resetChunkStats(reduce_from->chunkStats);
70 
71  encoder_to->reduceStats(*encoder_from);
72  auto updated_metadata = std::make_shared<ChunkMetadata>();
73  encoder_to->getMetadata(updated_metadata);
74  reduce_to->chunkStats = updated_metadata->chunkStats;
75 }
void initEncoder(const SQLTypeInfo &tmp_sql_type)
virtual bool resetChunkStats(const ChunkStats &)
: Reset chunk level stats (min, max, nulls) using new values from the argument.
Definition: Encoder.h:223
#define CHECK(condition)
Definition: Logger.h:197

+ Here is the call graph for this function:

+ Here is the caller graph for this function: