#include <NoneEncoder.h>

Inheritance diagram for NoneEncoder< T >:

Collaboration diagram for NoneEncoder< T >:

Public Member Functions
	NoneEncoder (Data_Namespace::AbstractBuffer *buffer)

size_t	getNumElemsForBytesEncodedDataAtIndices (const int8_t *index_data, const std::vector< size_t > &selected_idx, const size_t byte_limit) override

std::shared_ptr< ChunkMetadata >	appendEncodedDataAtIndices (const int8_t , int8_t data, const std::vector< size_t > &selected_idx) override

std::shared_ptr< ChunkMetadata >	appendEncodedData (const int8_t , int8_t data, const size_t start_idx, const size_t num_elements) override

std::shared_ptr< ChunkMetadata >	appendData (int8_t *&src_data, const size_t num_elems_to_append, const SQLTypeInfo &, const bool replicating=false, const int64_t offset=-1) override

void	getMetadata (const std::shared_ptr< ChunkMetadata > &chunkMetadata) override

std::shared_ptr< ChunkMetadata >	getMetadata (const SQLTypeInfo &ti) override

void	updateStats (const int64_t val, const bool is_null) override

void	updateStats (const double val, const bool is_null) override

void	updateStats (const int8_t *const src_data, const size_t num_elements) override

void	updateStatsEncoded (const int8_t *const dst_data, const size_t num_elements) override

void	updateStats (const std::vector< std::string > *const src_data, const size_t start_idx, const size_t num_elements) override

void	updateStats (const std::vector< ArrayDatum > *const src_data, const size_t start_idx, const size_t num_elements) override

void	reduceStats (const Encoder &that) override

void	writeMetadata (FILE *f) override

void	readMetadata (FILE *f) override

bool	resetChunkStats (const ChunkStats &stats) override
	: Reset chunk level stats (min, max, nulls) using new values from the argument. More...

void	copyMetadata (const Encoder *copyFromEncoder) override

void	resetChunkStats () override

Public Member Functions inherited from Encoder
	Encoder (Data_Namespace::AbstractBuffer *buffer)

virtual	~Encoder ()

size_t	getNumElems () const

void	setNumElems (const size_t num_elems)

Public Attributes
T	dataMin

T	dataMax

bool	has_nulls

Private Member Functions
std::shared_ptr< ChunkMetadata >	appendValidatedOrNonValidatedData (int8_t *&src_data, const size_t num_elems_to_append, const bool replicating, const int64_t offset, const bool is_validated_data)

T	validateDataAndUpdateStats (const T &unencoded_data, const bool is_validated_data=false)

void	updateStats (const int8_t *const src_data, const size_t num_elements, const bool is_validated_data)

Additional Inherited Members
Static Public Member Functions inherited from Encoder
static Encoder *	Create (Data_Namespace::AbstractBuffer *buffer, const SQLTypeInfo sqlType)

Protected Attributes inherited from Encoder
size_t	num_elems_

Data_Namespace::AbstractBuffer *	buffer_

DecimalOverflowValidator	decimal_overflow_validator_

DateDaysOverflowValidator	date_days_overflow_validator_

Detailed Description

template<typename T>
class NoneEncoder< T >

Definition at line 37 of file NoneEncoder.h.

Constructor & Destructor Documentation

template<typename T>

NoneEncoder< T >::NoneEncoder ( Data_Namespace::AbstractBuffer * buffer )

inline

Definition at line 39 of file NoneEncoder.h.

References NoneEncoder< T >::resetChunkStats().

                                                     : Encoder(buffer) {
     resetChunkStats();
   }

Here is the call graph for this function:

Member Function Documentation

template<typename T>

std::shared_ptr<ChunkMetadata> NoneEncoder< T >::appendData	(	int8_t *&	src_data,
		const size_t	num_elems_to_append,
		const SQLTypeInfo &	ti,
		const bool	replicating = `false`,
		const int64_t	offset = `-1`
	)

inlineoverridevirtual

Append data to the chunk buffer backing this encoder.

Parameters

src_data	Source data for the append
num_elems_to_append	Number of elements to append
ti	SQL Type Info for the column TODO(adb): used?
replicating	Pass one value and fill the chunk with it
offset	Write data starting at a given offset. Default is -1 which indicates an append, an offset of 0 rewrites the chunk up to `num_elems_to_append`.

Implements Encoder.

Definition at line 78 of file NoneEncoder.h.

References NoneEncoder< T >::appendValidatedOrNonValidatedData().

Referenced by NoneEncoder< T >::appendEncodedDataAtIndices().

                                                                                 {
     return appendValidatedOrNonValidatedData(
         src_data, num_elems_to_append, replicating, offset, false);
   }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T>

std::shared_ptr<ChunkMetadata> NoneEncoder< T >::appendEncodedData	(	const int8_t *	index_data,
		int8_t *	data,
		const size_t	start_idx,
		const size_t	num_elements
	)

inlineoverridevirtual

Append encoded data to the chunk buffer backing this encoder.

Parameters

index_data	- (optional) the index data of data to append
data	- the data to append
start_idx	- the position to start encoding from in the `data` array
num_elements	- the number of elements to encode from the `data` array

Returns: updated chunk metadata for the chunk buffer backing this encoder

NOTE: index_data must be non-null for varlen encoder types.

Implements Encoder.

Definition at line 70 of file NoneEncoder.h.

References NoneEncoder< T >::appendValidatedOrNonValidatedData(), and heavydb.dtypes::T.

                                                                                        {
     auto current_data = data + sizeof(T) * start_idx;
     return appendValidatedOrNonValidatedData(current_data, num_elements, false, -1, true);
   }

Here is the call graph for this function:

template<typename T>

std::shared_ptr<ChunkMetadata> NoneEncoder< T >::appendEncodedDataAtIndices	(	const int8_t *	index_data,
		int8_t *	data,
		const std::vector< size_t > &	selected_idx
	)

inlineoverridevirtual

Append selected encoded data to the chunk buffer backing this encoder.

Parameters

index_data	- (optional) the index data of data to append
data	- the data to append
selected_idx	- which indices in the encoded data to append

Returns: updated chunk metadata for the chunk buffer backing this encoder

NOTE: index_data must be non-null for varlen encoder types.

Implements Encoder.

Definition at line 52 of file NoneEncoder.h.

References NoneEncoder< T >::appendData(), shared::execute_over_contiguous_indices(), and heavydb.dtypes::T.

                                                       {
     std::shared_ptr<ChunkMetadata> chunk_metadata;
     // NOTE: the use of `execute_over_contiguous_indices` is an optimization;
     // it prevents having to copy or move the indexed data and instead performs
     // an append over contiguous sections of indices.
     shared::execute_over_contiguous_indices(
         selected_idx, [&](const size_t start_pos, const size_t end_pos) {
           size_t elem_count = end_pos - start_pos;
           auto data_ptr = data + sizeof(T) * selected_idx[start_pos];
           chunk_metadata = appendData(data_ptr, elem_count, SQLTypeInfo{}, false);
         });
 
     return chunk_metadata;
   }

Here is the call graph for this function:

template<typename T>

std::shared_ptr<ChunkMetadata> NoneEncoder< T >::appendValidatedOrNonValidatedData	(	int8_t *&	src_data,
		const size_t	num_elems_to_append,
		const bool	replicating,
		const int64_t	offset,
		const bool	is_validated_data
	)

inlineprivate

Definition at line 225 of file NoneEncoder.h.

References Data_Namespace::AbstractBuffer::append(), Encoder::buffer_, CHECK, CHECK_GE, gpu_enabled::fill(), NoneEncoder< T >::getMetadata(), Encoder::num_elems_, Data_Namespace::AbstractBuffer::reserve(), NoneEncoder< T >::resetChunkStats(), Data_Namespace::AbstractBuffer::size(), heavydb.dtypes::T, NoneEncoder< T >::updateStats(), NoneEncoder< T >::validateDataAndUpdateStats(), and Data_Namespace::AbstractBuffer::write().

Referenced by NoneEncoder< T >::appendData(), and NoneEncoder< T >::appendEncodedData().

                                     {
     if (offset == 0 && num_elems_to_append >= num_elems_) {
       resetChunkStats();
     }
     T* unencodedData = reinterpret_cast<T*>(src_data);
     std::vector<T> encoded_data;
     if (replicating) {
       if (num_elems_to_append > 0) {
         encoded_data.resize(num_elems_to_append);
         T data = validateDataAndUpdateStats(unencodedData[0]);
         std::fill(encoded_data.begin(), encoded_data.end(), data);
       }
     } else {
       updateStats(src_data, num_elems_to_append, is_validated_data);
     }
     if (offset == -1) {
       auto append_data_size = num_elems_to_append * sizeof(T);
       buffer_->reserve(buffer_->size() + append_data_size);
       num_elems_ += num_elems_to_append;
       buffer_->append(
           replicating ? reinterpret_cast<int8_t*>(encoded_data.data()) : src_data,
           append_data_size);
       if (!replicating) {
         src_data += num_elems_to_append * sizeof(T);
       }
     } else {
       num_elems_ = offset + num_elems_to_append;
       CHECK(!replicating);
       CHECK_GE(offset, 0);
       buffer_->write(
           src_data, num_elems_to_append * sizeof(T), static_cast<size_t>(offset));
     }
     auto chunk_metadata = std::make_shared<ChunkMetadata>();
     getMetadata(chunk_metadata);
     return chunk_metadata;
   }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T>

void NoneEncoder< T >::copyMetadata ( const Encoder * copyFromEncoder )

inlineoverridevirtual

Implements Encoder.

Definition at line 206 of file NoneEncoder.h.

References NoneEncoder< T >::dataMax, NoneEncoder< T >::dataMin, Encoder::getNumElems(), NoneEncoder< T >::has_nulls, and Encoder::num_elems_.

                                                              {
     num_elems_ = copyFromEncoder->getNumElems();
     auto castedEncoder = reinterpret_cast<const NoneEncoder<T>*>(copyFromEncoder);
     dataMin = castedEncoder->dataMin;
     dataMax = castedEncoder->dataMax;
     has_nulls = castedEncoder->has_nulls;
   }

Here is the call graph for this function:

template<typename T>

void NoneEncoder< T >::getMetadata ( const std::shared_ptr< ChunkMetadata > & chunkMetadata )

inlineoverridevirtual

Reimplemented from Encoder.

Definition at line 87 of file NoneEncoder.h.

References NoneEncoder< T >::dataMax, NoneEncoder< T >::dataMin, Encoder::getMetadata(), and NoneEncoder< T >::has_nulls.

Referenced by NoneEncoder< T >::appendValidatedOrNonValidatedData().

                                                                                {
     Encoder::getMetadata(chunkMetadata);  // call on parent class
     chunkMetadata->fillChunkStats(dataMin, dataMax, has_nulls);
   }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T>

std::shared_ptr<ChunkMetadata> NoneEncoder< T >::getMetadata ( const SQLTypeInfo & ti )

inlineoverridevirtual

Implements Encoder.

Definition at line 93 of file NoneEncoder.h.

References NoneEncoder< T >::dataMax, NoneEncoder< T >::dataMin, and NoneEncoder< T >::has_nulls.

                                                                            {
     auto chunk_metadata = std::make_shared<ChunkMetadata>(ti, 0, 0, ChunkStats{});
     chunk_metadata->fillChunkStats(dataMin, dataMax, has_nulls);
     return chunk_metadata;
   }

template<typename T>

size_t NoneEncoder< T >::getNumElemsForBytesEncodedDataAtIndices	(	const int8_t *	index_data,
		const std::vector< size_t > &	selected_idx,
		const size_t	byte_limit
	)

inlineoverridevirtual

Compute the maximum number of variable length encoded elements given a byte limit

Parameters

index_data	- (optional) index data for the encoded type
selected_idx	- which indices in the encoded data to consider
byte_limit	- byte limit that must be respected

Returns: the number of elements

NOTE: optional parameters above may be ignored by the implementation, but may or may not be required depending on the encoder type backing the implementation.

Implements Encoder.

Definition at line 43 of file NoneEncoder.h.

References UNREACHABLE.

                                                                                    {
     UNREACHABLE()
         << "getNumElemsForBytesEncodedDataAtIndices unexpectedly called for non varlen"
            " encoder";
     return {};
   }

template<typename T>

void NoneEncoder< T >::readMetadata ( FILE * f )

inlineoverridevirtual

Implements Encoder.

Definition at line 184 of file NoneEncoder.h.

References NoneEncoder< T >::dataMax, NoneEncoder< T >::dataMin, NoneEncoder< T >::has_nulls, Encoder::num_elems_, and heavydb.dtypes::T.

                                       {
     // assumes pointer is already in right place
     fread((int8_t*)&num_elems_, sizeof(size_t), 1, f);
     fread((int8_t*)&dataMin, sizeof(T), 1, f);
     fread((int8_t*)&dataMax, sizeof(T), 1, f);
     fread((int8_t*)&has_nulls, sizeof(bool), 1, f);
   }

template<typename T>

void NoneEncoder< T >::reduceStats ( const Encoder & that )

inlineoverridevirtual

Implements Encoder.

Definition at line 167 of file NoneEncoder.h.

References NoneEncoder< T >::dataMax, NoneEncoder< T >::dataMin, and NoneEncoder< T >::has_nulls.

                                                  {
     const auto that_typed = static_cast<const NoneEncoder&>(that);
     if (that_typed.has_nulls) {
       has_nulls = true;
     }
     dataMin = std::min(dataMin, that_typed.dataMin);
     dataMax = std::max(dataMax, that_typed.dataMax);
   }

template<typename T>

bool NoneEncoder< T >::resetChunkStats ( const ChunkStats & )

inlineoverridevirtual

: Reset chunk level stats (min, max, nulls) using new values from the argument.

Returns: : True if an update occurred and the chunk needs to be flushed. False otherwise. Default false if metadata update is unsupported. Only reset chunk stats if the incoming stats differ from the current stats.

Reimplemented from Encoder.

Definition at line 192 of file NoneEncoder.h.

References NoneEncoder< T >::dataMax, NoneEncoder< T >::dataMin, ChunkStats::has_nulls, NoneEncoder< T >::has_nulls, ChunkStats::max, and ChunkStats::min.

                                                          {
     const auto new_min = DatumFetcher::getDatumVal<T>(stats.min);
     const auto new_max = DatumFetcher::getDatumVal<T>(stats.max);
 
     if (dataMin == new_min && dataMax == new_max && has_nulls == stats.has_nulls) {
       return false;
     }
 
     dataMin = new_min;
     dataMax = new_max;
     has_nulls = stats.has_nulls;
     return true;
   }

template<typename T>

void NoneEncoder< T >::resetChunkStats ( )

inlineoverridevirtual

Resets chunk metadata stats to their default values.

Implements Encoder.

Definition at line 214 of file NoneEncoder.h.

References NoneEncoder< T >::dataMax, NoneEncoder< T >::dataMin, and NoneEncoder< T >::has_nulls.

Referenced by NoneEncoder< T >::appendValidatedOrNonValidatedData(), and NoneEncoder< T >::NoneEncoder().

                                   {
     dataMin = std::numeric_limits<T>::max();
     dataMax = std::numeric_limits<T>::lowest();
     has_nulls = false;
   }

Here is the caller graph for this function:

template<typename T>

void NoneEncoder< T >::updateStats	(	const int64_t	val,
		const bool	is_null
	)

inlineoverridevirtual

Implements Encoder.

Definition at line 100 of file NoneEncoder.h.

References NoneEncoder< T >::dataMax, NoneEncoder< T >::dataMin, NoneEncoder< T >::has_nulls, and heavydb.dtypes::T.

Referenced by NoneEncoder< T >::appendValidatedOrNonValidatedData(), and NoneEncoder< T >::updateStats().

                                                                    {
     if (is_null) {
       has_nulls = true;
     } else {
       const auto data = static_cast<T>(val);
       dataMin = std::min(dataMin, data);
       dataMax = std::max(dataMax, data);
     }
   }

Here is the caller graph for this function:

template<typename T>

void NoneEncoder< T >::updateStats	(	const double	val,
		const bool	is_null
	)

inlineoverridevirtual

Implements Encoder.

Definition at line 111 of file NoneEncoder.h.

References NoneEncoder< T >::dataMax, NoneEncoder< T >::dataMin, NoneEncoder< T >::has_nulls, and heavydb.dtypes::T.

                                                                   {
     if (is_null) {
       has_nulls = true;
     } else {
       const auto data = static_cast<T>(val);
       dataMin = std::min(dataMin, data);
       dataMax = std::max(dataMax, data);
     }
   }

template<typename T>

void NoneEncoder< T >::updateStats	(	const int8_t *const	src_data,
		const size_t	num_elements
	)

inlineoverridevirtual

Update statistics for data without appending.

Parameters

src_data	- the data with which to update statistics
num_elements	- the number of elements to scan in the data

Implements Encoder.

Definition at line 121 of file NoneEncoder.h.

References NoneEncoder< T >::updateStats().

                                                                                      {
     updateStats(src_data, num_elements, false);
   }

Here is the call graph for this function:

template<typename T>

void NoneEncoder< T >::updateStats	(	const std::vector< std::string > *const	src_data,
		const size_t	start_idx,
		const size_t	num_elements
	)

inlineoverridevirtual

Update statistics for string data without appending.

Parameters

src_data	- the string data with which to update statistics
start_idx	- the offset into `src_data` to start the update
num_elements	- the number of elements to scan in the string data

Implements Encoder.

Definition at line 154 of file NoneEncoder.h.

References UNREACHABLE.

                                                        {
     UNREACHABLE();
   }

template<typename T>

void NoneEncoder< T >::updateStats	(	const std::vector< ArrayDatum > *const	src_data,
		const size_t	start_idx,
		const size_t	num_elements
	)

inlineoverridevirtual

Update statistics for array data without appending.

Parameters

src_data	- the array data with which to update statistics
start_idx	- the offset into `src_data` to start the update
num_elements	- the number of elements to scan in the array data

Implements Encoder.

Definition at line 160 of file NoneEncoder.h.

References UNREACHABLE.

                                                        {
     UNREACHABLE();
   }

template<typename T>

void NoneEncoder< T >::updateStats	(	const int8_t *const	src_data,
		const size_t	num_elements,
		const bool	is_validated_data
	)

inlineprivate

Definition at line 281 of file NoneEncoder.h.

References heavydb.dtypes::T, and NoneEncoder< T >::validateDataAndUpdateStats().

                                                  {
     const T* unencoded_data = reinterpret_cast<const T*>(src_data);
     for (size_t i = 0; i < num_elements; ++i) {
       validateDataAndUpdateStats(unencoded_data[i], is_validated_data);
     }
   }

Here is the call graph for this function:

template<typename T>

void NoneEncoder< T >::updateStatsEncoded	(	const int8_t *const	dst_data,
		const size_t	num_elements
	)

inlineoverridevirtual

Update statistics for encoded data without appending.

Parameters

dst_data	- the data with which to update statistics
num_elements	- the number of elements to scan in the data

Reimplemented from Encoder.

Definition at line 125 of file NoneEncoder.h.

References NoneEncoder< T >::dataMax, NoneEncoder< T >::dataMin, Encoder::decimal_overflow_validator_, NoneEncoder< T >::has_nulls, logger::init(), threading_serial::parallel_reduce(), heavydb.dtypes::T, and DecimalOverflowValidator::validate().

                                                               {
     const T* data = reinterpret_cast<const T*>(dst_data);
 
     std::tie(dataMin, dataMax, has_nulls) = tbb::parallel_reduce(
         tbb::blocked_range(size_t(0), num_elements),
         std::tuple(dataMin, dataMax, has_nulls),
         [&](const auto& range, auto init) {
           auto [min, max, nulls] = init;
           for (size_t i = range.begin(); i < range.end(); i++) {
             if (data[i] != none_encoded_null_value<T>()) {
               decimal_overflow_validator_.validate(data[i]);
               min = std::min(min, data[i]);
               max = std::max(max, data[i]);
             } else {
               nulls = true;
             }
           }
           return std::tuple(min, max, nulls);
         },
         [&](auto lhs, auto rhs) {
           const auto [lhs_min, lhs_max, lhs_nulls] = lhs;
           const auto [rhs_min, rhs_max, rhs_nulls] = rhs;
           return std::tuple(std::min(lhs_min, rhs_min),
                             std::max(lhs_max, rhs_max),
                             lhs_nulls || rhs_nulls);
         });
   }

Here is the call graph for this function:

template<typename T>

T NoneEncoder< T >::validateDataAndUpdateStats	(	const T &	unencoded_data,
		const bool	is_validated_data = `false`
	)

inlineprivate

Definition at line 267 of file NoneEncoder.h.

References NoneEncoder< T >::dataMax, NoneEncoder< T >::dataMin, Encoder::decimal_overflow_validator_, NoneEncoder< T >::has_nulls, and DecimalOverflowValidator::validate().

Referenced by NoneEncoder< T >::appendValidatedOrNonValidatedData(), and NoneEncoder< T >::updateStats().

                                                                      {
     if (unencoded_data == none_encoded_null_value<T>()) {
       has_nulls = true;
     } else {
       if (!is_validated_data) {  // does not need validation
         decimal_overflow_validator_.validate(unencoded_data);
       }
       dataMin = std::min(dataMin, unencoded_data);
       dataMax = std::max(dataMax, unencoded_data);
     }
     return unencoded_data;
   }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T>

void NoneEncoder< T >::writeMetadata ( FILE * f )

inlineoverridevirtual

Implements Encoder.

Definition at line 176 of file NoneEncoder.h.

References NoneEncoder< T >::dataMax, NoneEncoder< T >::dataMin, NoneEncoder< T >::has_nulls, Encoder::num_elems_, and heavydb.dtypes::T.

                                        {
     // assumes pointer is already in right place
     fwrite((int8_t*)&num_elems_, sizeof(size_t), 1, f);
     fwrite((int8_t*)&dataMin, sizeof(T), 1, f);
     fwrite((int8_t*)&dataMax, sizeof(T), 1, f);
     fwrite((int8_t*)&has_nulls, sizeof(bool), 1, f);
   }

Member Data Documentation

template<typename T>

T NoneEncoder< T >::dataMax

Definition at line 221 of file NoneEncoder.h.

Referenced by NoneEncoder< T >::copyMetadata(), NoneEncoder< T >::getMetadata(), NoneEncoder< T >::readMetadata(), NoneEncoder< T >::reduceStats(), NoneEncoder< T >::resetChunkStats(), NoneEncoder< T >::updateStats(), NoneEncoder< T >::updateStatsEncoded(), NoneEncoder< T >::validateDataAndUpdateStats(), and NoneEncoder< T >::writeMetadata().

template<typename T>

T NoneEncoder< T >::dataMin

Definition at line 220 of file NoneEncoder.h.

Referenced by NoneEncoder< T >::copyMetadata(), NoneEncoder< T >::getMetadata(), NoneEncoder< T >::readMetadata(), NoneEncoder< T >::reduceStats(), NoneEncoder< T >::resetChunkStats(), NoneEncoder< T >::updateStats(), NoneEncoder< T >::updateStatsEncoded(), NoneEncoder< T >::validateDataAndUpdateStats(), and NoneEncoder< T >::writeMetadata().

template<typename T>

bool NoneEncoder< T >::has_nulls

Definition at line 222 of file NoneEncoder.h.

Referenced by NoneEncoder< T >::copyMetadata(), NoneEncoder< T >::getMetadata(), heavydb.thrift.ttypes.TColumnRange::read(), NoneEncoder< T >::readMetadata(), NoneEncoder< T >::reduceStats(), NoneEncoder< T >::resetChunkStats(), NoneEncoder< T >::updateStats(), NoneEncoder< T >::updateStatsEncoded(), NoneEncoder< T >::validateDataAndUpdateStats(), heavydb.thrift.ttypes.TColumnRange::write(), and NoneEncoder< T >::writeMetadata().

The documentation for this class was generated from the following file:

/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/NoneEncoder.h

Public Member Functions

Public Attributes

Private Member Functions

Additional Inherited Members

Detailed Description

template<typename T> class NoneEncoder< T >

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation

template<typename T>
class NoneEncoder< T >