OmniSciDB  c1a53651b2
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
FlatBufferManager Struct Reference

#include <FlatBuffer.h>

Classes

struct  DTypeMetadataSize
 
struct  DTypeMetadataSizeDictId
 

Public Types

enum  Status {
  Success = 0, IndexError, SizeError, ItemAlreadySpecifiedError,
  ItemUnspecifiedError, ValuesBufferTooSmallError, MemoryError, NotImplemnentedError,
  UnknownFormatError
}
 
enum  VarlenArrayHeader {
  FORMAT_ID = 0, FLATBUFFER_SIZE, ITEMS_COUNT, DTYPE_METADATA_KIND,
  DTYPE_METADATA_BUFFER_SIZE, MAX_NOF_VALUES, STORAGE_COUNT, DTYPE_METADATA_BUFFER_OFFSET,
  VALUES_OFFSET, COMPRESSED_INDICES_OFFSET, STORAGE_INDICES_OFFSET, EOFHEADER
}
 
enum  DTypeMetadataKind { SIZE = 1, SIZE_DICTID }
 

Public Member Functions

HOST DEVICE FlatBufferFormat format () const
 
int64_t flatbufferSize () const
 
HOST DEVICE DTypeMetadataKind getDTypeMetadataKind () const
 
int8_t * getDTypeMetadataBuffer ()
 
HOST DEVICE const int8_t * getDTypeMetadataBuffer () const
 
void setDTypeMetadataSize (int64_t size)
 
HOST DEVICE int64_t getDTypeMetadataSize () const
 
void setDTypeMetadataDictKey (int32_t db_id, int32_t dict_id)
 
int32_t getDTypeMetadataDictDbId () const
 
int32_t getDTypeMetadataDictId () const
 
void initializeVarlenArray (int64_t items_count, int64_t max_nof_values, int64_t dtype_size, DTypeMetadataKind dtype_metadata_kind)
 
HOST DEVICE int64_t itemsCount () const
 
HOST DEVICE int64_t dtypeSize () const
 
int64_t VarlenArray_max_nof_values () const
 
int64_t VarlenArray_nof_values () const
 
int64_t VarlenArray_values_buffer_size () const
 
int64_t & VarlenArray_storage_count ()
 
int64_t & VarlenArray_storage_count () const
 
HOST DEVICE int8_t * VarlenArray_values ()
 
const int8_t * VarlenArray_values () const
 
HOST DEVICE int64_t * VarlenArray_compressed_indices ()
 
const int64_t * VarlenArray_compressed_indices () const
 
HOST DEVICE int64_t * VarlenArray_storage_indices ()
 
const int64_t * VarlenArray_storage_indices () const
 
Status setItem (int64_t index, const int8_t *src, int64_t size, int8_t **dest=nullptr)
 
Status setItemNoValidation (int64_t index, const int8_t *src, int64_t size, int8_t **dest)
 
Status setEmptyItemNoValidation (int64_t index, int64_t size, int8_t **dest)
 
Status concatItem (int64_t index, const int8_t *src, int64_t size)
 
Status setNull (int64_t index)
 
Status setNullNoValidation (int64_t index)
 
Status isNull (int64_t index, bool &is_null) const
 
HOST DEVICE Status getItem (int64_t index, int64_t &size, int8_t *&dest, bool &is_null)
 
HOST DEVICE Status getItem (int64_t index, size_t &size, int8_t *&dest, bool &is_null)
 

Static Public Member Functions

HOST static DEVICE bool isFlatBuffer (const void *buffer)
 
static int64_t getBufferSize (const void *buffer)
 
static int64_t getDTypeMetadataBufferSize (DTypeMetadataKind kind)
 
static int64_t get_VarlenArray_flatbuffer_size (int64_t items_count, int64_t max_nof_values, int64_t dtype_size, DTypeMetadataKind dtype_metadata_kind)
 

Public Attributes

int8_t * buffer
 

Detailed Description

Definition at line 148 of file FlatBuffer.h.

Member Enumeration Documentation

Enumerator
SIZE 
SIZE_DICTID 

Definition at line 225 of file FlatBuffer.h.

Enumerator
FORMAT_ID 
FLATBUFFER_SIZE 
ITEMS_COUNT 
DTYPE_METADATA_KIND 
DTYPE_METADATA_BUFFER_SIZE 
MAX_NOF_VALUES 
STORAGE_COUNT 
DTYPE_METADATA_BUFFER_OFFSET 
VALUES_OFFSET 
COMPRESSED_INDICES_OFFSET 
STORAGE_INDICES_OFFSET 
EOFHEADER 

Definition at line 161 of file FlatBuffer.h.

161  {
162  FORMAT_ID = 0, // storage format id
163  FLATBUFFER_SIZE, // in bytes
164  ITEMS_COUNT, /* the number of items
165  (e.g. the number of
166  rows in a column) */
167  DTYPE_METADATA_KIND, /* the kind of dtype metadata */
168  DTYPE_METADATA_BUFFER_SIZE, /* the size of dtype metadata buffer */
169  MAX_NOF_VALUES, /* the upper bound to the
170  total number of values
171  in all items */
172  STORAGE_COUNT, /* the number of specified
173  items, incremented by
174  one on each
175  setNull/setItem call */
180  EOFHEADER
181  };

Member Function Documentation

Status FlatBufferManager::concatItem ( int64_t  index,
const int8_t *  src,
int64_t  size 
)
inline

Definition at line 543 of file FlatBuffer.h.

References dtypeSize(), format(), IndexError, itemsCount(), MemoryError, NotImplemnentedError, setItem(), SizeError, Success, UnknownFormatError, VarlenArray, VarlenArray_compressed_indices(), VarlenArray_storage_count(), VarlenArray_storage_indices(), and VarlenArray_values().

543  {
545  if (index < 0 || index >= itemsCount()) {
546  return IndexError;
547  }
548  int64_t next_storage_count = VarlenArray_storage_count();
549  int64_t storage_count = next_storage_count - 1;
550  int64_t* compressed_indices = VarlenArray_compressed_indices();
551  int64_t* storage_indices = VarlenArray_storage_indices();
552  int8_t* values = VarlenArray_values();
553  int64_t itemsize = dtypeSize();
554  int64_t storage_index = storage_indices[index];
555 
556  if (storage_index == -1) { // unspecified, so setting the item
557  return setItem(index, src, size, nullptr);
558  }
559  if (size % itemsize != 0) {
560  return SizeError;
561  }
562  if (storage_index != storage_count) {
563  return IndexError; // index does not correspond to the last set
564  // item, only the last item can be
565  // concatenated
566  }
567  if (compressed_indices[storage_index] < 0) {
568  return NotImplemnentedError; // todo: support concat to null when last
569  }
570  int64_t values_count =
571  compressed_indices[next_storage_count] - compressed_indices[storage_index];
572  int64_t extra_values_count = size / itemsize;
573  compressed_indices[next_storage_count] += extra_values_count;
574  int8_t* ptr = values + compressed_indices[storage_index] * itemsize;
575  if (size > 0 && src != nullptr &&
576  memcpy(ptr + values_count * itemsize, src, size) == nullptr) {
577  return MemoryError;
578  }
579  return Success;
580  }
581  return UnknownFormatError;
582  }
HOST DEVICE int64_t dtypeSize() const
Definition: FlatBuffer.h:420
Status setItem(int64_t index, const int8_t *src, int64_t size, int8_t **dest=nullptr)
Definition: FlatBuffer.h:481
HOST DEVICE FlatBufferFormat format() const
Definition: FlatBuffer.h:213
int64_t & VarlenArray_storage_count()
Definition: FlatBuffer.h:440
HOST DEVICE int64_t itemsCount() const
Definition: FlatBuffer.h:412
HOST DEVICE int64_t * VarlenArray_compressed_indices()
Definition: FlatBuffer.h:458
HOST DEVICE int64_t * VarlenArray_storage_indices()
Definition: FlatBuffer.h:468
HOST DEVICE int8_t * VarlenArray_values()
Definition: FlatBuffer.h:449

+ Here is the call graph for this function:

HOST DEVICE int64_t FlatBufferManager::dtypeSize ( ) const
inline

Definition at line 420 of file FlatBuffer.h.

References getDTypeMetadataSize().

Referenced by concatItem(), getItem(), setItem(), setItemNoValidation(), and VarlenArray_values_buffer_size().

420 { return getDTypeMetadataSize(); }
HOST DEVICE int64_t getDTypeMetadataSize() const
Definition: FlatBuffer.h:282

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

int64_t FlatBufferManager::flatbufferSize ( ) const
inline

Definition at line 219 of file FlatBuffer.h.

References buffer.

Referenced by TableFunctionExecutionContext::launchCpuCode(), and synthesize_metadata_table_function().

219  {
220  return ((const int64_t*)(buffer))[VarlenArrayHeader::FLATBUFFER_SIZE];
221  }

+ Here is the caller graph for this function:

HOST DEVICE FlatBufferFormat FlatBufferManager::format ( ) const
inline

Definition at line 213 of file FlatBuffer.h.

References buffer.

Referenced by concatItem(), getItem(), isNull(), itemsCount(), setItem(), and setNull().

213  {
214  return static_cast<FlatBufferFormat>(
215  ((int64_t*)buffer)[VarlenArrayHeader::FORMAT_ID]);
216  }
FlatBufferFormat
Definition: FlatBuffer.h:139

+ Here is the caller graph for this function:

static int64_t FlatBufferManager::get_VarlenArray_flatbuffer_size ( int64_t  items_count,
int64_t  max_nof_values,
int64_t  dtype_size,
DTypeMetadataKind  dtype_metadata_kind 
)
inlinestatic

Definition at line 343 of file FlatBuffer.h.

References _align_to_int64(), and getDTypeMetadataBufferSize().

Referenced by getVarlenArrayBufferSize(), and initializeVarlenArray().

346  {
347  const int64_t VarlenArray_buffer_header_size =
348  VarlenArrayHeader::EOFHEADER * sizeof(int64_t);
349  const int64_t dtype_metadata_buffer_size =
350  getDTypeMetadataBufferSize(dtype_metadata_kind);
351  const int64_t values_buffer_size = _align_to_int64(dtype_size * max_nof_values);
352  return (VarlenArray_buffer_header_size // see above
353  + dtype_metadata_buffer_size // size of dtype metadata buffer in bytes,
354  // aligned to int64
355  + values_buffer_size // size of values buffer, aligned to int64
356  + (items_count + 1 // size of compressed_indices buffer
357  + items_count // size of storage_indices buffer
358  + 1 // footer format id
359  ) * sizeof(int64_t));
360  }
int64_t _align_to_int64(int64_t addr)
Definition: FlatBuffer.h:143
static int64_t getDTypeMetadataBufferSize(DTypeMetadataKind kind)
Definition: FlatBuffer.h:240

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

static int64_t FlatBufferManager::getBufferSize ( const void *  buffer)
inlinestatic

Definition at line 204 of file FlatBuffer.h.

References isFlatBuffer().

Referenced by ColumnFetcher::transferColumnIfNeeded().

204  {
205  if (isFlatBuffer(buffer)) {
206  return ((const int64_t*)(buffer))[VarlenArrayHeader::FLATBUFFER_SIZE];
207  } else {
208  return -1;
209  }
210  }
HOST static DEVICE bool isFlatBuffer(const void *buffer)
Definition: FlatBuffer.h:186

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

int8_t* FlatBufferManager::getDTypeMetadataBuffer ( )
inline

Definition at line 256 of file FlatBuffer.h.

References buffer.

Referenced by getDTypeMetadataDictDbId(), getDTypeMetadataDictId(), getDTypeMetadataSize(), setDTypeMetadataDictKey(), and setDTypeMetadataSize().

256  {
257  return buffer + ((int64_t*)buffer)[VarlenArrayHeader::DTYPE_METADATA_BUFFER_OFFSET];
258  }

+ Here is the caller graph for this function:

HOST DEVICE const int8_t* FlatBufferManager::getDTypeMetadataBuffer ( ) const
inline

Definition at line 260 of file FlatBuffer.h.

References buffer.

260  {
261  return buffer +
262  ((const int64_t*)buffer)[VarlenArrayHeader::DTYPE_METADATA_BUFFER_OFFSET];
263  }
static int64_t FlatBufferManager::getDTypeMetadataBufferSize ( DTypeMetadataKind  kind)
inlinestatic

Definition at line 240 of file FlatBuffer.h.

References _align_to_int64().

Referenced by get_VarlenArray_flatbuffer_size(), and initializeVarlenArray().

240  {
241  switch (kind) {
242  case DTypeMetadataKind::SIZE:
243  return _align_to_int64(sizeof(DTypeMetadataSize));
244  case DTypeMetadataKind::SIZE_DICTID:
245  return _align_to_int64(sizeof(DTypeMetadataSizeDictId));
246  default:
247  return 0;
248  }
249  }
int64_t _align_to_int64(int64_t addr)
Definition: FlatBuffer.h:143

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

int32_t FlatBufferManager::getDTypeMetadataDictDbId ( ) const
inline

Definition at line 314 of file FlatBuffer.h.

References FlatBufferManager::DTypeMetadataSizeDictId::db_id, getDTypeMetadataBuffer(), and getDTypeMetadataKind().

Referenced by Column< Array< T > >::getDictDbId().

314  {
315  switch (getDTypeMetadataKind()) {
316  case DTypeMetadataKind::SIZE:
317  break;
318  case DTypeMetadataKind::SIZE_DICTID: {
319  const DTypeMetadataSizeDictId* metadata =
320  reinterpret_cast<const DTypeMetadataSizeDictId*>(getDTypeMetadataBuffer());
321  return metadata->db_id;
322  }
323  default:;
324  }
325  return {};
326  }
int8_t * getDTypeMetadataBuffer()
Definition: FlatBuffer.h:256
HOST DEVICE DTypeMetadataKind getDTypeMetadataKind() const
Definition: FlatBuffer.h:251

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

int32_t FlatBufferManager::getDTypeMetadataDictId ( ) const
inline

Definition at line 328 of file FlatBuffer.h.

References FlatBufferManager::DTypeMetadataSizeDictId::dict_id, getDTypeMetadataBuffer(), and getDTypeMetadataKind().

Referenced by Column< Array< T > >::getDictId().

328  {
329  switch (getDTypeMetadataKind()) {
330  case DTypeMetadataKind::SIZE:
331  break;
332  case DTypeMetadataKind::SIZE_DICTID: {
333  const DTypeMetadataSizeDictId* metadata =
334  reinterpret_cast<const DTypeMetadataSizeDictId*>(getDTypeMetadataBuffer());
335  return metadata->dict_id;
336  }
337  default:;
338  }
339  return {};
340  }
int8_t * getDTypeMetadataBuffer()
Definition: FlatBuffer.h:256
HOST DEVICE DTypeMetadataKind getDTypeMetadataKind() const
Definition: FlatBuffer.h:251

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

HOST DEVICE DTypeMetadataKind FlatBufferManager::getDTypeMetadataKind ( ) const
inline

Definition at line 251 of file FlatBuffer.h.

References buffer.

Referenced by getDTypeMetadataDictDbId(), getDTypeMetadataDictId(), getDTypeMetadataSize(), setDTypeMetadataDictKey(), and setDTypeMetadataSize().

251  {
252  return static_cast<DTypeMetadataKind>(
253  ((int64_t*)buffer)[VarlenArrayHeader::DTYPE_METADATA_KIND]);
254  }

+ Here is the caller graph for this function:

HOST DEVICE int64_t FlatBufferManager::getDTypeMetadataSize ( ) const
inline

Definition at line 282 of file FlatBuffer.h.

References getDTypeMetadataBuffer(), getDTypeMetadataKind(), FlatBufferManager::DTypeMetadataSize::size, and FlatBufferManager::DTypeMetadataSizeDictId::size.

Referenced by dtypeSize().

282  {
283  switch (getDTypeMetadataKind()) {
284  case DTypeMetadataKind::SIZE: {
285  const DTypeMetadataSize* metadata =
286  reinterpret_cast<const DTypeMetadataSize*>(getDTypeMetadataBuffer());
287  return metadata->size;
288  }
289  case DTypeMetadataKind::SIZE_DICTID: {
290  const DTypeMetadataSizeDictId* metadata =
291  reinterpret_cast<const DTypeMetadataSizeDictId*>(getDTypeMetadataBuffer());
292  return metadata->size;
293  }
294  default:;
295  }
296  return 0;
297  }
int8_t * getDTypeMetadataBuffer()
Definition: FlatBuffer.h:256
HOST DEVICE DTypeMetadataKind getDTypeMetadataKind() const
Definition: FlatBuffer.h:251

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

HOST DEVICE Status FlatBufferManager::getItem ( int64_t  index,
int64_t &  size,
int8_t *&  dest,
bool &  is_null 
)
inline

Definition at line 633 of file FlatBuffer.h.

References dtypeSize(), format(), IndexError, itemsCount(), ItemUnspecifiedError, Success, UnknownFormatError, VarlenArray, VarlenArray_compressed_indices(), VarlenArray_storage_indices(), and VarlenArray_values().

Referenced by getItem(), and VarlenArray_get_nth().

633  {
635  if (index < 0 || index >= itemsCount()) {
636  return IndexError;
637  }
638  int8_t* values = VarlenArray_values();
639  const int64_t* compressed_indices = VarlenArray_compressed_indices();
640  const int64_t* storage_indices = VarlenArray_storage_indices();
641  const int64_t storage_index = storage_indices[index];
642  if (storage_index < 0) {
643  return ItemUnspecifiedError;
644  }
645  const int64_t cindex = compressed_indices[storage_index];
646  if (cindex < 0) {
647  // null varlen array
648  size = 0;
649  dest = nullptr;
650  is_null = true;
651  } else {
652  const int64_t dtypesize = dtypeSize();
653  const int64_t next_cindex = compressed_indices[storage_index + 1];
654  const int64_t length =
655  (next_cindex < 0 ? -(next_cindex + 1) - cindex : next_cindex - cindex);
656  size = length * dtypesize;
657  dest = values + cindex * dtypesize;
658  is_null = false;
659  }
660  return Success;
661  } else {
662  return UnknownFormatError;
663  }
664  }
HOST DEVICE int64_t dtypeSize() const
Definition: FlatBuffer.h:420
HOST DEVICE FlatBufferFormat format() const
Definition: FlatBuffer.h:213
CONSTEXPR DEVICE bool is_null(const T &value)
HOST DEVICE int64_t itemsCount() const
Definition: FlatBuffer.h:412
HOST DEVICE int64_t * VarlenArray_compressed_indices()
Definition: FlatBuffer.h:458
HOST DEVICE int64_t * VarlenArray_storage_indices()
Definition: FlatBuffer.h:468
HOST DEVICE int8_t * VarlenArray_values()
Definition: FlatBuffer.h:449

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

HOST DEVICE Status FlatBufferManager::getItem ( int64_t  index,
size_t &  size,
int8_t *&  dest,
bool &  is_null 
)
inline

Definition at line 666 of file FlatBuffer.h.

References getItem().

666  {
667  int64_t sz{0};
668  Status status = getItem(index, sz, dest, is_null);
669  size = sz;
670  return status;
671  }
HOST DEVICE Status getItem(int64_t index, int64_t &size, int8_t *&dest, bool &is_null)
Definition: FlatBuffer.h:633
CONSTEXPR DEVICE bool is_null(const T &value)

+ Here is the call graph for this function:

void FlatBufferManager::initializeVarlenArray ( int64_t  items_count,
int64_t  max_nof_values,
int64_t  dtype_size,
DTypeMetadataKind  dtype_metadata_kind 
)
inline

Definition at line 363 of file FlatBuffer.h.

References _align_to_int64(), buffer, get_VarlenArray_flatbuffer_size(), getDTypeMetadataBufferSize(), setDTypeMetadataSize(), VarlenArray, VarlenArray_compressed_indices(), and VarlenArray_storage_indices().

Referenced by initializeVarlenArray().

366  {
367  const int64_t VarlenArray_buffer_header_size =
368  VarlenArrayHeader::EOFHEADER * sizeof(int64_t);
369  const int64_t values_buffer_size = _align_to_int64(dtype_size * max_nof_values);
370  const int64_t compressed_indices_buffer_size = (items_count + 1) * sizeof(int64_t);
371  const int64_t dtype_metadata_buffer_size =
372  getDTypeMetadataBufferSize(dtype_metadata_kind);
373  const int64_t flatbuffer_size = get_VarlenArray_flatbuffer_size(
374  items_count, max_nof_values, dtype_size, dtype_metadata_kind);
375  ((int64_t*)buffer)[VarlenArrayHeader::FORMAT_ID] =
376  static_cast<int64_t>(FlatBufferFormat::VarlenArray);
377  ((int64_t*)buffer)[VarlenArrayHeader::FLATBUFFER_SIZE] = flatbuffer_size;
378  ((int64_t*)buffer)[VarlenArrayHeader::ITEMS_COUNT] = items_count;
379  ((int64_t*)buffer)[VarlenArrayHeader::MAX_NOF_VALUES] = max_nof_values;
380  ((int64_t*)buffer)[VarlenArrayHeader::DTYPE_METADATA_KIND] =
381  static_cast<int64_t>(dtype_metadata_kind);
382  ((int64_t*)buffer)[VarlenArrayHeader::DTYPE_METADATA_BUFFER_SIZE] =
383  dtype_metadata_buffer_size;
384  ((int64_t*)buffer)[VarlenArrayHeader::STORAGE_COUNT] = 0;
385  ((int64_t*)buffer)[VarlenArrayHeader::DTYPE_METADATA_BUFFER_OFFSET] =
386  VarlenArray_buffer_header_size;
387  ((int64_t*)buffer)[VarlenArrayHeader::VALUES_OFFSET] =
388  ((const int64_t*)buffer)[VarlenArrayHeader::DTYPE_METADATA_BUFFER_OFFSET] +
389  dtype_metadata_buffer_size;
390  ((int64_t*)buffer)[VarlenArrayHeader::COMPRESSED_INDICES_OFFSET] =
391  ((const int64_t*)buffer)[VarlenArrayHeader::VALUES_OFFSET] + values_buffer_size;
392  ((int64_t*)buffer)[VarlenArrayHeader::STORAGE_INDICES_OFFSET] =
393  ((const int64_t*)buffer)[VarlenArrayHeader::COMPRESSED_INDICES_OFFSET] +
394  compressed_indices_buffer_size;
395 
396  setDTypeMetadataSize(dtype_size);
397 
398  // initialize indices buffers
399  int64_t* compressed_indices = VarlenArray_compressed_indices();
400  int64_t* storage_indices = VarlenArray_storage_indices();
401  for (int i = 0; i < items_count; i++) {
402  compressed_indices[i] = 0;
403  storage_indices[i] = -1;
404  }
405  compressed_indices[items_count] = 0;
406  // store footer format id in the last 8-bytes of the flatbuffer:
407  ((int64_t*)buffer)[flatbuffer_size / sizeof(int64_t) - 1] =
408  static_cast<int64_t>(FlatBufferFormat::VarlenArray);
409  }
static int64_t get_VarlenArray_flatbuffer_size(int64_t items_count, int64_t max_nof_values, int64_t dtype_size, DTypeMetadataKind dtype_metadata_kind)
Definition: FlatBuffer.h:343
HOST DEVICE int64_t * VarlenArray_compressed_indices()
Definition: FlatBuffer.h:458
HOST DEVICE int64_t * VarlenArray_storage_indices()
Definition: FlatBuffer.h:468
int64_t _align_to_int64(int64_t addr)
Definition: FlatBuffer.h:143
static int64_t getDTypeMetadataBufferSize(DTypeMetadataKind kind)
Definition: FlatBuffer.h:240
void setDTypeMetadataSize(int64_t size)
Definition: FlatBuffer.h:266

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

HOST static DEVICE bool FlatBufferManager::isFlatBuffer ( const void *  buffer)
inlinestatic

Definition at line 186 of file FlatBuffer.h.

References VarlenArray.

Referenced by ChunkIter_get_nth(), ChunkIter_get_nth_varlen(), ColumnarResults::ColumnarResults(), getBufferSize(), ResultSet::makeVarlenTargetValue(), synthesize_metadata_table_function(), ColumnFetcher::transferColumnIfNeeded(), and ColumnarResults::writeBackCell().

186  {
187  if (buffer) {
188  // warning: assume that buffer size is at least 8 bytes
189  FlatBufferFormat header_format =
190  static_cast<FlatBufferFormat>(((int64_t*)buffer)[VarlenArrayHeader::FORMAT_ID]);
191  if (header_format == FlatBufferFormat::VarlenArray) {
192  int64_t flatbuffer_size = ((int64_t*)buffer)[VarlenArrayHeader::FLATBUFFER_SIZE];
193  if (flatbuffer_size > 0) {
194  FlatBufferFormat footer_format = static_cast<FlatBufferFormat>(
195  ((int64_t*)buffer)[flatbuffer_size / sizeof(int64_t) - 1]);
196  return footer_format == header_format;
197  }
198  }
199  }
200  return false;
201  }
FlatBufferFormat
Definition: FlatBuffer.h:139

+ Here is the caller graph for this function:

Status FlatBufferManager::isNull ( int64_t  index,
bool &  is_null 
) const
inline

Definition at line 613 of file FlatBuffer.h.

References format(), IndexError, itemsCount(), ItemUnspecifiedError, Success, UnknownFormatError, VarlenArray, VarlenArray_compressed_indices(), and VarlenArray_storage_indices().

613  {
615  if (index < 0 || index >= itemsCount()) {
616  return IndexError;
617  }
618  const int64_t* compressed_indices = VarlenArray_compressed_indices();
619  const int64_t* storage_indices = VarlenArray_storage_indices();
620  const int64_t storage_index = storage_indices[index];
621  if (storage_index < 0) {
622  return ItemUnspecifiedError;
623  }
624  is_null = (compressed_indices[storage_index] < 0);
625  return Success;
626  }
627  return UnknownFormatError;
628  }
HOST DEVICE FlatBufferFormat format() const
Definition: FlatBuffer.h:213
CONSTEXPR DEVICE bool is_null(const T &value)
HOST DEVICE int64_t itemsCount() const
Definition: FlatBuffer.h:412
HOST DEVICE int64_t * VarlenArray_compressed_indices()
Definition: FlatBuffer.h:458
HOST DEVICE int64_t * VarlenArray_storage_indices()
Definition: FlatBuffer.h:468

+ Here is the call graph for this function:

HOST DEVICE int64_t FlatBufferManager::itemsCount ( ) const
inline

Definition at line 412 of file FlatBuffer.h.

References buffer, format(), and VarlenArray.

Referenced by concatItem(), getItem(), isNull(), setItem(), and setNull().

412  {
414  return ((int64_t*)buffer)[VarlenArrayHeader::ITEMS_COUNT];
415  }
416  return -1; // invalid value
417  }
HOST DEVICE FlatBufferFormat format() const
Definition: FlatBuffer.h:213

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void FlatBufferManager::setDTypeMetadataDictKey ( int32_t  db_id,
int32_t  dict_id 
)
inline

Definition at line 300 of file FlatBuffer.h.

References FlatBufferManager::DTypeMetadataSizeDictId::db_id, FlatBufferManager::DTypeMetadataSizeDictId::dict_id, getDTypeMetadataBuffer(), and getDTypeMetadataKind().

Referenced by initializeVarlenArray().

300  {
301  switch (getDTypeMetadataKind()) {
302  case DTypeMetadataKind::SIZE:
303  break;
304  case DTypeMetadataKind::SIZE_DICTID: {
305  DTypeMetadataSizeDictId* metadata =
306  reinterpret_cast<DTypeMetadataSizeDictId*>(getDTypeMetadataBuffer());
307  metadata->db_id = db_id;
308  metadata->dict_id = dict_id;
309  } break;
310  default:;
311  }
312  }
int8_t * getDTypeMetadataBuffer()
Definition: FlatBuffer.h:256
HOST DEVICE DTypeMetadataKind getDTypeMetadataKind() const
Definition: FlatBuffer.h:251

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void FlatBufferManager::setDTypeMetadataSize ( int64_t  size)
inline

Definition at line 266 of file FlatBuffer.h.

References getDTypeMetadataBuffer(), getDTypeMetadataKind(), FlatBufferManager::DTypeMetadataSize::size, and FlatBufferManager::DTypeMetadataSizeDictId::size.

Referenced by initializeVarlenArray().

266  {
267  switch (getDTypeMetadataKind()) {
268  case DTypeMetadataKind::SIZE: {
269  DTypeMetadataSize* metadata =
270  reinterpret_cast<DTypeMetadataSize*>(getDTypeMetadataBuffer());
271  metadata->size = size;
272  } break;
273  case DTypeMetadataKind::SIZE_DICTID: {
274  DTypeMetadataSizeDictId* metadata =
275  reinterpret_cast<DTypeMetadataSizeDictId*>(getDTypeMetadataBuffer());
276  metadata->size = size;
277  } break;
278  default:;
279  }
280  }
int8_t * getDTypeMetadataBuffer()
Definition: FlatBuffer.h:256
HOST DEVICE DTypeMetadataKind getDTypeMetadataKind() const
Definition: FlatBuffer.h:251

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Status FlatBufferManager::setEmptyItemNoValidation ( int64_t  index,
int64_t  size,
int8_t **  dest 
)
inline

Definition at line 539 of file FlatBuffer.h.

References setItemNoValidation().

539  {
540  return setItemNoValidation(index, nullptr, size, dest);
541  }
Status setItemNoValidation(int64_t index, const int8_t *src, int64_t size, int8_t **dest)
Definition: FlatBuffer.h:512

+ Here is the call graph for this function:

Status FlatBufferManager::setItem ( int64_t  index,
const int8_t *  src,
int64_t  size,
int8_t **  dest = nullptr 
)
inline

Definition at line 481 of file FlatBuffer.h.

References run_benchmark_import::dest, dtypeSize(), format(), IndexError, ItemAlreadySpecifiedError, itemsCount(), setItemNoValidation(), SizeError, UnknownFormatError, ValuesBufferTooSmallError, VarlenArray, VarlenArray_compressed_indices(), VarlenArray_storage_count(), VarlenArray_storage_indices(), and VarlenArray_values_buffer_size().

Referenced by concatItem().

484  {
486  if (index < 0 || index >= itemsCount()) {
487  return IndexError;
488  }
489  int64_t& storage_count = VarlenArray_storage_count();
490  int64_t* compressed_indices = VarlenArray_compressed_indices();
491  int64_t* storage_indices = VarlenArray_storage_indices();
492  const int64_t itemsize = dtypeSize();
493  if (size % itemsize != 0) {
494  return SizeError; // size must be multiple of itemsize. Perhaps size is not in
495  // bytes?
496  }
497  if (storage_indices[index] >= 0) {
499  }
500  const int64_t cindex = compressed_indices[storage_count];
501  const int64_t values_buffer_size = VarlenArray_values_buffer_size();
502  const int64_t csize = cindex * itemsize;
503  if (csize + size > values_buffer_size) {
505  }
506  return setItemNoValidation(index, src, size, dest);
507  }
508  return UnknownFormatError;
509  }
HOST DEVICE int64_t dtypeSize() const
Definition: FlatBuffer.h:420
HOST DEVICE FlatBufferFormat format() const
Definition: FlatBuffer.h:213
int64_t & VarlenArray_storage_count()
Definition: FlatBuffer.h:440
int64_t VarlenArray_values_buffer_size() const
Definition: FlatBuffer.h:435
HOST DEVICE int64_t itemsCount() const
Definition: FlatBuffer.h:412
HOST DEVICE int64_t * VarlenArray_compressed_indices()
Definition: FlatBuffer.h:458
HOST DEVICE int64_t * VarlenArray_storage_indices()
Definition: FlatBuffer.h:468
Status setItemNoValidation(int64_t index, const int8_t *src, int64_t size, int8_t **dest)
Definition: FlatBuffer.h:512

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Status FlatBufferManager::setItemNoValidation ( int64_t  index,
const int8_t *  src,
int64_t  size,
int8_t **  dest 
)
inline

Definition at line 512 of file FlatBuffer.h.

References dtypeSize(), MemoryError, Success, VarlenArray_compressed_indices(), VarlenArray_storage_count(), VarlenArray_storage_indices(), and VarlenArray_values().

Referenced by setEmptyItemNoValidation(), and setItem().

515  {
516  int64_t& storage_count = VarlenArray_storage_count();
517  int64_t* storage_indices = VarlenArray_storage_indices();
518  int64_t* compressed_indices = VarlenArray_compressed_indices();
519  int8_t* values = VarlenArray_values();
520  const int64_t itemsize = dtypeSize();
521  const int64_t values_count = size / itemsize;
522  const int64_t cindex = compressed_indices[storage_count];
523  const int64_t csize = cindex * itemsize;
524  storage_indices[index] = storage_count;
525  compressed_indices[storage_count + 1] = cindex + values_count;
526  if (size > 0 && src != nullptr && memcpy(values + csize, src, size) == nullptr) {
527  return MemoryError;
528  }
529  if (dest != nullptr) {
530  *dest = values + csize;
531  }
532  storage_count++;
533  return Success;
534  }
HOST DEVICE int64_t dtypeSize() const
Definition: FlatBuffer.h:420
int64_t & VarlenArray_storage_count()
Definition: FlatBuffer.h:440
HOST DEVICE int64_t * VarlenArray_compressed_indices()
Definition: FlatBuffer.h:458
HOST DEVICE int64_t * VarlenArray_storage_indices()
Definition: FlatBuffer.h:468
HOST DEVICE int8_t * VarlenArray_values()
Definition: FlatBuffer.h:449

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Status FlatBufferManager::setNull ( int64_t  index)
inline

Definition at line 585 of file FlatBuffer.h.

References format(), IndexError, ItemAlreadySpecifiedError, itemsCount(), setNullNoValidation(), UnknownFormatError, VarlenArray, and VarlenArray_storage_indices().

Referenced by Column< Array< T > >::concatItem(), Column< Array< T > >::setItem(), and Column< Array< T > >::setNull().

585  {
587  if (index < 0 || index >= itemsCount()) {
588  return IndexError;
589  }
590  int64_t* storage_indices = VarlenArray_storage_indices();
591  if (storage_indices[index] >= 0) {
593  }
594  return setNullNoValidation(index);
595  }
596  return UnknownFormatError;
597  }
HOST DEVICE FlatBufferFormat format() const
Definition: FlatBuffer.h:213
HOST DEVICE int64_t itemsCount() const
Definition: FlatBuffer.h:412
HOST DEVICE int64_t * VarlenArray_storage_indices()
Definition: FlatBuffer.h:468
Status setNullNoValidation(int64_t index)
Definition: FlatBuffer.h:600

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Status FlatBufferManager::setNullNoValidation ( int64_t  index)
inline

Definition at line 600 of file FlatBuffer.h.

References Success, VarlenArray_compressed_indices(), VarlenArray_storage_count(), and VarlenArray_storage_indices().

Referenced by setNull().

600  {
601  int64_t& storage_count = VarlenArray_storage_count();
602  int64_t* storage_indices = VarlenArray_storage_indices();
603  int64_t* compressed_indices = VarlenArray_compressed_indices();
604  const int64_t cindex = compressed_indices[storage_count];
605  storage_indices[index] = storage_count;
606  compressed_indices[storage_count] = -(cindex + 1);
607  compressed_indices[storage_count + 1] = cindex;
608  storage_count++;
609  return Success;
610  }
int64_t & VarlenArray_storage_count()
Definition: FlatBuffer.h:440
HOST DEVICE int64_t * VarlenArray_compressed_indices()
Definition: FlatBuffer.h:458
HOST DEVICE int64_t * VarlenArray_storage_indices()
Definition: FlatBuffer.h:468

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

HOST DEVICE int64_t* FlatBufferManager::VarlenArray_compressed_indices ( )
inline

Definition at line 458 of file FlatBuffer.h.

References buffer.

Referenced by concatItem(), getItem(), initializeVarlenArray(), isNull(), setItem(), setItemNoValidation(), setNullNoValidation(), and VarlenArray_nof_values().

458  {
459  return reinterpret_cast<int64_t*>(
460  buffer + ((const int64_t*)buffer)[VarlenArrayHeader::COMPRESSED_INDICES_OFFSET]);
461  }

+ Here is the caller graph for this function:

const int64_t* FlatBufferManager::VarlenArray_compressed_indices ( ) const
inline

Definition at line 462 of file FlatBuffer.h.

References buffer.

462  {
463  return reinterpret_cast<const int64_t*>(
464  buffer + ((const int64_t*)buffer)[VarlenArrayHeader::COMPRESSED_INDICES_OFFSET]);
465  }
int64_t FlatBufferManager::VarlenArray_max_nof_values ( ) const
inline

Definition at line 423 of file FlatBuffer.h.

References buffer.

Referenced by VarlenArray_values_buffer_size().

423  {
424  return ((int64_t*)buffer)[VarlenArrayHeader::MAX_NOF_VALUES];
425  }

+ Here is the caller graph for this function:

int64_t FlatBufferManager::VarlenArray_nof_values ( ) const
inline

Definition at line 428 of file FlatBuffer.h.

References VarlenArray_compressed_indices(), and VarlenArray_storage_count().

428  {
429  const int64_t storage_count = VarlenArray_storage_count();
430  const int64_t* compressed_indices = VarlenArray_compressed_indices();
431  return compressed_indices[storage_count];
432  }
int64_t & VarlenArray_storage_count()
Definition: FlatBuffer.h:440
HOST DEVICE int64_t * VarlenArray_compressed_indices()
Definition: FlatBuffer.h:458

+ Here is the call graph for this function:

int64_t& FlatBufferManager::VarlenArray_storage_count ( )
inline

Definition at line 440 of file FlatBuffer.h.

References buffer.

Referenced by concatItem(), setItem(), setItemNoValidation(), setNullNoValidation(), and VarlenArray_nof_values().

440  {
441  return ((int64_t*)buffer)[VarlenArrayHeader::STORAGE_COUNT];
442  }

+ Here is the caller graph for this function:

int64_t& FlatBufferManager::VarlenArray_storage_count ( ) const
inline

Definition at line 444 of file FlatBuffer.h.

References buffer.

444  {
445  return ((int64_t*)buffer)[VarlenArrayHeader::STORAGE_COUNT];
446  }
HOST DEVICE int64_t* FlatBufferManager::VarlenArray_storage_indices ( )
inline

Definition at line 468 of file FlatBuffer.h.

References buffer.

Referenced by concatItem(), getItem(), initializeVarlenArray(), isNull(), setItem(), setItemNoValidation(), setNull(), and setNullNoValidation().

468  {
469  return reinterpret_cast<int64_t*>(
470  buffer + ((const int64_t*)buffer)[VarlenArrayHeader::STORAGE_INDICES_OFFSET]);
471  }

+ Here is the caller graph for this function:

const int64_t* FlatBufferManager::VarlenArray_storage_indices ( ) const
inline

Definition at line 472 of file FlatBuffer.h.

References buffer.

472  {
473  return reinterpret_cast<const int64_t*>(
474  buffer + ((const int64_t*)buffer)[VarlenArrayHeader::STORAGE_INDICES_OFFSET]);
475  }
HOST DEVICE int8_t* FlatBufferManager::VarlenArray_values ( )
inline

Definition at line 449 of file FlatBuffer.h.

References buffer.

Referenced by concatItem(), getItem(), and setItemNoValidation().

449  {
450  return buffer + ((const int64_t*)buffer)[VarlenArrayHeader::VALUES_OFFSET];
451  }

+ Here is the caller graph for this function:

const int8_t* FlatBufferManager::VarlenArray_values ( ) const
inline

Definition at line 453 of file FlatBuffer.h.

References buffer.

453  {
454  return buffer + ((const int64_t*)buffer)[VarlenArrayHeader::VALUES_OFFSET];
455  }
int64_t FlatBufferManager::VarlenArray_values_buffer_size ( ) const
inline

Definition at line 435 of file FlatBuffer.h.

References _align_to_int64(), dtypeSize(), and VarlenArray_max_nof_values().

Referenced by setItem().

435  {
437  }
HOST DEVICE int64_t dtypeSize() const
Definition: FlatBuffer.h:420
int64_t VarlenArray_max_nof_values() const
Definition: FlatBuffer.h:423
int64_t _align_to_int64(int64_t addr)
Definition: FlatBuffer.h:143

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Member Data Documentation


The documentation for this struct was generated from the following file: