OmniSciDB  72c90bc290
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Namespaces | Functions
ColumnarResults.cpp File Reference
#include "ColumnarResults.h"
#include "Descriptors/RowSetMemoryOwner.h"
#include "ErrorHandling.h"
#include "Execute.h"
#include "Geospatial/Compression.h"
#include "Geospatial/Types.h"
#include "Shared/Intervals.h"
#include "Shared/likely.h"
#include "Shared/sqltypes.h"
#include "Shared/thread_count.h"
#include <tbb/parallel_reduce.h>
#include <atomic>
#include <future>
#include <numeric>
+ Include dependency graph for ColumnarResults.cpp:

Go to the source code of this file.

Namespaces

 anonymous_namespace{ColumnarResults.cpp}
 

Functions

int64_t anonymous_namespace{ColumnarResults.cpp}::fixed_encoding_nullable_val (const int64_t val, const SQLTypeInfo &type_info)
 
std::vector< size_t > anonymous_namespace{ColumnarResults.cpp}::get_padded_target_sizes (const ResultSet &rows, const std::vector< SQLTypeInfo > &target_types)
 
int64_t anonymous_namespace{ColumnarResults.cpp}::toBuffer (const TargetValue &col_val, const SQLTypeInfo &type_info, int8_t *buf)
 
int64_t anonymous_namespace{ColumnarResults.cpp}::computeTotalNofValuesForColumnArray (const ResultSet &rows, const size_t column_idx)
 
template<typename TargetValue , typename TargetValuePtr >
int64_t anonymous_namespace{ColumnarResults.cpp}::computeTotalNofValuesForColumnGeoType (const ResultSet &rows, const SQLTypeInfo &ti, const size_t column_idx)
 
int64_t anonymous_namespace{ColumnarResults.cpp}::computeTotalNofValuesForColumnTextEncodingNone (const ResultSet &rows, const size_t column_idx)
 
template<size_t NDIM, typename GeospatialGeoType , typename GeoTypeTargetValue , typename GeoTypeTargetValuePtr , bool is_multi>
void writeBackCellGeoNestedArray (FlatBufferManager &m, const int64_t index, const SQLTypeInfo &ti, const TargetValue &col_val, std::mutex *write_mutex)
 
template<typename scalar_type , typename value_type >
void writeBackCellArrayScalar (FlatBufferManager &m, const size_t row_idx, const TargetValue &col_val, std::mutex *write_mutex)
 
void writeBackCellTextEncodingNone (FlatBufferManager &m, const size_t row_idx, const TargetValue &col_val, std::mutex *write_mutex)
 
void writeBackCellGeoPoint (FlatBufferManager &m, const size_t row_idx, const SQLTypeInfo &type_info, const TargetValue &col_val, std::mutex *write_mutex)
 
int64_t anonymous_namespace{ColumnarResults.cpp}::invalid_read_func (const ResultSet &rows, const size_t input_buffer_entry_idx, const size_t target_idx, const size_t slot_idx)
 
template<QueryDescriptionType QUERY_TYPE, bool COLUMNAR_OUTPUT>
int64_t anonymous_namespace{ColumnarResults.cpp}::read_float_key_baseline (const ResultSet &rows, const size_t input_buffer_entry_idx, const size_t target_idx, const size_t slot_idx)
 
template<QueryDescriptionType QUERY_TYPE, bool COLUMNAR_OUTPUT>
int64_t anonymous_namespace{ColumnarResults.cpp}::read_int64_func (const ResultSet &rows, const size_t input_buffer_entry_idx, const size_t target_idx, const size_t slot_idx)
 
template<QueryDescriptionType QUERY_TYPE, bool COLUMNAR_OUTPUT>
int64_t anonymous_namespace{ColumnarResults.cpp}::read_int32_func (const ResultSet &rows, const size_t input_buffer_entry_idx, const size_t target_idx, const size_t slot_idx)
 
template<QueryDescriptionType QUERY_TYPE, bool COLUMNAR_OUTPUT>
int64_t anonymous_namespace{ColumnarResults.cpp}::read_int16_func (const ResultSet &rows, const size_t input_buffer_entry_idx, const size_t target_idx, const size_t slot_idx)
 
template<QueryDescriptionType QUERY_TYPE, bool COLUMNAR_OUTPUT>
int64_t anonymous_namespace{ColumnarResults.cpp}::read_int8_func (const ResultSet &rows, const size_t input_buffer_entry_idx, const size_t target_idx, const size_t slot_idx)
 
template<QueryDescriptionType QUERY_TYPE, bool COLUMNAR_OUTPUT>
int64_t anonymous_namespace{ColumnarResults.cpp}::read_float_func (const ResultSet &rows, const size_t input_buffer_entry_idx, const size_t target_idx, const size_t slot_idx)
 
template<QueryDescriptionType QUERY_TYPE, bool COLUMNAR_OUTPUT>
int64_t anonymous_namespace{ColumnarResults.cpp}::read_double_func (const ResultSet &rows, const size_t input_buffer_entry_idx, const size_t target_idx, const size_t slot_idx)
 

Function Documentation

template<typename scalar_type , typename value_type >
void writeBackCellArrayScalar ( FlatBufferManager &  m,
const size_t  row_idx,
const TargetValue &  col_val,
std::mutex *  write_mutex 
)

Definition at line 726 of file ColumnarResults.cpp.

References CHECK_EQ, FlatBufferManager::getItem(), FlatBufferManager::NestedArrayItem< NDIM >::nof_sizes, FlatBufferManager::NestedArrayItem< NDIM >::nof_values, FlatBufferManager::setItem(), FlatBufferManager::setNull(), and FlatBufferManager::NestedArrayItem< NDIM >::values.

729  {
730  FlatBufferManager::Status status{};
731  const auto arr_tv = boost::get<ArrayTargetValue>(&col_val);
732  if (arr_tv->is_initialized()) {
733  const auto& vec = arr_tv->get();
734  // add a new item to flatbuffer, no initialization
735  {
736  auto lock_scope =
737  (write_mutex == nullptr ? std::unique_lock<std::mutex>()
738  : std::unique_lock<std::mutex>(*write_mutex));
739  status = m.setItem<1, false>(row_idx, nullptr, vec.size());
740  }
741  CHECK_EQ(status, FlatBufferManager::Status::Success);
742  FlatBufferManager::NestedArrayItem<1> item;
743  // retrieve the item
744  status = m.getItem(row_idx, item);
745  CHECK_EQ(status, FlatBufferManager::Status::Success);
746  CHECK_EQ(item.nof_sizes, 0); // for sanity
747  CHECK_EQ(item.nof_values, vec.size()); // for sanity
748  // initialize the item's buffer
749  scalar_type* values = reinterpret_cast<scalar_type*>(item.values);
750  size_t index = 0;
751  for (const TargetValue val : vec) {
752  const auto& scalar_val = boost::get<ScalarTargetValue>(&val);
753  values[index++] = static_cast<scalar_type>(*boost::get<value_type>(scalar_val));
754  }
755  } else {
756  // add a new NULL item to flatbuffer
757  {
758  auto lock_scope =
759  (write_mutex == nullptr ? std::unique_lock<std::mutex>()
760  : std::unique_lock<std::mutex>(*write_mutex));
761  status = m.setNull(row_idx);
762  }
763  CHECK_EQ(status, FlatBufferManager::Status::Success);
764  }
765 }
CHECK_EQ
#define CHECK_EQ(x, y)
Definition: Logger.h:301
FlatBufferManager::getItem
HOST DEVICE Status getItem(const int64_t index, NestedArrayItem< NDIM > &result)
Definition: FlatBuffer.h:1349
FlatBufferManager::setItem
HOST DEVICE Status setItem(const int64_t index, const int8_t *values_buf, const int32_t nof_values)
Definition: FlatBuffer.h:1593
TargetValue
boost::variant< ScalarTargetValue, ArrayTargetValue, GeoTargetValue, GeoTargetValuePtr > TargetValue
Definition: TargetValue.h:195
FlatBufferManager::setNull
HOST DEVICE Status setNull(int64_t index)
Definition: FlatBuffer.h:2029

+ Here is the call graph for this function:

template<size_t NDIM, typename GeospatialGeoType , typename GeoTypeTargetValue , typename GeoTypeTargetValuePtr , bool is_multi>
void writeBackCellGeoNestedArray ( FlatBufferManager &  m,
const int64_t  index,
const SQLTypeInfo &  ti,
const TargetValue &  col_val,
std::mutex *  write_mutex 
)

Definition at line 564 of file ColumnarResults.cpp.

References CHECK, CHECK_EQ, CHECK_NE, Geospatial::compress_coords(), SQLTypeInfo::get_compression(), SQLTypeInfoLite::is_geoint(), kENCODING_GEOINT, kENCODING_NONE, VarlenDatum::length, VarlenDatum::pointer, FlatBufferManager::setItem(), FlatBufferManager::setNull(), and UNREACHABLE.

Referenced by ColumnarResults::writeBackCell().

568  {
569  const SQLTypeInfoLite* ti_lite =
570  reinterpret_cast<const SQLTypeInfoLite*>(m.get_user_data_buffer());
571  CHECK(ti_lite);
572  if (ti_lite->is_geoint()) {
573  CHECK_EQ(ti.get_compression(), kENCODING_GEOINT);
574  } else {
575  CHECK_EQ(ti.get_compression(), kENCODING_NONE);
576  }
577  FlatBufferManager::Status status{};
578  if (const auto tv = boost::get<ScalarTargetValue>(&col_val)) {
579  const auto ns = boost::get<NullableString>(tv);
580  CHECK(ns);
581  const auto s_ptr = boost::get<std::string>(ns);
582  if (s_ptr == nullptr || *s_ptr == "NULL") {
583  auto lock_scope =
584  (write_mutex == nullptr ? std::unique_lock<std::mutex>()
585  : std::unique_lock<std::mutex>(*write_mutex));
586  status = m.setNull(index);
587  } else {
588  std::vector<double> coords;
589  std::vector<double> bounds;
590  std::vector<int32_t> ring_sizes;
591  std::vector<int32_t> poly_rings;
592  int64_t approx_nof_coords = 2 * std::count(s_ptr->begin(), s_ptr->end(), ',');
593  coords.reserve(approx_nof_coords);
594  bounds.reserve(4);
595  const auto gdal_wkt_ls = GeospatialGeoType(*s_ptr);
596  if constexpr (NDIM == 1) {
597  gdal_wkt_ls.getColumns(coords, bounds);
598  } else if constexpr (NDIM == 2) {
599  int64_t approx_nof_rings = std::count(s_ptr->begin(), s_ptr->end(), '(') - 1;
600  ring_sizes.reserve(approx_nof_rings);
601  gdal_wkt_ls.getColumns(coords, ring_sizes, bounds);
602  } else if constexpr (NDIM == 3) {
603  int64_t approx_nof_rings = std::count(s_ptr->begin(), s_ptr->end(), '(') - 1;
604  ring_sizes.reserve(approx_nof_rings);
605  poly_rings.reserve(approx_nof_rings);
606  gdal_wkt_ls.getColumns(coords, ring_sizes, poly_rings, bounds);
607  } else {
608  UNREACHABLE();
609  }
610  const std::vector<uint8_t> compressed_coords =
611  Geospatial::compress_coords(coords, ti);
612  {
613  auto lock_scope =
614  (write_mutex == nullptr ? std::unique_lock<std::mutex>()
615  : std::unique_lock<std::mutex>(*write_mutex));
616  if constexpr (NDIM == 1) {
617  status = m.setItem(index, compressed_coords);
618  } else if constexpr (NDIM == 2) {
619  status = m.setItem(index, compressed_coords, ring_sizes);
620  } else if constexpr (NDIM == 3) {
621  status = m.setItem(index, compressed_coords, ring_sizes, poly_rings);
622  } else {
623  UNREACHABLE();
624  }
625  }
626  }
627  } else if (const auto tv = boost::get<GeoTargetValuePtr>(&col_val)) {
628  const auto s = boost::get<GeoTypeTargetValuePtr>(tv);
629  CHECK(s);
630  if (s->coords_data == nullptr || s->coords_data->pointer == nullptr) {
631  status = m.setNull(index);
632  } else {
633  const VarlenDatum* d = s->coords_data.get();
634  CHECK(d);
635  CHECK(d->pointer);
636 
637  int32_t nof_values =
638  d->length / (ti_lite->is_geoint() ? 2 * sizeof(int32_t) : 2 * sizeof(double));
639  {
640  auto lock_scope =
641  (write_mutex == nullptr ? std::unique_lock<std::mutex>()
642  : std::unique_lock<std::mutex>(*write_mutex));
643  if constexpr (NDIM == 1) {
644  status = m.setItem<0, false>(index, d->pointer, nof_values);
645  } else if constexpr (NDIM == 2) {
646  VarlenDatum* r = nullptr;
647  if constexpr (is_multi) {
648  r = s->linestring_sizes_data.get();
649  } else {
650  r = s->ring_sizes_data.get();
651  }
652  status = m.setItem<1, /*check_sizes=*/false>(
653  index,
654  d->pointer,
655  nof_values,
656  reinterpret_cast<const int32_t*>(r->pointer),
657  r->length / sizeof(int32_t));
658  } else if constexpr (NDIM == 3) {
659  const VarlenDatum* r = s->ring_sizes_data.get();
660  const VarlenDatum* p = s->poly_rings_data.get();
661  status = m.setItem<2, /*check_sizes=*/false>(
662  index,
663  d->pointer,
664  nof_values,
665  reinterpret_cast<const int32_t*>(r->pointer),
666  r->length / sizeof(int32_t),
667  reinterpret_cast<const int32_t*>(p->pointer),
668  p->length / sizeof(int32_t));
669  } else {
670  UNREACHABLE();
671  }
672  }
673  }
674  } else if (const auto tv = boost::get<GeoTargetValue>(&col_val)) {
675  if (tv->get_ptr() == nullptr) {
676  auto lock_scope =
677  (write_mutex == nullptr ? std::unique_lock<std::mutex>()
678  : std::unique_lock<std::mutex>(*write_mutex));
679  status = m.setNull(index);
680  } else {
681  const auto s = boost::get<GeoTypeTargetValue>(tv->get());
682  const std::vector<double>* d = s.coords.get();
683  const std::vector<int32_t>* r = nullptr;
684  const std::vector<int32_t>* p = nullptr;
685  if constexpr (NDIM == 1) {
686  CHECK(r == nullptr);
687  CHECK(p == nullptr);
688  } else if constexpr (NDIM == 2) {
689  if constexpr (is_multi) {
690  r = s.linestring_sizes.get();
691  } else {
692  r = s.ring_sizes.get();
693  }
694  CHECK(p == nullptr);
695  } else if constexpr (NDIM == 3) {
696  r = s.ring_sizes.get();
697  p = s.poly_rings.get();
698  } else {
699  UNREACHABLE();
700  }
701  CHECK(d);
702  CHECK_NE(d->size(), 0);
703  std::vector<uint8_t> compressed_coords = Geospatial::compress_coords(*d, ti);
704  {
705  auto lock_scope =
706  (write_mutex == nullptr ? std::unique_lock<std::mutex>()
707  : std::unique_lock<std::mutex>(*write_mutex));
708  if constexpr (NDIM == 1) {
709  status = m.setItem(index, compressed_coords);
710  } else if constexpr (NDIM == 2) {
711  status = m.setItem(index, compressed_coords, *r);
712  } else if constexpr (NDIM == 3) {
713  status = m.setItem(index, compressed_coords, *r, *p);
714  } else {
715  UNREACHABLE();
716  }
717  }
718  }
719  } else {
720  UNREACHABLE();
721  }
722  CHECK_EQ(status, FlatBufferManager::Status::Success);
723 }
SQLTypeInfoLite::is_geoint
bool is_geoint() const
Definition: sqltypes_lite.h:61
CHECK_EQ
#define CHECK_EQ(x, y)
Definition: Logger.h:301
UNREACHABLE
#define UNREACHABLE()
Definition: Logger.h:338
VarlenDatum::pointer
int8_t * pointer
Definition: Datum.h:56
Geospatial::compress_coords
std::vector< uint8_t > compress_coords(const std::vector< double > &coords, const SQLTypeInfo &ti)
Definition: Compression.cpp:52
CHECK_NE
#define CHECK_NE(x, y)
Definition: Logger.h:302
FlatBufferManager::setItem
HOST DEVICE Status setItem(const int64_t index, const int8_t *values_buf, const int32_t nof_values)
Definition: FlatBuffer.h:1593
SQLTypeInfo::get_compression
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:399
CHECK
#define CHECK(condition)
Definition: Logger.h:291
FlatBufferManager::setNull
HOST DEVICE Status setNull(int64_t index)
Definition: FlatBuffer.h:2029
VarlenDatum::length
size_t length
Definition: Datum.h:55

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void writeBackCellGeoPoint ( FlatBufferManager &  m,
const size_t  row_idx,
const SQLTypeInfo &  type_info,
const TargetValue &  col_val,
std::mutex *  write_mutex 
)
inline

Definition at line 792 of file ColumnarResults.cpp.

References CHECK, CHECK_EQ, Geospatial::compress_coords(), FlatBufferManager::dtypeSize(), SQLTypeInfo::get_compression(), kENCODING_GEOINT, VarlenDatum::length, NULL_ARRAY_DOUBLE, VarlenDatum::pointer, FlatBufferManager::setItemOld(), and UNREACHABLE.

Referenced by ColumnarResults::writeBackCell().

796  {
797  FlatBufferManager::Status status{};
798  // to be deprecated, this function uses old FlatBuffer API
799  if (const auto tv = boost::get<ScalarTargetValue>(&col_val)) {
800  const auto ns = boost::get<NullableString>(tv);
801  CHECK(ns);
802  const auto s_ptr = boost::get<std::string>(ns);
803  std::vector<double> coords;
804  coords.reserve(2);
805  if (s_ptr == nullptr) {
806  coords.push_back(NULL_ARRAY_DOUBLE);
807  coords.push_back(NULL_ARRAY_DOUBLE);
808  } else {
809  const auto gdal_wkt_pt = Geospatial::GeoPoint(*s_ptr);
810  gdal_wkt_pt.getColumns(coords);
811  CHECK_EQ(coords.size(), 2);
812  }
813  std::vector<std::uint8_t> data = Geospatial::compress_coords(coords, type_info);
814  {
815  auto lock_scope =
816  (write_mutex == nullptr ? std::unique_lock<std::mutex>()
817  : std::unique_lock<std::mutex>(*write_mutex));
818  status = m.setItemOld(
819  row_idx, reinterpret_cast<const int8_t*>(data.data()), data.size());
820  }
821  CHECK_EQ(status, FlatBufferManager::Status::Success);
822  } else if (const auto tv = boost::get<GeoTargetValuePtr>(&col_val)) {
823  const auto s = boost::get<GeoPointTargetValuePtr>(tv);
824  CHECK(s);
825  VarlenDatum* d = s->coords_data.get();
826  CHECK(d);
827  CHECK_EQ(type_info.get_compression() == kENCODING_GEOINT,
828  m.getGeoPointMetadata()->is_geoint);
829  {
830  auto lock_scope =
831  (write_mutex == nullptr ? std::unique_lock<std::mutex>()
832  : std::unique_lock<std::mutex>(*write_mutex));
833  status =
834  m.setItemOld(row_idx, reinterpret_cast<const int8_t*>(d->pointer), d->length);
835  }
836  CHECK_EQ(status, FlatBufferManager::Status::Success);
837  } else if (const auto tv = boost::get<GeoTargetValue>(&col_val)) {
838  /*
839  Warning: the following code fails for NULL row values
840  because of the failure to detect the nullness correctly.
841  */
842  const auto s = boost::get<GeoPointTargetValue>(tv->get());
843  const std::vector<double>* d = s.coords.get();
844  CHECK_EQ(d->size(), 2);
845  {
846  auto lock_scope =
847  (write_mutex == nullptr ? std::unique_lock<std::mutex>()
848  : std::unique_lock<std::mutex>(*write_mutex));
849  status = m.setItemOld(
850  row_idx, reinterpret_cast<const int8_t*>(d->data()), m.dtypeSize());
851  }
852  CHECK_EQ(d->size(), 2);
853  CHECK_EQ(status, FlatBufferManager::Status::Success);
854  } else {
855  UNREACHABLE();
856  }
857 }
CHECK_EQ
#define CHECK_EQ(x, y)
Definition: Logger.h:301
FlatBufferManager::dtypeSize
HOST DEVICE int64_t dtypeSize() const
Definition: FlatBuffer.h:628
UNREACHABLE
#define UNREACHABLE()
Definition: Logger.h:338
VarlenDatum::pointer
int8_t * pointer
Definition: Datum.h:56
Geospatial::compress_coords
std::vector< uint8_t > compress_coords(const std::vector< double > &coords, const SQLTypeInfo &ti)
Definition: Compression.cpp:52
FlatBufferManager::setItemOld
Status setItemOld(const int64_t index, const int8_t *src, const int64_t size, int8_t **dest=nullptr)
Definition: FlatBuffer.h:1963
SQLTypeInfo::get_compression
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:399
NULL_ARRAY_DOUBLE
#define NULL_ARRAY_DOUBLE
Definition: InlineNullValues.h:43
CHECK
#define CHECK(condition)
Definition: Logger.h:291
VarlenDatum::length
size_t length
Definition: Datum.h:55

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void writeBackCellTextEncodingNone ( FlatBufferManager &  m,
const size_t  row_idx,
const TargetValue &  col_val,
std::mutex *  write_mutex 
)
inline

Definition at line 767 of file ColumnarResults.cpp.

References CHECK, CHECK_EQ, FlatBufferManager::setItem(), FlatBufferManager::setNull(), and UNREACHABLE.

Referenced by ColumnarResults::writeBackCell().

770  {
771  FlatBufferManager::Status status{};
772  if (const auto tv = boost::get<ScalarTargetValue>(&col_val)) {
773  const auto ns = boost::get<NullableString>(tv);
774  CHECK(ns);
775  const auto s_ptr = boost::get<std::string>(ns);
776  {
777  auto lock_scope =
778  (write_mutex == nullptr ? std::unique_lock<std::mutex>()
779  : std::unique_lock<std::mutex>(*write_mutex));
780  if (s_ptr) {
781  status = m.setItem(row_idx, *s_ptr);
782  } else {
783  status = m.setNull(row_idx);
784  }
785  }
786  CHECK_EQ(status, FlatBufferManager::Status::Success);
787  } else {
788  UNREACHABLE();
789  }
790 }
CHECK_EQ
#define CHECK_EQ(x, y)
Definition: Logger.h:301
UNREACHABLE
#define UNREACHABLE()
Definition: Logger.h:338
FlatBufferManager::setItem
HOST DEVICE Status setItem(const int64_t index, const int8_t *values_buf, const int32_t nof_values)
Definition: FlatBuffer.h:1593
CHECK
#define CHECK(condition)
Definition: Logger.h:291
FlatBufferManager::setNull
HOST DEVICE Status setNull(int64_t index)
Definition: FlatBuffer.h:2029

+ Here is the call graph for this function:

+ Here is the caller graph for this function: