OmniSciDB  cde582ebc3
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Fragmenter_Namespace Namespace Reference

Namespaces

 anonymous_namespace{InsertOrderFragmenter.cpp}
 
 anonymous_namespace{UpdelStorage.cpp}
 

Classes

class  RowDataProvider
 
struct  UpdateValuesStats
 
struct  ChunkUpdateStats
 
class  AbstractFragmenter
 
struct  InsertChunks
 
struct  InsertData
 The data to be inserted using the fragment manager. More...
 
class  FragmentInfo
 Used by Fragmenter classes to store info about each fragment - the fragment id and number of tuples(rows) currently stored by that fragment. More...
 
class  TableInfo
 
struct  ShardDataOwner
 
struct  BlockWithColumnId
 
class  InsertDataLoader
 
class  LocalInsertConnector
 
class  InsertOrderFragmenter
 The InsertOrderFragmenter is a child class of AbstractFragmenter, and fragments data in insert order. Likely the default fragmenter. More...
 
class  SortedOrderFragmenter
 
struct  ChunkToInsertDataConverter
 
struct  ScalarChunkConverter
 
struct  FixedLenArrayChunkConverter
 
struct  ArrayChunkConverter
 
struct  StringChunkConverter
 
struct  DateChunkConverter
 

Enumerations

enum  FragmenterType { INSERT_ORDER = 0 }
 

Functions

template<typename SRC >
std::vector< std::vector
< size_t > > 
compute_row_indices_of_shards (size_t shard_count, size_t leaf_count, size_t row_count, SRC *src, bool duplicated_key_value)
 
template<typename T >
size_t indexOf (std::vector< T > &vec, T val)
 
bool isStringVectorData (const ColumnDescriptor *cd)
 
bool isDatumVectorData (const ColumnDescriptor *cd)
 
size_t size_of_raw_column (const Catalog_Namespace::Catalog &cat, const ColumnDescriptor *cd, const bool get_logical_size=true)
 
std::vector< std::vector
< size_t > > 
compute_row_indices_of_shards (const Catalog_Namespace::Catalog &cat, size_t leaf_count, const InsertChunks &insert_chunks)
 
std::vector< std::vector
< size_t > > 
computeRowIndicesOfShards (const Catalog_Namespace::Catalog &cat, size_t leafCount, InsertData &insert_data)
 
template<typename T >
void copyColumnDataOfShard (const std::vector< size_t > &rowIndices, T *src, T *dst)
 
BlockWithColumnId copyColumnDataOfShard (const Catalog_Namespace::Catalog &cat, ShardDataOwner &dataOwner, const std::vector< size_t > &rowIndices, const ColumnDescriptor *pCol, size_t columnIndex, DataBlockPtr dataBlock, bool is_default)
 
std::pair< std::list
< std::unique_ptr
< foreign_storage::ForeignStorageBuffer >
>, InsertChunks
copy_data_of_shard (const Catalog_Namespace::Catalog &cat, const InsertChunks &insert_chunks, int shardTableIndex, const std::vector< size_t > &rowIndices)
 
InsertData copyDataOfShard (const Catalog_Namespace::Catalog &cat, ShardDataOwner &dataOwner, InsertData &insert_data, int shardTableIndex, const std::vector< size_t > &rowIndices)
 
template<typename T >
void shuffleByIndexesImpl (const std::vector< size_t > &indexes, T *buffer)
 
template<typename T >
void shuffleByIndexesImpl (const std::vector< size_t > &indexes, std::vector< T > &buffer)
 
void shuffleByIndexes (const ColumnDescriptor *cd, const std::vector< size_t > &indexes, DataBlockPtr &data)
 
template<typename T >
void sortIndexesImpl (std::vector< size_t > &indexes, const T *buffer)
 
void sortIndexesImpl (std::vector< size_t > &indexes, const std::vector< std::string > &buffer)
 
void sortIndexesImpl (std::vector< size_t > &indexes, const std::vector< ArrayDatum > &buffer)
 
void sortIndexes (const ColumnDescriptor *cd, std::vector< size_t > &indexes, const DataBlockPtr &data)
 
void wait_cleanup_threads (std::vector< std::future< void >> &threads)
 
bool is_integral (const SQLTypeInfo &t)
 
static int get_chunks (const Catalog_Namespace::Catalog *catalog, const TableDescriptor *td, const FragmentInfo &fragment, const Data_Namespace::MemoryLevel memory_level, std::vector< std::shared_ptr< Chunk_NS::Chunk >> &chunks)
 
template<typename T >
static void set_chunk_stats (const SQLTypeInfo &col_type, int8_t *data_addr, bool &has_null, T &min, T &max)
 
static void set_chunk_metadata (const Catalog_Namespace::Catalog *catalog, FragmentInfo &fragment, const std::shared_ptr< Chunk_NS::Chunk > &chunk, const size_t nrows_to_keep, UpdelRoll &updel_roll)
 
size_t get_null_padding (bool is_varlen_array, const std::vector< uint64_t > &frag_offsets, const StringOffsetT *index_array, size_t fragment_row_count)
 
std::set< size_t > get_var_len_null_array_indexes (const SQLTypeInfo sql_type_info, const std::vector< uint64_t > &frag_offsets, const StringOffsetT *index_array, size_t fragment_row_count)
 
StringOffsetT get_buffer_offset (bool is_varlen_array, const StringOffsetT *index_array, size_t index)
 

Enumeration Type Documentation

stores the type of a child class of AbstractTableFragmenter

Enumerator
INSERT_ORDER 

Definition at line 45 of file Fragmenter.h.

45  {
46  INSERT_ORDER = 0 // these values persist in catalog. make explicit
47 };

Function Documentation

template<typename SRC >
std::vector<std::vector<size_t> > Fragmenter_Namespace::compute_row_indices_of_shards ( size_t  shard_count,
size_t  leaf_count,
size_t  row_count,
SRC *  src,
bool  duplicated_key_value 
)

Definition at line 35 of file InsertDataLoader.cpp.

References SHARD_FOR_KEY.

Referenced by compute_row_indices_of_shards(), computeRowIndicesOfShards(), and Fragmenter_Namespace::InsertDataLoader::insertChunks().

40  {
41  const auto n_shard_tables = shard_count * leaf_count;
42  std::vector<std::vector<size_t>> row_indices_of_shards(n_shard_tables);
43  if (!duplicated_key_value) {
44  for (size_t row = 0; row < row_count; row++) {
45  // expecting unsigned data
46  // thus, no need for double remainder
47  auto shard_id = (std::is_unsigned<SRC>::value)
48  ? src[row] % n_shard_tables
49  : SHARD_FOR_KEY(src[row], n_shard_tables);
50  row_indices_of_shards[shard_id].push_back(row);
51  }
52  } else {
53  auto shard_id = (std::is_unsigned<SRC>::value)
54  ? src[0] % n_shard_tables
55  : SHARD_FOR_KEY(src[0], n_shard_tables);
56  row_indices_of_shards[shard_id].reserve(row_count);
57  for (size_t row = 0; row < row_count; row++) {
58  row_indices_of_shards[shard_id].push_back(row);
59  }
60  }
61 
62  return row_indices_of_shards;
63 }
#define SHARD_FOR_KEY(key, num_shards)
Definition: shard_key.h:20

+ Here is the caller graph for this function:

std::vector<std::vector<size_t> > Fragmenter_Namespace::compute_row_indices_of_shards ( const Catalog_Namespace::Catalog cat,
size_t  leaf_count,
const InsertChunks &  insert_chunks 
)

Definition at line 122 of file InsertDataLoader.cpp.

References CHECK, Fragmenter_Namespace::InsertChunks::chunks, compute_row_indices_of_shards(), Chunk_NS::Chunk::getBuffer(), Data_Namespace::AbstractBuffer::getEncoder(), Catalog_Namespace::Catalog::getMetadataForTable(), Encoder::getNumElems(), Catalog_Namespace::Catalog::getShardColumnMetadataForTable(), isDatumVectorData(), isStringVectorData(), size_of_raw_column(), Fragmenter_Namespace::InsertChunks::table_id, and UNREACHABLE.

125  {
126  const auto* td = cat.getMetadataForTable(insert_chunks.table_id);
127  const auto* shard_cd = cat.getShardColumnMetadataForTable(td);
128  auto find_it = insert_chunks.chunks.find(shard_cd->columnId);
129  CHECK(find_it != insert_chunks.chunks.end());
130  Chunk_NS::Chunk& shard_chunk = *find_it->second;
131  auto row_count = shard_chunk.getBuffer()->getEncoder()->getNumElems();
132  auto shard_count = td->nShards;
133 
134  CHECK(!isStringVectorData(shard_cd));
135  CHECK(!isDatumVectorData(shard_cd));
136 
137  auto memory_ptr = shard_chunk.getBuffer()->getMemoryPtr();
138  CHECK(memory_ptr);
139  switch (size_of_raw_column(cat, shard_cd, false)) {
140  case 1:
141  return compute_row_indices_of_shards(shard_count,
142  leaf_count,
143  row_count,
144  reinterpret_cast<uint8_t*>(memory_ptr),
145  false);
146  case 2:
147  return compute_row_indices_of_shards(shard_count,
148  leaf_count,
149  row_count,
150  reinterpret_cast<uint16_t*>(memory_ptr),
151  false);
152  case 4:
153  return compute_row_indices_of_shards(shard_count,
154  leaf_count,
155  row_count,
156  reinterpret_cast<uint32_t*>(memory_ptr),
157  false);
158  case 8:
159  return compute_row_indices_of_shards(shard_count,
160  leaf_count,
161  row_count,
162  reinterpret_cast<uint64_t*>(memory_ptr),
163  false);
164  default:
165  UNREACHABLE() << "unexpected data element size of column";
166  }
167  return {};
168 }
#define UNREACHABLE()
Definition: Logger.h:266
const ColumnDescriptor * getShardColumnMetadataForTable(const TableDescriptor *td) const
Definition: Catalog.cpp:4583
bool isStringVectorData(const ColumnDescriptor *cd)
std::string chunks
size_t size_of_raw_column(const Catalog_Namespace::Catalog &cat, const ColumnDescriptor *cd, const bool get_logical_size=true)
size_t getNumElems() const
Definition: Encoder.h:284
std::vector< std::vector< size_t > > compute_row_indices_of_shards(size_t shard_count, size_t leaf_count, size_t row_count, SRC *src, bool duplicated_key_value)
AbstractBuffer * getBuffer() const
Definition: Chunk.h:146
bool isDatumVectorData(const ColumnDescriptor *cd)
#define CHECK(condition)
Definition: Logger.h:222
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.

+ Here is the call graph for this function:

std::vector<std::vector<size_t> > Fragmenter_Namespace::computeRowIndicesOfShards ( const Catalog_Namespace::Catalog cat,
size_t  leafCount,
InsertData &  insert_data 
)

Definition at line 170 of file InsertDataLoader.cpp.

References CHECK, Fragmenter_Namespace::InsertData::columnIds, compute_row_indices_of_shards(), Fragmenter_Namespace::InsertData::data, Catalog_Namespace::Catalog::getMetadataForTable(), Catalog_Namespace::Catalog::getShardColumnMetadataForTable(), indexOf(), Fragmenter_Namespace::InsertData::is_default, isDatumVectorData(), isStringVectorData(), DataBlockPtr::numbersPtr, Fragmenter_Namespace::InsertData::numRows, size_of_raw_column(), and Fragmenter_Namespace::InsertData::tableId.

Referenced by Fragmenter_Namespace::InsertDataLoader::insertData().

173  {
174  const auto* td = cat.getMetadataForTable(insert_data.tableId);
175  const auto* shard_cd = cat.getShardColumnMetadataForTable(td);
176  auto shardDataBlockIndex = indexOf(insert_data.columnIds, shard_cd->columnId);
177  DataBlockPtr& shardDataBlock = insert_data.data[shardDataBlockIndex];
178  auto rowCount = insert_data.numRows;
179  auto shardCount = td->nShards;
180 
181  CHECK(!isStringVectorData(shard_cd));
182  CHECK(!isDatumVectorData(shard_cd));
183 
184  CHECK(insert_data.is_default.size() == insert_data.columnIds.size());
185  bool is_default = insert_data.is_default[shardDataBlockIndex];
186  switch (size_of_raw_column(cat, shard_cd)) {
187  case 1:
189  shardCount,
190  leafCount,
191  rowCount,
192  reinterpret_cast<uint8_t*>(shardDataBlock.numbersPtr),
193  is_default);
194  case 2:
196  shardCount,
197  leafCount,
198  rowCount,
199  reinterpret_cast<uint16_t*>(shardDataBlock.numbersPtr),
200  is_default);
201  case 4:
203  shardCount,
204  leafCount,
205  rowCount,
206  reinterpret_cast<uint32_t*>(shardDataBlock.numbersPtr),
207  is_default);
208  case 8:
210  shardCount,
211  leafCount,
212  rowCount,
213  reinterpret_cast<uint64_t*>(shardDataBlock.numbersPtr),
214  is_default);
215  }
216  throw std::runtime_error("Unexpected data block element size");
217 }
const ColumnDescriptor * getShardColumnMetadataForTable(const TableDescriptor *td) const
Definition: Catalog.cpp:4583
bool isStringVectorData(const ColumnDescriptor *cd)
size_t size_of_raw_column(const Catalog_Namespace::Catalog &cat, const ColumnDescriptor *cd, const bool get_logical_size=true)
std::vector< std::vector< size_t > > compute_row_indices_of_shards(size_t shard_count, size_t leaf_count, size_t row_count, SRC *src, bool duplicated_key_value)
size_t indexOf(std::vector< T > &vec, T val)
bool isDatumVectorData(const ColumnDescriptor *cd)
#define CHECK(condition)
Definition: Logger.h:222
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
int8_t * numbersPtr
Definition: sqltypes.h:226

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::pair<std::list<std::unique_ptr<foreign_storage::ForeignStorageBuffer> >, InsertChunks> Fragmenter_Namespace::copy_data_of_shard ( const Catalog_Namespace::Catalog cat,
const InsertChunks &  insert_chunks,
int  shardTableIndex,
const std::vector< size_t > &  rowIndices 
)

Definition at line 296 of file InsertDataLoader.cpp.

References CHECK_EQ, Fragmenter_Namespace::InsertChunks::chunks, Fragmenter_Namespace::InsertChunks::db_id, Catalog_Namespace::Catalog::getMetadataForTable(), Catalog_Namespace::Catalog::getPhysicalTablesDescriptors(), Fragmenter_Namespace::InsertChunks::table_id, and Fragmenter_Namespace::InsertChunks::valid_row_indices.

Referenced by Fragmenter_Namespace::InsertDataLoader::insertChunks().

299  {
300  const auto* table = cat.getMetadataForTable(insert_chunks.table_id);
301  const auto* physical_table = cat.getPhysicalTablesDescriptors(table)[shardTableIndex];
302 
303  InsertChunks insert_chunks_for_shard{
304  physical_table->tableId, insert_chunks.db_id, {}, {}};
305 
306  std::list<std::unique_ptr<foreign_storage::ForeignStorageBuffer>> buffers;
307 
308  for (const auto& [column_id, chunk] : insert_chunks.chunks) {
309  auto column = chunk->getColumnDesc();
310  insert_chunks_for_shard.chunks[column_id] = std::make_shared<Chunk_NS::Chunk>(column);
311  auto& chunk_for_shard = *insert_chunks_for_shard.chunks[column_id];
312  chunk_for_shard.setBuffer(
313  buffers.emplace_back(std::make_unique<foreign_storage::ForeignStorageBuffer>())
314  .get());
315  if (column->columnType.is_varlen_indeed()) { // requires an index buffer
316  chunk_for_shard.setIndexBuffer(
317  buffers.emplace_back(std::make_unique<foreign_storage::ForeignStorageBuffer>())
318  .get());
319  }
320  chunk_for_shard.initEncoder();
321  chunk_for_shard.appendEncodedDataAtIndices(*chunk, rowIndices);
322  CHECK_EQ(chunk_for_shard.getBuffer()->getEncoder()->getNumElems(), rowIndices.size());
323  }
324 
325  // mark which row indices are valid for import
326  auto row_count = rowIndices.size();
327  insert_chunks_for_shard.valid_row_indices.reserve(row_count);
328  for (size_t irow = 0; irow < row_count; ++irow) {
329  auto row_index = rowIndices[irow];
330  if (std::binary_search(insert_chunks.valid_row_indices.begin(),
331  insert_chunks.valid_row_indices.end(),
332  row_index)) {
333  insert_chunks_for_shard.valid_row_indices.emplace_back(irow);
334  }
335  }
336 
337  return {std::move(buffers), insert_chunks_for_shard};
338 }
#define CHECK_EQ(x, y)
Definition: Logger.h:230
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4601
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename T >
void Fragmenter_Namespace::copyColumnDataOfShard ( const std::vector< size_t > &  rowIndices,
T *  src,
T *  dst 
)

Definition at line 220 of file InsertDataLoader.cpp.

Referenced by copyColumnDataOfShard(), and copyDataOfShard().

220  {
221  for (size_t row = 0; row < rowIndices.size(); row++) {
222  auto srcRowIndex = rowIndices[row];
223  dst[row] = src[srcRowIndex];
224  }
225 }

+ Here is the caller graph for this function:

BlockWithColumnId Fragmenter_Namespace::copyColumnDataOfShard ( const Catalog_Namespace::Catalog cat,
ShardDataOwner &  dataOwner,
const std::vector< size_t > &  rowIndices,
const ColumnDescriptor pCol,
size_t  columnIndex,
DataBlockPtr  dataBlock,
bool  is_default 
)

Definition at line 233 of file InsertDataLoader.cpp.

References Fragmenter_Namespace::ShardDataOwner::arrayData, DataBlockPtr::arraysPtr, ColumnDescriptor::columnId, copyColumnDataOfShard(), isDatumVectorData(), isStringVectorData(), DataBlockPtr::numbersPtr, Fragmenter_Namespace::ShardDataOwner::rawData, size_of_raw_column(), Fragmenter_Namespace::ShardDataOwner::stringData, and DataBlockPtr::stringsPtr.

239  {
240  DataBlockPtr ret;
241  std::vector<size_t> single_row_idx({0ul});
242  const std::vector<size_t>& rows = is_default ? single_row_idx : rowIndices;
243  if (isStringVectorData(pCol)) {
244  auto& data = dataOwner.stringData[columnIndex];
245  data.resize(rows.size());
246  copyColumnDataOfShard(rows, &(*(dataBlock.stringsPtr))[0], &data[0]);
247  ret.stringsPtr = &data;
248 
249  } else if (isDatumVectorData(pCol)) {
250  auto& data = dataOwner.arrayData[columnIndex];
251  data.resize(rows.size());
252  copyColumnDataOfShard(rows, &(*(dataBlock.arraysPtr))[0], &data[0]);
253  ret.arraysPtr = &data;
254 
255  } else {
256  auto rawArrayElementSize = size_of_raw_column(cat, pCol);
257  auto& data = dataOwner.rawData[columnIndex];
258  data.resize(rows.size() * rawArrayElementSize);
259 
260  switch (rawArrayElementSize) {
261  case 1: {
263  reinterpret_cast<uint8_t*>(dataBlock.numbersPtr),
264  reinterpret_cast<uint8_t*>(&data[0]));
265  break;
266  }
267  case 2: {
269  reinterpret_cast<uint16_t*>(dataBlock.numbersPtr),
270  reinterpret_cast<uint16_t*>(&data[0]));
271  break;
272  }
273  case 4: {
275  reinterpret_cast<uint32_t*>(dataBlock.numbersPtr),
276  reinterpret_cast<uint32_t*>(&data[0]));
277  break;
278  }
279  case 8: {
281  reinterpret_cast<uint64_t*>(dataBlock.numbersPtr),
282  reinterpret_cast<uint64_t*>(&data[0]));
283  break;
284  }
285  default:
286  throw std::runtime_error("Unexpected data block element size");
287  }
288 
289  ret.numbersPtr = reinterpret_cast<int8_t*>(&data[0]);
290  }
291 
292  return {pCol->columnId, ret, is_default};
293 }
std::vector< std::string > * stringsPtr
Definition: sqltypes.h:227
std::vector< ArrayDatum > * arraysPtr
Definition: sqltypes.h:228
bool isStringVectorData(const ColumnDescriptor *cd)
size_t size_of_raw_column(const Catalog_Namespace::Catalog &cat, const ColumnDescriptor *cd, const bool get_logical_size=true)
void copyColumnDataOfShard(const std::vector< size_t > &rowIndices, T *src, T *dst)
bool isDatumVectorData(const ColumnDescriptor *cd)
int8_t * numbersPtr
Definition: sqltypes.h:226

+ Here is the call graph for this function:

InsertData Fragmenter_Namespace::copyDataOfShard ( const Catalog_Namespace::Catalog cat,
ShardDataOwner &  dataOwner,
InsertData &  insert_data,
int  shardTableIndex,
const std::vector< size_t > &  rowIndices 
)

Definition at line 340 of file InsertDataLoader.cpp.

References Fragmenter_Namespace::ShardDataOwner::arrayData, threading_serial::async(), cat(), Fragmenter_Namespace::InsertData::columnIds, copyColumnDataOfShard(), Fragmenter_Namespace::InsertData::data, Fragmenter_Namespace::InsertData::databaseId, Catalog_Namespace::Catalog::getAllColumnMetadataForTable(), Catalog_Namespace::Catalog::getMetadataForTable(), Catalog_Namespace::Catalog::getPhysicalTablesDescriptors(), indexOf(), Fragmenter_Namespace::InsertData::is_default, Fragmenter_Namespace::InsertData::numRows, Fragmenter_Namespace::ShardDataOwner::rawData, Fragmenter_Namespace::ShardDataOwner::stringData, and Fragmenter_Namespace::InsertData::tableId.

Referenced by Fragmenter_Namespace::InsertDataLoader::insertData().

344  {
345  const auto* td = cat.getMetadataForTable(insert_data.tableId);
346  const auto* ptd = cat.getPhysicalTablesDescriptors(td)[shardTableIndex];
347 
348  InsertData shardData;
349  shardData.databaseId = insert_data.databaseId;
350  shardData.tableId = ptd->tableId;
351  shardData.numRows = rowIndices.size();
352 
353  std::vector<const ColumnDescriptor*> pCols;
354  std::vector<int> lCols;
355 
356  {
357  auto logicalColumns = cat.getAllColumnMetadataForTable(td->tableId, true, true, true);
358  for (const auto& cd : logicalColumns) {
359  lCols.push_back(cd->columnId);
360  }
361 
362  auto physicalColumns =
363  cat.getAllColumnMetadataForTable(ptd->tableId, true, true, true);
364  for (const auto& cd : physicalColumns) {
365  pCols.push_back(cd);
366  }
367  }
368 
369  for (size_t col = 0; col < insert_data.columnIds.size(); col++) {
370  dataOwner.arrayData.emplace_back();
371  dataOwner.rawData.emplace_back();
372  dataOwner.stringData.emplace_back();
373  }
374 
375  auto copycat = [&cat, &dataOwner, &rowIndices, &lCols, &pCols, &insert_data](int col) {
376  const auto lColId = insert_data.columnIds[col];
377  const auto pCol = pCols[indexOf(lCols, lColId)];
378  return copyColumnDataOfShard(cat,
379  dataOwner,
380  rowIndices,
381  pCol,
382  col,
383  insert_data.data[col],
384  insert_data.is_default[col]);
385  };
386 
387  std::vector<std::future<BlockWithColumnId>> worker_threads;
388  for (size_t col = 0; col < insert_data.columnIds.size(); col++) {
389  worker_threads.push_back(std::async(std::launch::async, copycat, col));
390  }
391 
392  for (auto& child : worker_threads) {
393  child.wait();
394  }
395 
396  for (auto& child : worker_threads) {
397  auto shardColumnData = child.get();
398  shardData.columnIds.push_back(shardColumnData.columnId);
399  shardData.data.push_back(shardColumnData.block);
400  shardData.is_default.push_back(shardColumnData.is_default);
401  }
402 
403  return shardData;
404 }
std::string cat(Ts &&...args)
future< Result > async(Fn &&fn, Args &&...args)
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4601
size_t indexOf(std::vector< T > &vec, T val)
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:2228
void copyColumnDataOfShard(const std::vector< size_t > &rowIndices, T *src, T *dst)
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

StringOffsetT Fragmenter_Namespace::get_buffer_offset ( bool  is_varlen_array,
const StringOffsetT index_array,
size_t  index 
)

Definition at line 1179 of file UpdelStorage.cpp.

References CHECK.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::vacuum_varlen_rows().

1181  {
1182  auto offset = index_array[index];
1183  if (offset < 0) {
1184  // Variable length arrays encode null arrays as negative offsets
1185  CHECK(is_varlen_array);
1186  offset = -offset;
1187  }
1188  return offset;
1189 }
#define CHECK(condition)
Definition: Logger.h:222

+ Here is the caller graph for this function:

static int Fragmenter_Namespace::get_chunks ( const Catalog_Namespace::Catalog catalog,
const TableDescriptor td,
const FragmentInfo &  fragment,
const Data_Namespace::MemoryLevel  memory_level,
std::vector< std::shared_ptr< Chunk_NS::Chunk >> &  chunks 
)
static

Definition at line 75 of file UpdelStorage.cpp.

References CHECK, Catalog_Namespace::DBMetadata::dbId, Fragmenter_Namespace::FragmentInfo::fragmentId, Chunk_NS::Chunk::getChunk(), Fragmenter_Namespace::FragmentInfo::getChunkMetadataMapPhysical(), Catalog_Namespace::Catalog::getCurrentDB(), Catalog_Namespace::Catalog::getDataMgr(), Catalog_Namespace::Catalog::getMetadataForColumn(), TableDescriptor::nColumns, and TableDescriptor::tableId.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::updateColumns().

79  {
80  for (int cid = 1, nc = 0; nc < td->nColumns; ++cid) {
81  if (const auto cd = catalog->getMetadataForColumn(td->tableId, cid)) {
82  ++nc;
83  if (!cd->isVirtualCol) {
84  auto chunk_meta_it = fragment.getChunkMetadataMapPhysical().find(cid);
85  CHECK(chunk_meta_it != fragment.getChunkMetadataMapPhysical().end());
86  ChunkKey chunk_key{
87  catalog->getCurrentDB().dbId, td->tableId, cid, fragment.fragmentId};
88  auto chunk = Chunk_NS::Chunk::getChunk(cd,
89  &catalog->getDataMgr(),
90  chunk_key,
91  memory_level,
92  0,
93  chunk_meta_it->second->numBytes,
94  chunk_meta_it->second->numElements);
95  chunks.push_back(chunk);
96  }
97  }
98  }
99  return chunks.size();
100 }
std::vector< int > ChunkKey
Definition: types.h:36
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:243
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:242
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
#define CHECK(condition)
Definition: Logger.h:222
static std::shared_ptr< Chunk > getChunk(const ColumnDescriptor *cd, DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId, const size_t num_bytes, const size_t num_elems, const bool pinnable=true)
Definition: Chunk.cpp:31

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

size_t Fragmenter_Namespace::get_null_padding ( bool  is_varlen_array,
const std::vector< uint64_t > &  frag_offsets,
const StringOffsetT index_array,
size_t  fragment_row_count 
)

Definition at line 1116 of file UpdelStorage.cpp.

References CHECK_GT, CHECK_LT, and ArrayNoneEncoder::DEFAULT_NULL_PADDING_SIZE.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::vacuum_varlen_rows().

1119  {
1120  if (is_varlen_array) {
1121  size_t first_non_deleted_row_index{0};
1122  for (auto deleted_offset : frag_offsets) {
1123  if (first_non_deleted_row_index < deleted_offset) {
1124  break;
1125  } else {
1126  first_non_deleted_row_index++;
1127  }
1128  }
1129  CHECK_LT(first_non_deleted_row_index, fragment_row_count);
1130  if (first_non_deleted_row_index == 0) {
1131  // If the first row in the fragment is not deleted, then the first offset in the
1132  // index buffer/array already contains expected padding.
1133  return index_array[0];
1134  } else {
1135  // If the first non-deleted element is a null array (indentified by a negative
1136  // offset), get a padding value for the chunk buffer.
1137  if (index_array[first_non_deleted_row_index + 1] < 0) {
1138  size_t first_non_zero_offset{0};
1139  for (size_t i = 0; i <= first_non_deleted_row_index; i++) {
1140  if (index_array[i] != 0) {
1141  first_non_zero_offset = index_array[i];
1142  break;
1143  }
1144  }
1145  CHECK_GT(first_non_zero_offset, static_cast<size_t>(0));
1147  first_non_zero_offset);
1148  } else {
1149  return 0;
1150  }
1151  }
1152  } else {
1153  return 0;
1154  }
1155 }
#define CHECK_GT(x, y)
Definition: Logger.h:234
#define CHECK_LT(x, y)
Definition: Logger.h:232
static constexpr size_t DEFAULT_NULL_PADDING_SIZE

+ Here is the caller graph for this function:

std::set<size_t> Fragmenter_Namespace::get_var_len_null_array_indexes ( const SQLTypeInfo  sql_type_info,
const std::vector< uint64_t > &  frag_offsets,
const StringOffsetT index_array,
size_t  fragment_row_count 
)

Definition at line 1158 of file UpdelStorage.cpp.

References SQLTypeInfo::get_notnull(), and SQLTypeInfo::is_varlen_array().

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::vacuum_varlen_rows().

1161  {
1162  std::set<size_t> null_array_indexes;
1163  if (sql_type_info.is_varlen_array() && !sql_type_info.get_notnull()) {
1164  size_t frag_offset_index{0};
1165  size_t vacuum_offset{0};
1166  for (size_t i = 0; i < fragment_row_count; i++) {
1167  if (frag_offset_index < frag_offsets.size() &&
1168  i == frag_offsets[frag_offset_index]) {
1169  frag_offset_index++;
1170  vacuum_offset++;
1171  } else if (index_array[i + 1] < 0) {
1172  null_array_indexes.emplace(i - vacuum_offset);
1173  }
1174  }
1175  }
1176  return null_array_indexes;
1177 }
bool is_varlen_array() const
Definition: sqltypes.h:519
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:336

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename T >
size_t Fragmenter_Namespace::indexOf ( std::vector< T > &  vec,
val 
)

Definition at line 66 of file InsertDataLoader.cpp.

References CHECK.

Referenced by org.apache.calcite.sql2rel.SqlToRelConverter::collectInsertTargets(), computeRowIndicesOfShards(), copyDataOfShard(), and com.mapd.utility.db_vendors.PostGis_types::get_wkt().

66  {
67  typename std::vector<T>::iterator it = std::find(vec.begin(), vec.end(), val);
68  CHECK(it != vec.end());
69  return std::distance(vec.begin(), it);
70 }
#define CHECK(condition)
Definition: Logger.h:222

+ Here is the caller graph for this function:

bool Fragmenter_Namespace::is_integral ( const SQLTypeInfo t)
inline

Definition at line 49 of file UpdelStorage.cpp.

References SQLTypeInfo::is_boolean(), SQLTypeInfo::is_integer(), SQLTypeInfo::is_time(), and SQLTypeInfo::is_timeinterval().

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::updateColumn(), and Fragmenter_Namespace::InsertOrderFragmenter::updateColumnMetadata().

49  {
50  return t.is_integer() || t.is_boolean() || t.is_time() || t.is_timeinterval();
51 }
bool is_time() const
Definition: sqltypes.h:516
bool is_integer() const
Definition: sqltypes.h:512
bool is_timeinterval() const
Definition: sqltypes.h:521
bool is_boolean() const
Definition: sqltypes.h:517

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

bool Fragmenter_Namespace::isDatumVectorData ( const ColumnDescriptor cd)

Definition at line 78 of file InsertDataLoader.cpp.

References ColumnDescriptor::columnType, and SQLTypeInfo::is_array().

Referenced by compute_row_indices_of_shards(), computeRowIndicesOfShards(), and copyColumnDataOfShard().

78  {
79  return cd->columnType.is_array();
80 }
SQLTypeInfo columnType
bool is_array() const
Definition: sqltypes.h:518

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

bool Fragmenter_Namespace::isStringVectorData ( const ColumnDescriptor cd)

Definition at line 72 of file InsertDataLoader.cpp.

References ColumnDescriptor::columnType, SQLTypeInfo::get_compression(), SQLTypeInfo::is_geometry(), SQLTypeInfo::is_string(), and kENCODING_NONE.

Referenced by compute_row_indices_of_shards(), computeRowIndicesOfShards(), and copyColumnDataOfShard().

72  {
73  return (cd->columnType.is_geometry()) ||
74  (cd->columnType.is_string() &&
76 }
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:337
bool is_geometry() const
Definition: sqltypes.h:522
SQLTypeInfo columnType
bool is_string() const
Definition: sqltypes.h:510

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

static void Fragmenter_Namespace::set_chunk_metadata ( const Catalog_Namespace::Catalog catalog,
FragmentInfo &  fragment,
const std::shared_ptr< Chunk_NS::Chunk > &  chunk,
const size_t  nrows_to_keep,
UpdelRoll updel_roll 
)
static

Definition at line 1062 of file UpdelStorage.cpp.

References UpdelRoll::addDirtyChunk(), Fragmenter_Namespace::FragmentInfo::fragmentId, UpdelRoll::getChunkMetadata(), and Catalog_Namespace::Catalog::getMetadataForTable().

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::compactRows().

1066  {
1067  auto cd = chunk->getColumnDesc();
1068  auto td = catalog->getMetadataForTable(cd->tableId);
1069  auto data_buffer = chunk->getBuffer();
1070  auto chunkMetadata =
1071  updel_roll.getChunkMetadata({td, &fragment}, cd->columnId, fragment);
1072  chunkMetadata->numElements = nrows_to_keep;
1073  chunkMetadata->numBytes = data_buffer->size();
1074  updel_roll.addDirtyChunk(chunk, fragment.fragmentId);
1075 }
void addDirtyChunk(std::shared_ptr< Chunk_NS::Chunk > chunk, int fragment_id)
std::shared_ptr< ChunkMetadata > getChunkMetadata(const MetaDataKey &key, int32_t column_id, Fragmenter_Namespace::FragmentInfo &fragment_info)
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename T >
static void Fragmenter_Namespace::set_chunk_stats ( const SQLTypeInfo col_type,
int8_t *  data_addr,
bool &  has_null,
T &  min,
T &  max 
)
static

Definition at line 1047 of file UpdelStorage.cpp.

References SQLTypeInfo::get_notnull(), is_null(), anonymous_namespace{TypedDataAccessors.h}::set_minmax(), and heavydb.dtypes::T.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::compactRows().

1051  {
1052  T v;
1053  const auto can_be_null = !col_type.get_notnull();
1054  const auto is_null = get_scalar<T>(data_addr, col_type, v);
1055  if (is_null) {
1056  has_null = has_null || (can_be_null && is_null);
1057  } else {
1058  set_minmax(min, max, v);
1059  }
1060 }
CONSTEXPR DEVICE bool is_null(const T &value)
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:336
void set_minmax(T &min, T &max, T const val)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Fragmenter_Namespace::shuffleByIndexes ( const ColumnDescriptor cd,
const std::vector< size_t > &  indexes,
DataBlockPtr data 
)

Definition at line 45 of file SortedOrderFragmenter.cpp.

References DataBlockPtr::arraysPtr, CHECK, ColumnDescriptor::columnType, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, DataBlockPtr::numbersPtr, shuffleByIndexesImpl(), and DataBlockPtr::stringsPtr.

Referenced by Fragmenter_Namespace::SortedOrderFragmenter::sortData().

47  {
48  const auto& ti = cd->columnType;
49  switch (ti.get_type()) {
50  case kBOOLEAN:
51  shuffleByIndexesImpl(indexes, reinterpret_cast<int8_t*>(data.numbersPtr));
52  break;
53  case kTINYINT:
54  shuffleByIndexesImpl(indexes, reinterpret_cast<int8_t*>(data.numbersPtr));
55  break;
56  case kSMALLINT:
57  shuffleByIndexesImpl(indexes, reinterpret_cast<int16_t*>(data.numbersPtr));
58  break;
59  case kINT:
60  shuffleByIndexesImpl(indexes, reinterpret_cast<int32_t*>(data.numbersPtr));
61  break;
62  case kBIGINT:
63  case kNUMERIC:
64  case kDECIMAL:
65  shuffleByIndexesImpl(indexes, reinterpret_cast<int64_t*>(data.numbersPtr));
66  break;
67  case kFLOAT:
68  shuffleByIndexesImpl(indexes, reinterpret_cast<float*>(data.numbersPtr));
69  break;
70  case kDOUBLE:
71  shuffleByIndexesImpl(indexes, reinterpret_cast<double*>(data.numbersPtr));
72  break;
73  case kTEXT:
74  case kVARCHAR:
75  case kCHAR:
76  if (ti.is_varlen()) {
77  shuffleByIndexesImpl(indexes, *data.stringsPtr);
78  } else {
79  switch (ti.get_size()) {
80  case 1:
81  shuffleByIndexesImpl(indexes, reinterpret_cast<int8_t*>(data.numbersPtr));
82  break;
83  case 2:
84  shuffleByIndexesImpl(indexes, reinterpret_cast<int16_t*>(data.numbersPtr));
85  break;
86  case 4:
87  shuffleByIndexesImpl(indexes, reinterpret_cast<int32_t*>(data.numbersPtr));
88  break;
89  default:
90  CHECK(false);
91  }
92  }
93  break;
94  case kDATE:
95  case kTIME:
96  case kTIMESTAMP:
97  shuffleByIndexesImpl(indexes, reinterpret_cast<int64_t*>(data.numbersPtr));
98  break;
99  case kARRAY:
100  shuffleByIndexesImpl(indexes, *data.arraysPtr);
101  break;
102  case kPOINT:
103  case kLINESTRING:
104  case kPOLYGON:
105  case kMULTIPOLYGON:
106  shuffleByIndexesImpl(indexes, *data.stringsPtr);
107  break;
108  default:
109  CHECK(false);
110  }
111 }
Definition: sqltypes.h:49
std::vector< std::string > * stringsPtr
Definition: sqltypes.h:227
std::vector< ArrayDatum > * arraysPtr
Definition: sqltypes.h:228
Definition: sqltypes.h:52
Definition: sqltypes.h:53
Definition: sqltypes.h:41
#define CHECK(condition)
Definition: Logger.h:222
Definition: sqltypes.h:45
SQLTypeInfo columnType
int8_t * numbersPtr
Definition: sqltypes.h:226
void shuffleByIndexesImpl(const std::vector< size_t > &indexes, T *buffer)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename T >
void Fragmenter_Namespace::shuffleByIndexesImpl ( const std::vector< size_t > &  indexes,
T *  buffer 
)

Definition at line 26 of file SortedOrderFragmenter.cpp.

References heavydb.dtypes::T.

Referenced by shuffleByIndexes().

26  {
27  std::vector<T> new_buffer;
28  new_buffer.reserve(indexes.size());
29  for (const auto i : indexes) {
30  new_buffer.push_back(buffer[i]);
31  }
32  std::memcpy(buffer, new_buffer.data(), indexes.size() * sizeof(T));
33 }

+ Here is the caller graph for this function:

template<typename T >
void Fragmenter_Namespace::shuffleByIndexesImpl ( const std::vector< size_t > &  indexes,
std::vector< T > &  buffer 
)

Definition at line 36 of file SortedOrderFragmenter.cpp.

36  {
37  std::vector<T> new_buffer;
38  new_buffer.reserve(indexes.size());
39  for (const auto i : indexes) {
40  new_buffer.push_back(buffer[i]);
41  }
42  buffer.swap(new_buffer);
43 }
size_t Fragmenter_Namespace::size_of_raw_column ( const Catalog_Namespace::Catalog cat,
const ColumnDescriptor cd,
const bool  get_logical_size = true 
)

Definition at line 82 of file InsertDataLoader.cpp.

References ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::get_compression(), SQLTypeInfo::get_logical_size(), SQLTypeInfo::get_size(), SQLTypeInfo::get_type(), SQLTypeInfo::get_type_name(), kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kENCODING_NONE, kFLOAT, kINT, kINTERVAL_DAY_TIME, kINTERVAL_YEAR_MONTH, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, and kVARCHAR.

Referenced by compute_row_indices_of_shards(), computeRowIndicesOfShards(), and copyColumnDataOfShard().

84  {
85  switch (cd->columnType.get_type()) {
86  case kPOINT:
87  case kLINESTRING:
88  case kPOLYGON:
89  case kMULTIPOLYGON:
90  case kARRAY:
91  throw std::runtime_error("geo and array columns have variable length elements");
92  case kBOOLEAN:
93  case kTINYINT:
94  case kSMALLINT:
95  case kINT:
96  case kBIGINT:
97  case kNUMERIC:
98  case kDECIMAL:
99  case kFLOAT:
100  case kDOUBLE:
101  case kTIMESTAMP:
102  case kTIME:
103  case kINTERVAL_DAY_TIME:
105  case kDATE:
106  return get_logical_size ? cd->columnType.get_logical_size()
107  : cd->columnType.get_size();
108  case kTEXT:
109  case kVARCHAR:
110  case kCHAR:
112  throw std::runtime_error(
113  "non encoded string columns have variable length elements");
114  }
115  return cd->columnType.get_size();
116  default:
117  throw std::runtime_error("not supported column type: " + cd->columnName + " (" +
118  cd->columnType.get_type_name() + ")");
119  }
120 }
HOST DEVICE int get_size() const
Definition: sqltypes.h:339
Definition: sqltypes.h:49
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:329
int get_logical_size() const
Definition: sqltypes.h:349
Definition: sqltypes.h:52
Definition: sqltypes.h:53
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:337
std::string get_type_name() const
Definition: sqltypes.h:443
Definition: sqltypes.h:41
Definition: sqltypes.h:45
SQLTypeInfo columnType
std::string columnName

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Fragmenter_Namespace::sortIndexes ( const ColumnDescriptor cd,
std::vector< size_t > &  indexes,
const DataBlockPtr data 
)

Definition at line 138 of file SortedOrderFragmenter.cpp.

References DataBlockPtr::arraysPtr, CHECK, ColumnDescriptor::columnType, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kNUMERIC, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, DataBlockPtr::numbersPtr, sortIndexesImpl(), and DataBlockPtr::stringsPtr.

Referenced by Fragmenter_Namespace::SortedOrderFragmenter::sortData().

140  {
141  const auto& ti = cd->columnType;
142  switch (ti.get_type()) {
143  case kBOOLEAN:
144  sortIndexesImpl(indexes, reinterpret_cast<int8_t*>(data.numbersPtr));
145  break;
146  case kTINYINT:
147  sortIndexesImpl(indexes, reinterpret_cast<int8_t*>(data.numbersPtr));
148  break;
149  case kSMALLINT:
150  sortIndexesImpl(indexes, reinterpret_cast<int16_t*>(data.numbersPtr));
151  break;
152  case kINT:
153  sortIndexesImpl(indexes, reinterpret_cast<int32_t*>(data.numbersPtr));
154  break;
155  case kBIGINT:
156  case kNUMERIC:
157  case kDECIMAL:
158  sortIndexesImpl(indexes, reinterpret_cast<int64_t*>(data.numbersPtr));
159  break;
160  case kFLOAT:
161  sortIndexesImpl(indexes, reinterpret_cast<float*>(data.numbersPtr));
162  break;
163  case kDOUBLE:
164  sortIndexesImpl(indexes, reinterpret_cast<double*>(data.numbersPtr));
165  break;
166  case kTEXT:
167  case kVARCHAR:
168  case kCHAR:
169  if (ti.is_varlen()) {
170  sortIndexesImpl(indexes, *data.stringsPtr);
171  } else {
172  switch (ti.get_size()) {
173  case 1:
174  sortIndexesImpl(indexes, reinterpret_cast<int8_t*>(data.numbersPtr));
175  break;
176  case 2:
177  sortIndexesImpl(indexes, reinterpret_cast<int16_t*>(data.numbersPtr));
178  break;
179  case 4:
180  sortIndexesImpl(indexes, reinterpret_cast<int32_t*>(data.numbersPtr));
181  break;
182  default:
183  CHECK(false);
184  }
185  }
186  break;
187  case kDATE:
188  case kTIME:
189  case kTIMESTAMP:
190  sortIndexesImpl(indexes, reinterpret_cast<int64_t*>(data.numbersPtr));
191  break;
192  case kARRAY:
193  sortIndexesImpl(indexes, *data.arraysPtr);
194  break;
195  default:
196  CHECK(false) << "invalid type '" << ti.get_type() << "' to sort";
197  }
198 }
Definition: sqltypes.h:49
std::vector< std::string > * stringsPtr
Definition: sqltypes.h:227
std::vector< ArrayDatum > * arraysPtr
Definition: sqltypes.h:228
void sortIndexesImpl(std::vector< size_t > &indexes, const T *buffer)
Definition: sqltypes.h:52
Definition: sqltypes.h:53
Definition: sqltypes.h:41
#define CHECK(condition)
Definition: Logger.h:222
Definition: sqltypes.h:45
SQLTypeInfo columnType
int8_t * numbersPtr
Definition: sqltypes.h:226

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename T >
void Fragmenter_Namespace::sortIndexesImpl ( std::vector< size_t > &  indexes,
const T *  buffer 
)

Definition at line 114 of file SortedOrderFragmenter.cpp.

References anonymous_namespace{Utm.h}::a, CHECK, and gpu_enabled::sort().

Referenced by sortIndexes().

114  {
115  CHECK(buffer);
116  std::sort(indexes.begin(), indexes.end(), [&](const auto a, const auto b) {
117  return buffer[a] < buffer[b];
118  });
119 }
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
constexpr double a
Definition: Utm.h:32
#define CHECK(condition)
Definition: Logger.h:222

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Fragmenter_Namespace::sortIndexesImpl ( std::vector< size_t > &  indexes,
const std::vector< std::string > &  buffer 
)

Definition at line 121 of file SortedOrderFragmenter.cpp.

References anonymous_namespace{Utm.h}::a, and gpu_enabled::sort().

122  {
123  std::sort(indexes.begin(), indexes.end(), [&](const auto a, const auto b) {
124  return buffer[a].size() < buffer[b].size() ||
125  (buffer[a].size() == buffer[b].size() && buffer[a] < buffer[b]);
126  });
127 }
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
constexpr double a
Definition: Utm.h:32

+ Here is the call graph for this function:

void Fragmenter_Namespace::sortIndexesImpl ( std::vector< size_t > &  indexes,
const std::vector< ArrayDatum > &  buffer 
)

Definition at line 129 of file SortedOrderFragmenter.cpp.

References anonymous_namespace{Utm.h}::a, and gpu_enabled::sort().

130  {
131  std::sort(indexes.begin(), indexes.end(), [&](const auto a, const auto b) {
132  return buffer[a].is_null || buffer[a].length < buffer[b].length ||
133  (!buffer[b].is_null && buffer[a].length == buffer[b].length &&
134  memcmp(buffer[a].pointer, buffer[b].pointer, buffer[a].length) < 0);
135  });
136 }
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
constexpr double a
Definition: Utm.h:32

+ Here is the call graph for this function:

void Fragmenter_Namespace::wait_cleanup_threads ( std::vector< std::future< void >> &  threads)
inline

Definition at line 42 of file UpdelStorage.cpp.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::compactRows(), Fragmenter_Namespace::InsertOrderFragmenter::getVacuumOffsets(), and Fragmenter_Namespace::InsertOrderFragmenter::updateColumn().

42  {
43  for (auto& t : threads) {
44  t.get();
45  }
46  threads.clear();
47 }

+ Here is the caller graph for this function: