OmniSciDB  72c90bc290
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Fragmenter_Namespace Namespace Reference

Namespaces

 anonymous_namespace{InsertOrderFragmenter.cpp}
 
 anonymous_namespace{UpdelStorage.cpp}
 

Classes

class  RowDataProvider
 
struct  UpdateValuesStats
 
struct  ChunkUpdateStats
 
class  AbstractFragmenter
 
struct  InsertChunks
 
struct  InsertData
 The data to be inserted using the fragment manager. More...
 
class  FragmentInfo
 Used by Fragmenter classes to store info about each fragment - the fragment id and number of tuples(rows) currently stored by that fragment. More...
 
class  TableInfo
 
struct  ShardDataOwner
 
struct  BlockWithColumnId
 
class  InsertDataLoader
 
class  LocalInsertConnector
 
class  InsertOrderFragmenter
 The InsertOrderFragmenter is a child class of AbstractFragmenter, and fragments data in insert order. Likely the default fragmenter. More...
 
class  SortedOrderFragmenter
 
struct  ChunkToInsertDataConverter
 
struct  ScalarChunkConverter
 
struct  FixedLenArrayChunkConverter
 
struct  ArrayChunkConverter
 
struct  StringChunkConverter
 
struct  DateChunkConverter
 

Enumerations

enum  FragmenterType { INSERT_ORDER = 0 }
 

Functions

template<typename SRC >
std::vector< std::vector
< size_t > > 
compute_row_indices_of_shards (size_t shard_count, size_t leaf_count, size_t row_count, SRC *src, bool duplicated_key_value)
 
template<typename T >
size_t indexOf (std::vector< T > &vec, T val)
 
bool isStringVectorData (const ColumnDescriptor *cd)
 
bool isDatumVectorData (const ColumnDescriptor *cd)
 
size_t size_of_raw_column (const Catalog_Namespace::Catalog &cat, const ColumnDescriptor *cd, const bool get_logical_size=true)
 
std::vector< std::vector
< size_t > > 
compute_row_indices_of_shards (const Catalog_Namespace::Catalog &cat, size_t leaf_count, const InsertChunks &insert_chunks)
 
std::vector< std::vector
< size_t > > 
computeRowIndicesOfShards (const Catalog_Namespace::Catalog &cat, size_t leafCount, InsertData &insert_data)
 
template<typename T >
void copyColumnDataOfShard (const std::vector< size_t > &rowIndices, T *src, T *dst)
 
BlockWithColumnId copyColumnDataOfShard (const Catalog_Namespace::Catalog &cat, ShardDataOwner &dataOwner, const std::vector< size_t > &rowIndices, const ColumnDescriptor *pCol, size_t columnIndex, DataBlockPtr dataBlock, bool is_default)
 
std::pair< std::list
< std::unique_ptr
< foreign_storage::ForeignStorageBuffer >
>, InsertChunks
copy_data_of_shard (const Catalog_Namespace::Catalog &cat, const InsertChunks &insert_chunks, int shardTableIndex, const std::vector< size_t > &rowIndices)
 
InsertData copyDataOfShard (const Catalog_Namespace::Catalog &cat, ShardDataOwner &dataOwner, InsertData &insert_data, int shardTableIndex, const std::vector< size_t > &rowIndices)
 
template<typename T >
void shuffleByIndexesImpl (const std::vector< size_t > &indexes, T *buffer)
 
template<typename T >
void shuffleByIndexesImpl (const std::vector< size_t > &indexes, std::vector< T > &buffer)
 
void shuffleByIndexes (const ColumnDescriptor *cd, const std::vector< size_t > &indexes, DataBlockPtr &data)
 
template<typename T >
void sortIndexesImpl (std::vector< size_t > &indexes, const T *buffer)
 
void sortIndexesImpl (std::vector< size_t > &indexes, const std::vector< std::string > &buffer)
 
void sortIndexesImpl (std::vector< size_t > &indexes, const std::vector< ArrayDatum > &buffer)
 
void sortIndexes (const ColumnDescriptor *cd, std::vector< size_t > &indexes, const DataBlockPtr &data)
 
void wait_cleanup_threads (std::vector< std::future< void >> &threads)
 
bool is_integral (const SQLTypeInfo &t)
 
static int get_chunks (const Catalog_Namespace::Catalog *catalog, const TableDescriptor *td, const FragmentInfo &fragment, const Data_Namespace::MemoryLevel memory_level, std::vector< std::shared_ptr< Chunk_NS::Chunk >> &chunks)
 
template<typename T >
static void set_chunk_stats (const SQLTypeInfo &col_type, int8_t *data_addr, bool &has_null, T &min, T &max)
 
static void set_chunk_metadata (const Catalog_Namespace::Catalog *catalog, FragmentInfo &fragment, const std::shared_ptr< Chunk_NS::Chunk > &chunk, const size_t nrows_to_keep, UpdelRoll &updel_roll)
 
size_t get_null_padding (bool is_varlen_array, const std::vector< uint64_t > &frag_offsets, const StringOffsetT *index_array, size_t fragment_row_count)
 
std::set< size_t > get_var_len_null_array_indexes (const SQLTypeInfo sql_type_info, const std::vector< uint64_t > &frag_offsets, const StringOffsetT *index_array, size_t fragment_row_count)
 
StringOffsetT get_buffer_offset (bool is_varlen_array, const StringOffsetT *index_array, size_t index)
 

Enumeration Type Documentation

stores the type of a child class of AbstractTableFragmenter

Enumerator
INSERT_ORDER 

Definition at line 45 of file Fragmenter.h.

45  {
46  INSERT_ORDER = 0 // these values persist in catalog. make explicit
47 };

Function Documentation

template<typename SRC >
std::vector<std::vector<size_t> > Fragmenter_Namespace::compute_row_indices_of_shards ( size_t  shard_count,
size_t  leaf_count,
size_t  row_count,
SRC *  src,
bool  duplicated_key_value 
)

Definition at line 35 of file InsertDataLoader.cpp.

References SHARD_FOR_KEY.

Referenced by compute_row_indices_of_shards(), computeRowIndicesOfShards(), and Fragmenter_Namespace::InsertDataLoader::insertChunks().

40  {
41  const auto n_shard_tables = shard_count * leaf_count;
42  std::vector<std::vector<size_t>> row_indices_of_shards(n_shard_tables);
43  if (!duplicated_key_value) {
44  for (size_t row = 0; row < row_count; row++) {
45  // expecting unsigned data
46  // thus, no need for double remainder
47  auto shard_id = (std::is_unsigned<SRC>::value)
48  ? src[row] % n_shard_tables
49  : SHARD_FOR_KEY(src[row], n_shard_tables);
50  row_indices_of_shards[shard_id].push_back(row);
51  }
52  } else {
53  auto shard_id = (std::is_unsigned<SRC>::value)
54  ? src[0] % n_shard_tables
55  : SHARD_FOR_KEY(src[0], n_shard_tables);
56  row_indices_of_shards[shard_id].reserve(row_count);
57  for (size_t row = 0; row < row_count; row++) {
58  row_indices_of_shards[shard_id].push_back(row);
59  }
60  }
61 
62  return row_indices_of_shards;
63 }
#define SHARD_FOR_KEY(key, num_shards)
Definition: shard_key.h:20

+ Here is the caller graph for this function:

std::vector<std::vector<size_t> > Fragmenter_Namespace::compute_row_indices_of_shards ( const Catalog_Namespace::Catalog cat,
size_t  leaf_count,
const InsertChunks &  insert_chunks 
)

Definition at line 124 of file InsertDataLoader.cpp.

References CHECK, Fragmenter_Namespace::InsertChunks::chunks, compute_row_indices_of_shards(), Chunk_NS::Chunk::getBuffer(), Data_Namespace::AbstractBuffer::getEncoder(), Catalog_Namespace::Catalog::getMetadataForTable(), Encoder::getNumElems(), Catalog_Namespace::Catalog::getShardColumnMetadataForTable(), isDatumVectorData(), isStringVectorData(), size_of_raw_column(), Fragmenter_Namespace::InsertChunks::table_id, and UNREACHABLE.

127  {
128  const auto* td = cat.getMetadataForTable(insert_chunks.table_id);
129  const auto* shard_cd = cat.getShardColumnMetadataForTable(td);
130  auto find_it = insert_chunks.chunks.find(shard_cd->columnId);
131  CHECK(find_it != insert_chunks.chunks.end());
132  Chunk_NS::Chunk& shard_chunk = *find_it->second;
133  auto row_count = shard_chunk.getBuffer()->getEncoder()->getNumElems();
134  auto shard_count = td->nShards;
135 
136  CHECK(!isStringVectorData(shard_cd));
137  CHECK(!isDatumVectorData(shard_cd));
138 
139  auto memory_ptr = shard_chunk.getBuffer()->getMemoryPtr();
140  CHECK(memory_ptr);
141  switch (size_of_raw_column(cat, shard_cd, false)) {
142  case 1:
143  return compute_row_indices_of_shards(shard_count,
144  leaf_count,
145  row_count,
146  reinterpret_cast<uint8_t*>(memory_ptr),
147  false);
148  case 2:
149  return compute_row_indices_of_shards(shard_count,
150  leaf_count,
151  row_count,
152  reinterpret_cast<uint16_t*>(memory_ptr),
153  false);
154  case 4:
155  return compute_row_indices_of_shards(shard_count,
156  leaf_count,
157  row_count,
158  reinterpret_cast<uint32_t*>(memory_ptr),
159  false);
160  case 8:
161  return compute_row_indices_of_shards(shard_count,
162  leaf_count,
163  row_count,
164  reinterpret_cast<uint64_t*>(memory_ptr),
165  false);
166  default:
167  UNREACHABLE() << "unexpected data element size of column";
168  }
169  return {};
170 }
#define UNREACHABLE()
Definition: Logger.h:338
const ColumnDescriptor * getShardColumnMetadataForTable(const TableDescriptor *td) const
Definition: Catalog.cpp:4851
bool isStringVectorData(const ColumnDescriptor *cd)
std::string chunks
size_t size_of_raw_column(const Catalog_Namespace::Catalog &cat, const ColumnDescriptor *cd, const bool get_logical_size=true)
size_t getNumElems() const
Definition: Encoder.h:284
std::vector< std::vector< size_t > > compute_row_indices_of_shards(size_t shard_count, size_t leaf_count, size_t row_count, SRC *src, bool duplicated_key_value)
AbstractBuffer * getBuffer() const
Definition: Chunk.h:146
bool isDatumVectorData(const ColumnDescriptor *cd)
#define CHECK(condition)
Definition: Logger.h:291
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.

+ Here is the call graph for this function:

std::vector<std::vector<size_t> > Fragmenter_Namespace::computeRowIndicesOfShards ( const Catalog_Namespace::Catalog cat,
size_t  leafCount,
InsertData &  insert_data 
)

Definition at line 172 of file InsertDataLoader.cpp.

References CHECK, Fragmenter_Namespace::InsertData::columnIds, compute_row_indices_of_shards(), Fragmenter_Namespace::InsertData::data, Catalog_Namespace::Catalog::getMetadataForTable(), Catalog_Namespace::Catalog::getShardColumnMetadataForTable(), indexOf(), Fragmenter_Namespace::InsertData::is_default, isDatumVectorData(), isStringVectorData(), DataBlockPtr::numbersPtr, Fragmenter_Namespace::InsertData::numRows, size_of_raw_column(), and Fragmenter_Namespace::InsertData::tableId.

Referenced by Fragmenter_Namespace::InsertDataLoader::insertData().

175  {
176  const auto* td = cat.getMetadataForTable(insert_data.tableId);
177  const auto* shard_cd = cat.getShardColumnMetadataForTable(td);
178  auto shardDataBlockIndex = indexOf(insert_data.columnIds, shard_cd->columnId);
179  DataBlockPtr& shardDataBlock = insert_data.data[shardDataBlockIndex];
180  auto rowCount = insert_data.numRows;
181  auto shardCount = td->nShards;
182 
183  CHECK(!isStringVectorData(shard_cd));
184  CHECK(!isDatumVectorData(shard_cd));
185 
186  CHECK(insert_data.is_default.size() == insert_data.columnIds.size());
187  bool is_default = insert_data.is_default[shardDataBlockIndex];
188  switch (size_of_raw_column(cat, shard_cd)) {
189  case 1:
191  shardCount,
192  leafCount,
193  rowCount,
194  reinterpret_cast<uint8_t*>(shardDataBlock.numbersPtr),
195  is_default);
196  case 2:
198  shardCount,
199  leafCount,
200  rowCount,
201  reinterpret_cast<uint16_t*>(shardDataBlock.numbersPtr),
202  is_default);
203  case 4:
205  shardCount,
206  leafCount,
207  rowCount,
208  reinterpret_cast<uint32_t*>(shardDataBlock.numbersPtr),
209  is_default);
210  case 8:
212  shardCount,
213  leafCount,
214  rowCount,
215  reinterpret_cast<uint64_t*>(shardDataBlock.numbersPtr),
216  is_default);
217  }
218  throw std::runtime_error("Unexpected data block element size");
219 }
const ColumnDescriptor * getShardColumnMetadataForTable(const TableDescriptor *td) const
Definition: Catalog.cpp:4851
bool isStringVectorData(const ColumnDescriptor *cd)
size_t size_of_raw_column(const Catalog_Namespace::Catalog &cat, const ColumnDescriptor *cd, const bool get_logical_size=true)
std::vector< std::vector< size_t > > compute_row_indices_of_shards(size_t shard_count, size_t leaf_count, size_t row_count, SRC *src, bool duplicated_key_value)
size_t indexOf(std::vector< T > &vec, T val)
bool isDatumVectorData(const ColumnDescriptor *cd)
#define CHECK(condition)
Definition: Logger.h:291
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
int8_t * numbersPtr
Definition: sqltypes.h:233

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::pair<std::list<std::unique_ptr<foreign_storage::ForeignStorageBuffer> >, InsertChunks> Fragmenter_Namespace::copy_data_of_shard ( const Catalog_Namespace::Catalog cat,
const InsertChunks &  insert_chunks,
int  shardTableIndex,
const std::vector< size_t > &  rowIndices 
)

Definition at line 298 of file InsertDataLoader.cpp.

References CHECK_EQ, Fragmenter_Namespace::InsertChunks::chunks, Fragmenter_Namespace::InsertChunks::db_id, Catalog_Namespace::Catalog::getMetadataForTable(), Catalog_Namespace::Catalog::getPhysicalTablesDescriptors(), Fragmenter_Namespace::InsertChunks::table_id, and Fragmenter_Namespace::InsertChunks::valid_row_indices.

Referenced by Fragmenter_Namespace::InsertDataLoader::insertChunks().

301  {
302  const auto* table = cat.getMetadataForTable(insert_chunks.table_id);
303  const auto* physical_table = cat.getPhysicalTablesDescriptors(table)[shardTableIndex];
304 
305  InsertChunks insert_chunks_for_shard{
306  physical_table->tableId, insert_chunks.db_id, {}, {}};
307 
308  std::list<std::unique_ptr<foreign_storage::ForeignStorageBuffer>> buffers;
309 
310  for (const auto& [column_id, chunk] : insert_chunks.chunks) {
311  auto column = chunk->getColumnDesc();
312  insert_chunks_for_shard.chunks[column_id] = std::make_shared<Chunk_NS::Chunk>(column);
313  auto& chunk_for_shard = *insert_chunks_for_shard.chunks[column_id];
314  chunk_for_shard.setBuffer(
315  buffers.emplace_back(std::make_unique<foreign_storage::ForeignStorageBuffer>())
316  .get());
317  if (column->columnType.is_varlen_indeed()) { // requires an index buffer
318  chunk_for_shard.setIndexBuffer(
319  buffers.emplace_back(std::make_unique<foreign_storage::ForeignStorageBuffer>())
320  .get());
321  }
322  chunk_for_shard.initEncoder();
323  chunk_for_shard.appendEncodedDataAtIndices(*chunk, rowIndices);
324  CHECK_EQ(chunk_for_shard.getBuffer()->getEncoder()->getNumElems(), rowIndices.size());
325  }
326 
327  // mark which row indices are valid for import
328  auto row_count = rowIndices.size();
329  insert_chunks_for_shard.valid_row_indices.reserve(row_count);
330  for (size_t irow = 0; irow < row_count; ++irow) {
331  auto row_index = rowIndices[irow];
332  if (std::binary_search(insert_chunks.valid_row_indices.begin(),
333  insert_chunks.valid_row_indices.end(),
334  row_index)) {
335  insert_chunks_for_shard.valid_row_indices.emplace_back(irow);
336  }
337  }
338 
339  return {std::move(buffers), insert_chunks_for_shard};
340 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4869
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename T >
void Fragmenter_Namespace::copyColumnDataOfShard ( const std::vector< size_t > &  rowIndices,
T *  src,
T *  dst 
)

Definition at line 222 of file InsertDataLoader.cpp.

Referenced by copyColumnDataOfShard(), and copyDataOfShard().

222  {
223  for (size_t row = 0; row < rowIndices.size(); row++) {
224  auto srcRowIndex = rowIndices[row];
225  dst[row] = src[srcRowIndex];
226  }
227 }

+ Here is the caller graph for this function:

BlockWithColumnId Fragmenter_Namespace::copyColumnDataOfShard ( const Catalog_Namespace::Catalog cat,
ShardDataOwner &  dataOwner,
const std::vector< size_t > &  rowIndices,
const ColumnDescriptor pCol,
size_t  columnIndex,
DataBlockPtr  dataBlock,
bool  is_default 
)

Definition at line 235 of file InsertDataLoader.cpp.

References Fragmenter_Namespace::ShardDataOwner::arrayData, DataBlockPtr::arraysPtr, ColumnDescriptor::columnId, copyColumnDataOfShard(), isDatumVectorData(), isStringVectorData(), DataBlockPtr::numbersPtr, Fragmenter_Namespace::ShardDataOwner::rawData, report::rows, size_of_raw_column(), Fragmenter_Namespace::ShardDataOwner::stringData, and DataBlockPtr::stringsPtr.

241  {
242  DataBlockPtr ret;
243  std::vector<size_t> single_row_idx({0ul});
244  const std::vector<size_t>& rows = is_default ? single_row_idx : rowIndices;
245  if (isStringVectorData(pCol)) {
246  auto& data = dataOwner.stringData[columnIndex];
247  data.resize(rows.size());
248  copyColumnDataOfShard(rows, &(*(dataBlock.stringsPtr))[0], &data[0]);
249  ret.stringsPtr = &data;
250 
251  } else if (isDatumVectorData(pCol)) {
252  auto& data = dataOwner.arrayData[columnIndex];
253  data.resize(rows.size());
254  copyColumnDataOfShard(rows, &(*(dataBlock.arraysPtr))[0], &data[0]);
255  ret.arraysPtr = &data;
256 
257  } else {
258  auto rawArrayElementSize = size_of_raw_column(cat, pCol);
259  auto& data = dataOwner.rawData[columnIndex];
260  data.resize(rows.size() * rawArrayElementSize);
261 
262  switch (rawArrayElementSize) {
263  case 1: {
265  reinterpret_cast<uint8_t*>(dataBlock.numbersPtr),
266  reinterpret_cast<uint8_t*>(&data[0]));
267  break;
268  }
269  case 2: {
271  reinterpret_cast<uint16_t*>(dataBlock.numbersPtr),
272  reinterpret_cast<uint16_t*>(&data[0]));
273  break;
274  }
275  case 4: {
277  reinterpret_cast<uint32_t*>(dataBlock.numbersPtr),
278  reinterpret_cast<uint32_t*>(&data[0]));
279  break;
280  }
281  case 8: {
283  reinterpret_cast<uint64_t*>(dataBlock.numbersPtr),
284  reinterpret_cast<uint64_t*>(&data[0]));
285  break;
286  }
287  default:
288  throw std::runtime_error("Unexpected data block element size");
289  }
290 
291  ret.numbersPtr = reinterpret_cast<int8_t*>(&data[0]);
292  }
293 
294  return {pCol->columnId, ret, is_default};
295 }
std::vector< std::string > * stringsPtr
Definition: sqltypes.h:234
std::vector< ArrayDatum > * arraysPtr
Definition: sqltypes.h:235
bool isStringVectorData(const ColumnDescriptor *cd)
tuple rows
Definition: report.py:114
size_t size_of_raw_column(const Catalog_Namespace::Catalog &cat, const ColumnDescriptor *cd, const bool get_logical_size=true)
void copyColumnDataOfShard(const std::vector< size_t > &rowIndices, T *src, T *dst)
bool isDatumVectorData(const ColumnDescriptor *cd)
int8_t * numbersPtr
Definition: sqltypes.h:233

+ Here is the call graph for this function:

InsertData Fragmenter_Namespace::copyDataOfShard ( const Catalog_Namespace::Catalog cat,
ShardDataOwner &  dataOwner,
InsertData &  insert_data,
int  shardTableIndex,
const std::vector< size_t > &  rowIndices 
)

Definition at line 342 of file InsertDataLoader.cpp.

References Fragmenter_Namespace::ShardDataOwner::arrayData, threading_serial::async(), cat(), Fragmenter_Namespace::InsertData::columnIds, copyColumnDataOfShard(), Fragmenter_Namespace::InsertData::data, Fragmenter_Namespace::InsertData::databaseId, Catalog_Namespace::Catalog::getAllColumnMetadataForTable(), Catalog_Namespace::Catalog::getMetadataForTable(), Catalog_Namespace::Catalog::getPhysicalTablesDescriptors(), indexOf(), Fragmenter_Namespace::InsertData::is_default, Fragmenter_Namespace::InsertData::numRows, Fragmenter_Namespace::ShardDataOwner::rawData, Fragmenter_Namespace::ShardDataOwner::stringData, and Fragmenter_Namespace::InsertData::tableId.

Referenced by Fragmenter_Namespace::InsertDataLoader::insertData().

346  {
347  const auto* td = cat.getMetadataForTable(insert_data.tableId);
348  const auto* ptd = cat.getPhysicalTablesDescriptors(td)[shardTableIndex];
349 
350  InsertData shardData;
351  shardData.databaseId = insert_data.databaseId;
352  shardData.tableId = ptd->tableId;
353  shardData.numRows = rowIndices.size();
354 
355  std::vector<const ColumnDescriptor*> pCols;
356  std::vector<int> lCols;
357 
358  {
359  auto logicalColumns = cat.getAllColumnMetadataForTable(td->tableId, true, true, true);
360  for (const auto& cd : logicalColumns) {
361  lCols.push_back(cd->columnId);
362  }
363 
364  auto physicalColumns =
365  cat.getAllColumnMetadataForTable(ptd->tableId, true, true, true);
366  for (const auto& cd : physicalColumns) {
367  pCols.push_back(cd);
368  }
369  }
370 
371  for (size_t col = 0; col < insert_data.columnIds.size(); col++) {
372  dataOwner.arrayData.emplace_back();
373  dataOwner.rawData.emplace_back();
374  dataOwner.stringData.emplace_back();
375  }
376 
377  auto copycat = [&cat, &dataOwner, &rowIndices, &lCols, &pCols, &insert_data](int col) {
378  const auto lColId = insert_data.columnIds[col];
379  const auto pCol = pCols[indexOf(lCols, lColId)];
380  return copyColumnDataOfShard(cat,
381  dataOwner,
382  rowIndices,
383  pCol,
384  col,
385  insert_data.data[col],
386  insert_data.is_default[col]);
387  };
388 
389  std::vector<std::future<BlockWithColumnId>> worker_threads;
390  for (size_t col = 0; col < insert_data.columnIds.size(); col++) {
391  worker_threads.push_back(std::async(std::launch::async, copycat, col));
392  }
393 
394  for (auto& child : worker_threads) {
395  child.wait();
396  }
397 
398  for (auto& child : worker_threads) {
399  auto shardColumnData = child.get();
400  shardData.columnIds.push_back(shardColumnData.columnId);
401  shardData.data.push_back(shardColumnData.block);
402  shardData.is_default.push_back(shardColumnData.is_default);
403  }
404 
405  return shardData;
406 }
std::string cat(Ts &&...args)
future< Result > async(Fn &&fn, Args &&...args)
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4869
size_t indexOf(std::vector< T > &vec, T val)
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:2172
void copyColumnDataOfShard(const std::vector< size_t > &rowIndices, T *src, T *dst)
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

StringOffsetT Fragmenter_Namespace::get_buffer_offset ( bool  is_varlen_array,
const StringOffsetT index_array,
size_t  index 
)

Definition at line 1182 of file UpdelStorage.cpp.

References CHECK.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::vacuum_varlen_rows().

1184  {
1185  auto offset = index_array[index];
1186  if (offset < 0) {
1187  // Variable length arrays encode null arrays as negative offsets
1188  CHECK(is_varlen_array);
1189  offset = -offset;
1190  }
1191  return offset;
1192 }
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the caller graph for this function:

static int Fragmenter_Namespace::get_chunks ( const Catalog_Namespace::Catalog catalog,
const TableDescriptor td,
const FragmentInfo &  fragment,
const Data_Namespace::MemoryLevel  memory_level,
std::vector< std::shared_ptr< Chunk_NS::Chunk >> &  chunks 
)
static

Definition at line 75 of file UpdelStorage.cpp.

References CHECK, Catalog_Namespace::DBMetadata::dbId, Fragmenter_Namespace::FragmentInfo::fragmentId, Chunk_NS::Chunk::getChunk(), Fragmenter_Namespace::FragmentInfo::getChunkMetadataMapPhysical(), Catalog_Namespace::Catalog::getCurrentDB(), Catalog_Namespace::Catalog::getDataMgr(), Catalog_Namespace::Catalog::getMetadataForColumn(), TableDescriptor::nColumns, and TableDescriptor::tableId.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::updateColumns().

79  {
80  for (int cid = 1, nc = 0; nc < td->nColumns; ++cid) {
81  if (const auto cd = catalog->getMetadataForColumn(td->tableId, cid)) {
82  ++nc;
83  if (!cd->isVirtualCol) {
84  auto chunk_meta_it = fragment.getChunkMetadataMapPhysical().find(cid);
85  CHECK(chunk_meta_it != fragment.getChunkMetadataMapPhysical().end());
86  ChunkKey chunk_key{
87  catalog->getCurrentDB().dbId, td->tableId, cid, fragment.fragmentId};
88  auto chunk = Chunk_NS::Chunk::getChunk(cd,
89  &catalog->getDataMgr(),
90  chunk_key,
91  memory_level,
92  0,
93  chunk_meta_it->second->numBytes,
94  chunk_meta_it->second->numElements);
95  chunks.push_back(chunk);
96  }
97  }
98  }
99  return chunks.size();
100 }
std::vector< int > ChunkKey
Definition: types.h:36
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:266
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:265
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
#define CHECK(condition)
Definition: Logger.h:291
static std::shared_ptr< Chunk > getChunk(const ColumnDescriptor *cd, DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId, const size_t num_bytes, const size_t num_elems, const bool pinnable=true)
Definition: Chunk.cpp:31

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

size_t Fragmenter_Namespace::get_null_padding ( bool  is_varlen_array,
const std::vector< uint64_t > &  frag_offsets,
const StringOffsetT index_array,
size_t  fragment_row_count 
)

Definition at line 1119 of file UpdelStorage.cpp.

References CHECK_GT, CHECK_LT, and ArrayNoneEncoder::DEFAULT_NULL_PADDING_SIZE.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::vacuum_varlen_rows().

1122  {
1123  if (is_varlen_array) {
1124  size_t first_non_deleted_row_index{0};
1125  for (auto deleted_offset : frag_offsets) {
1126  if (first_non_deleted_row_index < deleted_offset) {
1127  break;
1128  } else {
1129  first_non_deleted_row_index++;
1130  }
1131  }
1132  CHECK_LT(first_non_deleted_row_index, fragment_row_count);
1133  if (first_non_deleted_row_index == 0) {
1134  // If the first row in the fragment is not deleted, then the first offset in the
1135  // index buffer/array already contains expected padding.
1136  return index_array[0];
1137  } else {
1138  // If the first non-deleted element is a null array (indentified by a negative
1139  // offset), get a padding value for the chunk buffer.
1140  if (index_array[first_non_deleted_row_index + 1] < 0) {
1141  size_t first_non_zero_offset{0};
1142  for (size_t i = 0; i <= first_non_deleted_row_index; i++) {
1143  if (index_array[i] != 0) {
1144  first_non_zero_offset = index_array[i];
1145  break;
1146  }
1147  }
1148  CHECK_GT(first_non_zero_offset, static_cast<size_t>(0));
1150  first_non_zero_offset);
1151  } else {
1152  return 0;
1153  }
1154  }
1155  } else {
1156  return 0;
1157  }
1158 }
#define CHECK_GT(x, y)
Definition: Logger.h:305
#define CHECK_LT(x, y)
Definition: Logger.h:303
static constexpr size_t DEFAULT_NULL_PADDING_SIZE

+ Here is the caller graph for this function:

std::set<size_t> Fragmenter_Namespace::get_var_len_null_array_indexes ( const SQLTypeInfo  sql_type_info,
const std::vector< uint64_t > &  frag_offsets,
const StringOffsetT index_array,
size_t  fragment_row_count 
)

Definition at line 1161 of file UpdelStorage.cpp.

References SQLTypeInfo::get_notnull(), and SQLTypeInfo::is_varlen_array().

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::vacuum_varlen_rows().

1164  {
1165  std::set<size_t> null_array_indexes;
1166  if (sql_type_info.is_varlen_array() && !sql_type_info.get_notnull()) {
1167  size_t frag_offset_index{0};
1168  size_t vacuum_offset{0};
1169  for (size_t i = 0; i < fragment_row_count; i++) {
1170  if (frag_offset_index < frag_offsets.size() &&
1171  i == frag_offsets[frag_offset_index]) {
1172  frag_offset_index++;
1173  vacuum_offset++;
1174  } else if (index_array[i + 1] < 0) {
1175  null_array_indexes.emplace(i - vacuum_offset);
1176  }
1177  }
1178  }
1179  return null_array_indexes;
1180 }
bool is_varlen_array() const
Definition: sqltypes.h:586
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:398

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename T >
size_t Fragmenter_Namespace::indexOf ( std::vector< T > &  vec,
val 
)

Definition at line 66 of file InsertDataLoader.cpp.

References CHECK.

Referenced by org.apache.calcite.sql2rel.SqlToRelConverter::collectInsertTargets(), computeRowIndicesOfShards(), copyDataOfShard(), org.apache.calcite.sql.validate.SqlValidatorImpl::expandExprFromJoin(), and com.mapd.utility.db_vendors.PostGis_types::get_wkt().

66  {
67  typename std::vector<T>::iterator it = std::find(vec.begin(), vec.end(), val);
68  CHECK(it != vec.end());
69  return std::distance(vec.begin(), it);
70 }
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the caller graph for this function:

bool Fragmenter_Namespace::is_integral ( const SQLTypeInfo t)
inline

Definition at line 49 of file UpdelStorage.cpp.

References SQLTypeInfo::is_boolean(), SQLTypeInfo::is_integer(), SQLTypeInfo::is_time(), and SQLTypeInfo::is_timeinterval().

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::updateColumn(), and Fragmenter_Namespace::InsertOrderFragmenter::updateColumnMetadata().

49  {
50  return t.is_integer() || t.is_boolean() || t.is_time() || t.is_timeinterval();
51 }
bool is_time() const
Definition: sqltypes.h:577
bool is_integer() const
Definition: sqltypes.h:565
bool is_timeinterval() const
Definition: sqltypes.h:592
bool is_boolean() const
Definition: sqltypes.h:580

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

bool Fragmenter_Namespace::isDatumVectorData ( const ColumnDescriptor cd)

Definition at line 78 of file InsertDataLoader.cpp.

References ColumnDescriptor::columnType, and SQLTypeInfo::is_array().

Referenced by compute_row_indices_of_shards(), computeRowIndicesOfShards(), and copyColumnDataOfShard().

78  {
79  return cd->columnType.is_array();
80 }
SQLTypeInfo columnType
bool is_array() const
Definition: sqltypes.h:583

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

bool Fragmenter_Namespace::isStringVectorData ( const ColumnDescriptor cd)

Definition at line 72 of file InsertDataLoader.cpp.

References ColumnDescriptor::columnType, SQLTypeInfo::get_compression(), SQLTypeInfo::is_geometry(), SQLTypeInfo::is_string(), and kENCODING_NONE.

Referenced by compute_row_indices_of_shards(), computeRowIndicesOfShards(), and copyColumnDataOfShard().

72  {
73  return (cd->columnType.is_geometry()) ||
74  (cd->columnType.is_string() &&
76 }
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:399
bool is_geometry() const
Definition: sqltypes.h:595
SQLTypeInfo columnType
bool is_string() const
Definition: sqltypes.h:559

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

static void Fragmenter_Namespace::set_chunk_metadata ( const Catalog_Namespace::Catalog catalog,
FragmentInfo &  fragment,
const std::shared_ptr< Chunk_NS::Chunk > &  chunk,
const size_t  nrows_to_keep,
UpdelRoll updel_roll 
)
static

Definition at line 1065 of file UpdelStorage.cpp.

References UpdelRoll::addDirtyChunk(), Fragmenter_Namespace::FragmentInfo::fragmentId, UpdelRoll::getChunkMetadata(), and Catalog_Namespace::Catalog::getMetadataForTable().

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::compactRows().

1069  {
1070  auto cd = chunk->getColumnDesc();
1071  auto td = catalog->getMetadataForTable(cd->tableId);
1072  auto data_buffer = chunk->getBuffer();
1073  auto chunkMetadata =
1074  updel_roll.getChunkMetadata({td, &fragment}, cd->columnId, fragment);
1075  chunkMetadata->numElements = nrows_to_keep;
1076  chunkMetadata->numBytes = data_buffer->size();
1077  updel_roll.addDirtyChunk(chunk, fragment.fragmentId);
1078 }
void addDirtyChunk(std::shared_ptr< Chunk_NS::Chunk > chunk, int fragment_id)
std::shared_ptr< ChunkMetadata > getChunkMetadata(const MetaDataKey &key, int32_t column_id, Fragmenter_Namespace::FragmentInfo &fragment_info)
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename T >
static void Fragmenter_Namespace::set_chunk_stats ( const SQLTypeInfo col_type,
int8_t *  data_addr,
bool &  has_null,
T &  min,
T &  max 
)
static

Definition at line 1050 of file UpdelStorage.cpp.

References SQLTypeInfo::get_notnull(), is_null(), anonymous_namespace{TypedDataAccessors.h}::set_minmax(), and heavydb.dtypes::T.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::compactRows().

1054  {
1055  T v;
1056  const auto can_be_null = !col_type.get_notnull();
1057  const auto is_null = get_scalar<T>(data_addr, col_type, v);
1058  if (is_null) {
1059  has_null = has_null || (can_be_null && is_null);
1060  } else {
1061  set_minmax(min, max, v);
1062  }
1063 }
CONSTEXPR DEVICE bool is_null(const T &value)
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:398
void set_minmax(T &min, T &max, T const val)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Fragmenter_Namespace::shuffleByIndexes ( const ColumnDescriptor cd,
const std::vector< size_t > &  indexes,
DataBlockPtr data 
)

Definition at line 45 of file SortedOrderFragmenter.cpp.

References DataBlockPtr::arraysPtr, CHECK, ColumnDescriptor::columnType, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTILINESTRING, kMULTIPOINT, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, DataBlockPtr::numbersPtr, shuffleByIndexesImpl(), and DataBlockPtr::stringsPtr.

Referenced by Fragmenter_Namespace::SortedOrderFragmenter::sortData().

47  {
48  const auto& ti = cd->columnType;
49  switch (ti.get_type()) {
50  case kBOOLEAN:
51  shuffleByIndexesImpl(indexes, reinterpret_cast<int8_t*>(data.numbersPtr));
52  break;
53  case kTINYINT:
54  shuffleByIndexesImpl(indexes, reinterpret_cast<int8_t*>(data.numbersPtr));
55  break;
56  case kSMALLINT:
57  shuffleByIndexesImpl(indexes, reinterpret_cast<int16_t*>(data.numbersPtr));
58  break;
59  case kINT:
60  shuffleByIndexesImpl(indexes, reinterpret_cast<int32_t*>(data.numbersPtr));
61  break;
62  case kBIGINT:
63  case kNUMERIC:
64  case kDECIMAL:
65  shuffleByIndexesImpl(indexes, reinterpret_cast<int64_t*>(data.numbersPtr));
66  break;
67  case kFLOAT:
68  shuffleByIndexesImpl(indexes, reinterpret_cast<float*>(data.numbersPtr));
69  break;
70  case kDOUBLE:
71  shuffleByIndexesImpl(indexes, reinterpret_cast<double*>(data.numbersPtr));
72  break;
73  case kTEXT:
74  case kVARCHAR:
75  case kCHAR:
76  if (ti.is_varlen()) {
77  shuffleByIndexesImpl(indexes, *data.stringsPtr);
78  } else {
79  switch (ti.get_size()) {
80  case 1:
81  shuffleByIndexesImpl(indexes, reinterpret_cast<int8_t*>(data.numbersPtr));
82  break;
83  case 2:
84  shuffleByIndexesImpl(indexes, reinterpret_cast<int16_t*>(data.numbersPtr));
85  break;
86  case 4:
87  shuffleByIndexesImpl(indexes, reinterpret_cast<int32_t*>(data.numbersPtr));
88  break;
89  default:
90  CHECK(false);
91  }
92  }
93  break;
94  case kDATE:
95  case kTIME:
96  case kTIMESTAMP:
97  shuffleByIndexesImpl(indexes, reinterpret_cast<int64_t*>(data.numbersPtr));
98  break;
99  case kARRAY:
100  shuffleByIndexesImpl(indexes, *data.arraysPtr);
101  break;
102  case kPOINT:
103  case kMULTIPOINT:
104  case kLINESTRING:
105  case kMULTILINESTRING:
106  case kPOLYGON:
107  case kMULTIPOLYGON:
108  shuffleByIndexesImpl(indexes, *data.stringsPtr);
109  break;
110  default:
111  CHECK(false);
112  }
113 }
Definition: sqltypes.h:76
std::vector< std::string > * stringsPtr
Definition: sqltypes.h:234
std::vector< ArrayDatum > * arraysPtr
Definition: sqltypes.h:235
Definition: sqltypes.h:79
Definition: sqltypes.h:80
Definition: sqltypes.h:68
#define CHECK(condition)
Definition: Logger.h:291
Definition: sqltypes.h:72
SQLTypeInfo columnType
int8_t * numbersPtr
Definition: sqltypes.h:233
void shuffleByIndexesImpl(const std::vector< size_t > &indexes, T *buffer)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename T >
void Fragmenter_Namespace::shuffleByIndexesImpl ( const std::vector< size_t > &  indexes,
T *  buffer 
)

Definition at line 26 of file SortedOrderFragmenter.cpp.

References heavydb.dtypes::T.

Referenced by shuffleByIndexes().

26  {
27  std::vector<T> new_buffer;
28  new_buffer.reserve(indexes.size());
29  for (const auto i : indexes) {
30  new_buffer.push_back(buffer[i]);
31  }
32  std::memcpy(buffer, new_buffer.data(), indexes.size() * sizeof(T));
33 }

+ Here is the caller graph for this function:

template<typename T >
void Fragmenter_Namespace::shuffleByIndexesImpl ( const std::vector< size_t > &  indexes,
std::vector< T > &  buffer 
)

Definition at line 36 of file SortedOrderFragmenter.cpp.

36  {
37  std::vector<T> new_buffer;
38  new_buffer.reserve(indexes.size());
39  for (const auto i : indexes) {
40  new_buffer.push_back(buffer[i]);
41  }
42  buffer.swap(new_buffer);
43 }
size_t Fragmenter_Namespace::size_of_raw_column ( const Catalog_Namespace::Catalog cat,
const ColumnDescriptor cd,
const bool  get_logical_size = true 
)

Definition at line 82 of file InsertDataLoader.cpp.

References ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::get_compression(), SQLTypeInfo::get_logical_size(), SQLTypeInfo::get_size(), SQLTypeInfo::get_type(), SQLTypeInfo::get_type_name(), kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kENCODING_NONE, kFLOAT, kINT, kINTERVAL_DAY_TIME, kINTERVAL_YEAR_MONTH, kLINESTRING, kMULTILINESTRING, kMULTIPOINT, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, and kVARCHAR.

Referenced by compute_row_indices_of_shards(), computeRowIndicesOfShards(), and copyColumnDataOfShard().

84  {
85  switch (cd->columnType.get_type()) {
86  case kPOINT:
87  case kMULTIPOINT:
88  case kLINESTRING:
89  case kMULTILINESTRING:
90  case kPOLYGON:
91  case kMULTIPOLYGON:
92  case kARRAY:
93  throw std::runtime_error("geo and array columns have variable length elements");
94  case kBOOLEAN:
95  case kTINYINT:
96  case kSMALLINT:
97  case kINT:
98  case kBIGINT:
99  case kNUMERIC:
100  case kDECIMAL:
101  case kFLOAT:
102  case kDOUBLE:
103  case kTIMESTAMP:
104  case kTIME:
105  case kINTERVAL_DAY_TIME:
107  case kDATE:
108  return get_logical_size ? cd->columnType.get_logical_size()
109  : cd->columnType.get_size();
110  case kTEXT:
111  case kVARCHAR:
112  case kCHAR:
114  throw std::runtime_error(
115  "non encoded string columns have variable length elements");
116  }
117  return cd->columnType.get_size();
118  default:
119  throw std::runtime_error("not supported column type: " + cd->columnName + " (" +
120  cd->columnType.get_type_name() + ")");
121  }
122 }
HOST DEVICE int get_size() const
Definition: sqltypes.h:403
Definition: sqltypes.h:76
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:391
int get_logical_size() const
Definition: sqltypes.h:419
Definition: sqltypes.h:79
Definition: sqltypes.h:80
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:399
std::string get_type_name() const
Definition: sqltypes.h:482
Definition: sqltypes.h:68
Definition: sqltypes.h:72
SQLTypeInfo columnType
std::string columnName

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Fragmenter_Namespace::sortIndexes ( const ColumnDescriptor cd,
std::vector< size_t > &  indexes,
const DataBlockPtr data 
)

Definition at line 140 of file SortedOrderFragmenter.cpp.

References DataBlockPtr::arraysPtr, CHECK, ColumnDescriptor::columnType, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kNUMERIC, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, DataBlockPtr::numbersPtr, sortIndexesImpl(), and DataBlockPtr::stringsPtr.

Referenced by Fragmenter_Namespace::SortedOrderFragmenter::sortData().

142  {
143  const auto& ti = cd->columnType;
144  switch (ti.get_type()) {
145  case kBOOLEAN:
146  sortIndexesImpl(indexes, reinterpret_cast<int8_t*>(data.numbersPtr));
147  break;
148  case kTINYINT:
149  sortIndexesImpl(indexes, reinterpret_cast<int8_t*>(data.numbersPtr));
150  break;
151  case kSMALLINT:
152  sortIndexesImpl(indexes, reinterpret_cast<int16_t*>(data.numbersPtr));
153  break;
154  case kINT:
155  sortIndexesImpl(indexes, reinterpret_cast<int32_t*>(data.numbersPtr));
156  break;
157  case kBIGINT:
158  case kNUMERIC:
159  case kDECIMAL:
160  sortIndexesImpl(indexes, reinterpret_cast<int64_t*>(data.numbersPtr));
161  break;
162  case kFLOAT:
163  sortIndexesImpl(indexes, reinterpret_cast<float*>(data.numbersPtr));
164  break;
165  case kDOUBLE:
166  sortIndexesImpl(indexes, reinterpret_cast<double*>(data.numbersPtr));
167  break;
168  case kTEXT:
169  case kVARCHAR:
170  case kCHAR:
171  if (ti.is_varlen()) {
172  sortIndexesImpl(indexes, *data.stringsPtr);
173  } else {
174  switch (ti.get_size()) {
175  case 1:
176  sortIndexesImpl(indexes, reinterpret_cast<int8_t*>(data.numbersPtr));
177  break;
178  case 2:
179  sortIndexesImpl(indexes, reinterpret_cast<int16_t*>(data.numbersPtr));
180  break;
181  case 4:
182  sortIndexesImpl(indexes, reinterpret_cast<int32_t*>(data.numbersPtr));
183  break;
184  default:
185  CHECK(false);
186  }
187  }
188  break;
189  case kDATE:
190  case kTIME:
191  case kTIMESTAMP:
192  sortIndexesImpl(indexes, reinterpret_cast<int64_t*>(data.numbersPtr));
193  break;
194  case kARRAY:
195  sortIndexesImpl(indexes, *data.arraysPtr);
196  break;
197  default:
198  CHECK(false) << "invalid type '" << ti.get_type() << "' to sort";
199  }
200 }
Definition: sqltypes.h:76
std::vector< std::string > * stringsPtr
Definition: sqltypes.h:234
std::vector< ArrayDatum > * arraysPtr
Definition: sqltypes.h:235
void sortIndexesImpl(std::vector< size_t > &indexes, const T *buffer)
Definition: sqltypes.h:79
Definition: sqltypes.h:80
Definition: sqltypes.h:68
#define CHECK(condition)
Definition: Logger.h:291
Definition: sqltypes.h:72
SQLTypeInfo columnType
int8_t * numbersPtr
Definition: sqltypes.h:233

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename T >
void Fragmenter_Namespace::sortIndexesImpl ( std::vector< size_t > &  indexes,
const T *  buffer 
)

Definition at line 116 of file SortedOrderFragmenter.cpp.

References anonymous_namespace{Utm.h}::a, CHECK, and gpu_enabled::sort().

Referenced by sortIndexes().

116  {
117  CHECK(buffer);
118  std::sort(indexes.begin(), indexes.end(), [&](const auto a, const auto b) {
119  return buffer[a] < buffer[b];
120  });
121 }
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
constexpr double a
Definition: Utm.h:32
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Fragmenter_Namespace::sortIndexesImpl ( std::vector< size_t > &  indexes,
const std::vector< std::string > &  buffer 
)

Definition at line 123 of file SortedOrderFragmenter.cpp.

References anonymous_namespace{Utm.h}::a, and gpu_enabled::sort().

124  {
125  std::sort(indexes.begin(), indexes.end(), [&](const auto a, const auto b) {
126  return buffer[a].size() < buffer[b].size() ||
127  (buffer[a].size() == buffer[b].size() && buffer[a] < buffer[b]);
128  });
129 }
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
constexpr double a
Definition: Utm.h:32

+ Here is the call graph for this function:

void Fragmenter_Namespace::sortIndexesImpl ( std::vector< size_t > &  indexes,
const std::vector< ArrayDatum > &  buffer 
)

Definition at line 131 of file SortedOrderFragmenter.cpp.

References anonymous_namespace{Utm.h}::a, and gpu_enabled::sort().

132  {
133  std::sort(indexes.begin(), indexes.end(), [&](const auto a, const auto b) {
134  return buffer[a].is_null || buffer[a].length < buffer[b].length ||
135  (!buffer[b].is_null && buffer[a].length == buffer[b].length &&
136  memcmp(buffer[a].pointer, buffer[b].pointer, buffer[a].length) < 0);
137  });
138 }
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
constexpr double a
Definition: Utm.h:32

+ Here is the call graph for this function:

void Fragmenter_Namespace::wait_cleanup_threads ( std::vector< std::future< void >> &  threads)
inline

Definition at line 42 of file UpdelStorage.cpp.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::compactRows(), Fragmenter_Namespace::InsertOrderFragmenter::getVacuumOffsets(), and Fragmenter_Namespace::InsertOrderFragmenter::updateColumn().

42  {
43  for (auto& t : threads) {
44  t.get();
45  }
46  threads.clear();
47 }

+ Here is the caller graph for this function: