OmniSciDB  85c2d10cdc
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Fragmenter_Namespace::InsertOrderFragmenter Class Reference

The InsertOrderFragmenter is a child class of AbstractFragmenter, and fragments data in insert order. Likely the default fragmenter. More...

#include <InsertOrderFragmenter.h>

+ Inheritance diagram for Fragmenter_Namespace::InsertOrderFragmenter:
+ Collaboration diagram for Fragmenter_Namespace::InsertOrderFragmenter:

Public Types

using ModifyTransactionTracker = UpdelRoll
 

Public Member Functions

 InsertOrderFragmenter (const std::vector< int > chunkKeyPrefix, std::vector< Chunk_NS::Chunk > &chunkVec, Data_Namespace::DataMgr *dataMgr, Catalog_Namespace::Catalog *catalog, const int physicalTableId, const int shard, const size_t maxFragmentRows=DEFAULT_FRAGMENT_ROWS, const size_t maxChunkSize=DEFAULT_MAX_CHUNK_SIZE, const size_t pageSize=DEFAULT_PAGE_SIZE, const size_t maxRows=DEFAULT_MAX_ROWS, const Data_Namespace::MemoryLevel defaultInsertLevel=Data_Namespace::DISK_LEVEL, const bool uses_foreign_storage=false)
 
 ~InsertOrderFragmenter () override
 
TableInfo getFragmentsForQuery () override
 returns (inside QueryInfo) object all ids and row sizes of fragments More...
 
void insertData (InsertData &insert_data_struct) override
 appends data onto the most recently occuring fragment, creating a new one if necessary More...
 
void insertDataNoCheckpoint (InsertData &insert_data_struct) override
 Given data wrapped in an InsertData struct, inserts it into the correct partitions No locks and checkpoints taken needs to be managed externally. More...
 
void dropFragmentsToSize (const size_t maxRows) override
 Will truncate table to less than maxRows by dropping fragments. More...
 
void updateColumnChunkMetadata (const ColumnDescriptor *cd, const int fragment_id, const std::shared_ptr< ChunkMetadata > metadata) override
 Updates the metadata for a column chunk. More...
 
void updateChunkStats (const ColumnDescriptor *cd, std::unordered_map< int, ChunkStats > &stats_map, std::optional< Data_Namespace::MemoryLevel > memory_level) override
 Update chunk stats. More...
 
FragmentInfogetFragmentInfo (const int fragment_id) const override
 Retrieve the fragment info object for an individual fragment for editing. More...
 
int getFragmenterId () override
 get fragmenter's id More...
 
std::vector< int > getChunkKeyPrefix () const
 
std::string getFragmenterType () override
 get fragmenter's type (as string More...
 
size_t getNumRows () override
 
void setNumRows (const size_t numTuples) override
 
std::optional< ChunkUpdateStatsupdateColumn (const Catalog_Namespace::Catalog *catalog, const TableDescriptor *td, const ColumnDescriptor *cd, const int fragment_id, const std::vector< uint64_t > &frag_offsets, const std::vector< ScalarTargetValue > &rhs_values, const SQLTypeInfo &rhs_type, const Data_Namespace::MemoryLevel memory_level, UpdelRoll &updel_roll) override
 
void updateColumns (const Catalog_Namespace::Catalog *catalog, const TableDescriptor *td, const int fragmentId, const std::vector< TargetMetaInfo > sourceMetaInfo, const std::vector< const ColumnDescriptor * > columnDescriptors, const RowDataProvider &sourceDataProvider, const size_t indexOffFragmentOffsetColumn, const Data_Namespace::MemoryLevel memoryLevel, UpdelRoll &updelRoll, Executor *executor) override
 
void updateColumn (const Catalog_Namespace::Catalog *catalog, const TableDescriptor *td, const ColumnDescriptor *cd, const int fragment_id, const std::vector< uint64_t > &frag_offsets, const ScalarTargetValue &rhs_value, const SQLTypeInfo &rhs_type, const Data_Namespace::MemoryLevel memory_level, UpdelRoll &updel_roll) override
 
void updateColumnMetadata (const ColumnDescriptor *cd, FragmentInfo &fragment, std::shared_ptr< Chunk_NS::Chunk > chunk, const UpdateValuesStats &update_values_stats, const SQLTypeInfo &rhs_type, UpdelRoll &updel_roll) override
 
void updateMetadata (const Catalog_Namespace::Catalog *catalog, const MetaDataKey &key, UpdelRoll &updel_roll) override
 
void compactRows (const Catalog_Namespace::Catalog *catalog, const TableDescriptor *td, const int fragment_id, const std::vector< uint64_t > &frag_offsets, const Data_Namespace::MemoryLevel memory_level, UpdelRoll &updel_roll) override
 
const std::vector< uint64_t > getVacuumOffsets (const std::shared_ptr< Chunk_NS::Chunk > &chunk) override
 
auto getChunksForAllColumns (const TableDescriptor *td, const FragmentInfo &fragment, const Data_Namespace::MemoryLevel memory_level)
 
void dropColumns (const std::vector< int > &columnIds) override
 
bool hasDeletedRows (const int delete_column_id) override
 Iterates through chunk metadata to return whether any rows have been deleted. More...
 
- Public Member Functions inherited from Fragmenter_Namespace::AbstractFragmenter
virtual ~AbstractFragmenter ()
 

Protected Member Functions

FragmentInfocreateNewFragment (const Data_Namespace::MemoryLevel memory_level=Data_Namespace::DISK_LEVEL)
 creates new fragment, calling createChunk() method of BufferMgr to make a new chunk for each column of the table. More...
 
void deleteFragments (const std::vector< int > &dropFragIds)
 
void conditionallyInstantiateFileMgrWithParams ()
 
void getChunkMetadata ()
 
void lockInsertCheckpointData (const InsertData &insertDataStruct)
 
void insertDataImpl (InsertData &insert_data)
 
void addColumns (const InsertData &insertDataStruct)
 
 InsertOrderFragmenter (const InsertOrderFragmenter &)
 
InsertOrderFragmenteroperator= (const InsertOrderFragmenter &)
 
FragmentInfogetFragmentInfoFromId (const int fragment_id)
 
auto vacuum_fixlen_rows (const FragmentInfo &fragment, const std::shared_ptr< Chunk_NS::Chunk > &chunk, const std::vector< uint64_t > &frag_offsets)
 
auto vacuum_varlen_rows (const FragmentInfo &fragment, const std::shared_ptr< Chunk_NS::Chunk > &chunk, const std::vector< uint64_t > &frag_offsets)
 

Protected Attributes

std::vector< int > chunkKeyPrefix_
 
std::map< int, Chunk_NS::ChunkcolumnMap_
 
std::deque< std::unique_ptr
< FragmentInfo > > 
fragmentInfoVec_
 
Data_Namespace::DataMgrdataMgr_
 
Catalog_Namespace::Catalogcatalog_
 
const int physicalTableId_
 
const int shard_
 
size_t maxFragmentRows_
 
size_t pageSize_
 
size_t numTuples_
 
int maxFragmentId_
 
size_t maxChunkSize_
 
size_t maxRows_
 
std::string fragmenterType_
 
mapd_shared_mutex fragmentInfoMutex_
 
mapd_shared_mutex insertMutex_
 
Data_Namespace::MemoryLevel defaultInsertLevel_
 
const bool uses_foreign_storage_
 
bool hasMaterializedRowId_
 
int rowIdColId_
 
std::unordered_map< int, size_t > varLenColInfo_
 
std::shared_ptr< std::mutex > mutex_access_inmem_states
 
std::mutex temp_mutex_
 

Private Member Functions

bool isAddingNewColumns (const InsertData &insert_data) const
 
void dropFragmentsToSizeNoInsertLock (const size_t max_rows)
 

Detailed Description

The InsertOrderFragmenter is a child class of AbstractFragmenter, and fragments data in insert order. Likely the default fragmenter.

InsertOrderFragmenter

Definition at line 53 of file InsertOrderFragmenter.h.

Member Typedef Documentation

Constructor & Destructor Documentation

Fragmenter_Namespace::InsertOrderFragmenter::InsertOrderFragmenter ( const std::vector< int >  chunkKeyPrefix,
std::vector< Chunk_NS::Chunk > &  chunkVec,
Data_Namespace::DataMgr dataMgr,
Catalog_Namespace::Catalog catalog,
const int  physicalTableId,
const int  shard,
const size_t  maxFragmentRows = DEFAULT_FRAGMENT_ROWS,
const size_t  maxChunkSize = DEFAULT_MAX_CHUNK_SIZE,
const size_t  pageSize = DEFAULT_PAGE_SIZE,
const size_t  maxRows = DEFAULT_MAX_ROWS,
const Data_Namespace::MemoryLevel  defaultInsertLevel = Data_Namespace::DISK_LEVEL,
const bool  uses_foreign_storage = false 
)
Fragmenter_Namespace::InsertOrderFragmenter::~InsertOrderFragmenter ( )
override

Definition at line 95 of file InsertOrderFragmenter.cpp.

95 {}
Fragmenter_Namespace::InsertOrderFragmenter::InsertOrderFragmenter ( const InsertOrderFragmenter )
protected

Member Function Documentation

void Fragmenter_Namespace::InsertOrderFragmenter::addColumns ( const InsertData insertDataStruct)
protected

Definition at line 453 of file InsertOrderFragmenter.cpp.

References catalog_(), CHECK, Fragmenter_Namespace::InsertData::columnIds, Fragmenter_Namespace::InsertData::data, i, and Fragmenter_Namespace::InsertData::numRows.

453  {
454  // synchronize concurrent accesses to fragmentInfoVec_
455  mapd_unique_lock<mapd_shared_mutex> writeLock(fragmentInfoMutex_);
456  size_t numRowsLeft = insertDataStruct.numRows;
457  for (const auto columnId : insertDataStruct.columnIds) {
458  CHECK(columnMap_.end() == columnMap_.find(columnId));
459  const auto columnDesc = catalog_->getMetadataForColumn(physicalTableId_, columnId);
460  CHECK(columnDesc);
461  columnMap_.emplace(columnId, Chunk_NS::Chunk(columnDesc));
462  }
463  try {
464  for (auto const& fragmentInfo : fragmentInfoVec_) {
465  fragmentInfo->shadowChunkMetadataMap = fragmentInfo->getChunkMetadataMapPhysical();
466  auto numRowsToInsert = fragmentInfo->getPhysicalNumTuples(); // not getNumTuples()
467  size_t numRowsCanBeInserted;
468  for (size_t i = 0; i < insertDataStruct.columnIds.size(); i++) {
469  auto columnId = insertDataStruct.columnIds[i];
470  auto colDesc = catalog_->getMetadataForColumn(physicalTableId_, columnId);
471  CHECK(colDesc);
472  CHECK(columnMap_.find(columnId) != columnMap_.end());
473 
474  ChunkKey chunkKey = chunkKeyPrefix_;
475  chunkKey.push_back(columnId);
476  chunkKey.push_back(fragmentInfo->fragmentId);
477 
478  auto colMapIt = columnMap_.find(columnId);
479  auto& chunk = colMapIt->second;
480  if (chunk.isChunkOnDevice(
481  dataMgr_,
482  chunkKey,
484  fragmentInfo->deviceIds[static_cast<int>(defaultInsertLevel_)])) {
485  dataMgr_->deleteChunksWithPrefix(chunkKey);
486  }
487  chunk.createChunkBuffer(
488  dataMgr_,
489  chunkKey,
491  fragmentInfo->deviceIds[static_cast<int>(defaultInsertLevel_)]);
492  chunk.initEncoder();
493 
494  try {
495  DataBlockPtr dataCopy = insertDataStruct.data[i];
496  auto size = colDesc->columnType.get_size();
497  if (0 > size) {
498  std::unique_lock<std::mutex> lck(*mutex_access_inmem_states);
499  varLenColInfo_[columnId] = 0;
500  numRowsCanBeInserted = chunk.getNumElemsForBytesInsertData(
501  dataCopy, numRowsToInsert, 0, maxChunkSize_, true);
502  } else {
503  numRowsCanBeInserted = maxChunkSize_ / size;
504  }
505 
506  // FIXME: abort a case in which new column is wider than existing columns
507  if (numRowsCanBeInserted < numRowsToInsert) {
508  throw std::runtime_error("new column '" + colDesc->columnName +
509  "' wider than existing columns is not supported");
510  }
511 
512  auto chunkMetadata = chunk.appendData(dataCopy, numRowsToInsert, 0, true);
513  fragmentInfo->shadowChunkMetadataMap[columnId] = chunkMetadata;
514 
515  // update total size of var-len column in (actually the last) fragment
516  if (0 > size) {
517  std::unique_lock<std::mutex> lck(*mutex_access_inmem_states);
518  varLenColInfo_[columnId] = chunk.getBuffer()->size();
519  }
520  } catch (...) {
521  dataMgr_->deleteChunksWithPrefix(chunkKey);
522  throw;
523  }
524  }
525  numRowsLeft -= numRowsToInsert;
526  }
527  CHECK(0 == numRowsLeft);
528  } catch (const std::exception& e) {
529  for (const auto columnId : insertDataStruct.columnIds) {
530  columnMap_.erase(columnId);
531  }
532  throw e;
533  }
534 
535  for (auto const& fragmentInfo : fragmentInfoVec_) {
536  fragmentInfo->setChunkMetadataMap(fragmentInfo->shadowChunkMetadataMap);
537  }
538 }
std::vector< int > ChunkKey
Definition: types.h:37
std::deque< std::unique_ptr< FragmentInfo > > fragmentInfoVec_
std::shared_ptr< std::mutex > mutex_access_inmem_states
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
void deleteChunksWithPrefix(const ChunkKey &keyPrefix)
Definition: DataMgr.cpp:436
std::unordered_map< int, size_t > varLenColInfo_
#define CHECK(condition)
Definition: Logger.h:197
std::map< int, Chunk_NS::Chunk > columnMap_

+ Here is the call graph for this function:

void Fragmenter_Namespace::InsertOrderFragmenter::compactRows ( const Catalog_Namespace::Catalog catalog,
const TableDescriptor td,
const int  fragment_id,
const std::vector< uint64_t > &  frag_offsets,
const Data_Namespace::MemoryLevel  memory_level,
UpdelRoll updel_roll 
)
overridevirtual

Implements Fragmenter_Namespace::AbstractFragmenter.

Definition at line 1305 of file UpdelStorage.cpp.

References CHECK, cpu_threads(), anonymous_namespace{ResultSetReductionInterpreter.cpp}::get_element_size(), DateConverters::get_epoch_seconds_from_days(), getChunksForAllColumns(), getFragmentInfo(), UpdelRoll::numTuples, Fragmenter_Namespace::set_chunk_metadata(), Fragmenter_Namespace::set_chunk_stats(), updateColumnMetadata(), vacuum_fixlen_rows(), vacuum_varlen_rows(), and Fragmenter_Namespace::wait_cleanup_threads().

Referenced by updateColumn().

1310  {
1311  auto fragment_ptr = getFragmentInfo(fragment_id);
1312  auto& fragment = *fragment_ptr;
1313  auto chunks = getChunksForAllColumns(td, fragment, memory_level);
1314  const auto ncol = chunks.size();
1315 
1316  std::vector<ChunkUpdateStats> update_stats_per_thread(ncol);
1317 
1318  // parallel delete columns
1319  std::vector<std::future<void>> threads;
1320  auto nrows_to_vacuum = frag_offsets.size();
1321  auto nrows_in_fragment = fragment.getPhysicalNumTuples();
1322  auto nrows_to_keep = nrows_in_fragment - nrows_to_vacuum;
1323 
1324  for (size_t ci = 0; ci < chunks.size(); ++ci) {
1325  auto chunk = chunks[ci];
1326  const auto cd = chunk->getColumnDesc();
1327  const auto& col_type = cd->columnType;
1328  auto data_buffer = chunk->getBuffer();
1329  auto index_buffer = chunk->getIndexBuf();
1330  auto data_addr = data_buffer->getMemoryPtr();
1331  auto indices_addr = index_buffer ? index_buffer->getMemoryPtr() : nullptr;
1332  auto index_array = (StringOffsetT*)indices_addr;
1333  bool is_varlen = col_type.is_varlen_indeed();
1334 
1335  auto fixlen_vacuum =
1336  [=, &update_stats_per_thread, &updel_roll, &frag_offsets, &fragment] {
1337  size_t nbytes_fix_data_to_keep;
1338  if (nrows_to_keep == 0) {
1339  nbytes_fix_data_to_keep = 0;
1340  } else {
1341  nbytes_fix_data_to_keep = vacuum_fixlen_rows(fragment, chunk, frag_offsets);
1342  }
1343 
1344  data_buffer->getEncoder()->setNumElems(nrows_to_keep);
1345  data_buffer->setSize(nbytes_fix_data_to_keep);
1346  data_buffer->setUpdated();
1347 
1348  set_chunk_metadata(catalog, fragment, chunk, nrows_to_keep, updel_roll);
1349 
1350  auto daddr = data_addr;
1351  auto element_size = col_type.is_fixlen_array() ? col_type.get_size()
1352  : get_element_size(col_type);
1353  data_buffer->getEncoder()->resetChunkStats();
1354  for (size_t irow = 0; irow < nrows_to_keep; ++irow, daddr += element_size) {
1355  if (col_type.is_fixlen_array()) {
1356  auto encoder =
1357  dynamic_cast<FixedLengthArrayNoneEncoder*>(data_buffer->getEncoder());
1358  CHECK(encoder);
1359  encoder->updateMetadata((int8_t*)daddr);
1360  } else if (col_type.is_fp()) {
1361  set_chunk_stats(col_type,
1362  daddr,
1363  update_stats_per_thread[ci].new_values_stats.has_null,
1364  update_stats_per_thread[ci].new_values_stats.min_double,
1365  update_stats_per_thread[ci].new_values_stats.max_double);
1366  } else {
1367  set_chunk_stats(col_type,
1368  daddr,
1369  update_stats_per_thread[ci].new_values_stats.has_null,
1370  update_stats_per_thread[ci].new_values_stats.min_int64t,
1371  update_stats_per_thread[ci].new_values_stats.max_int64t);
1372  }
1373  }
1374  };
1375 
1376  auto varlen_vacuum = [=, &updel_roll, &frag_offsets, &fragment] {
1377  size_t nbytes_var_data_to_keep;
1378  if (nrows_to_keep == 0) {
1379  nbytes_var_data_to_keep = 0;
1380  } else {
1381  nbytes_var_data_to_keep = vacuum_varlen_rows(fragment, chunk, frag_offsets);
1382  }
1383 
1384  data_buffer->getEncoder()->setNumElems(nrows_to_keep);
1385  data_buffer->setSize(nbytes_var_data_to_keep);
1386  data_buffer->setUpdated();
1387 
1388  index_buffer->setSize(sizeof(*index_array) *
1389  (nrows_to_keep ? 1 + nrows_to_keep : 0));
1390  index_buffer->setUpdated();
1391 
1392  set_chunk_metadata(catalog, fragment, chunk, nrows_to_keep, updel_roll);
1393  };
1394 
1395  if (is_varlen) {
1396  threads.emplace_back(std::async(std::launch::async, varlen_vacuum));
1397  } else {
1398  threads.emplace_back(std::async(std::launch::async, fixlen_vacuum));
1399  }
1400  if (threads.size() >= (size_t)cpu_threads()) {
1401  wait_cleanup_threads(threads);
1402  }
1403  }
1404 
1405  wait_cleanup_threads(threads);
1406 
1407  auto key = std::make_pair(td, &fragment);
1408  updel_roll.numTuples[key] = nrows_to_keep;
1409  for (size_t ci = 0; ci < chunks.size(); ++ci) {
1410  auto chunk = chunks[ci];
1411  auto cd = chunk->getColumnDesc();
1412  if (!cd->columnType.is_fixlen_array()) {
1413  // For DATE_IN_DAYS encoded columns, data is stored in days but the metadata is
1414  // stored in seconds. Do the metadata conversion here before updating the chunk
1415  // stats.
1416  if (cd->columnType.is_date_in_days()) {
1417  auto& stats = update_stats_per_thread[ci].new_values_stats;
1418  stats.min_int64t = DateConverters::get_epoch_seconds_from_days(stats.min_int64t);
1419  stats.max_int64t = DateConverters::get_epoch_seconds_from_days(stats.max_int64t);
1420  }
1422  fragment,
1423  chunk,
1424  update_stats_per_thread[ci].new_values_stats,
1425  cd->columnType,
1426  updel_roll);
1427  }
1428  }
1429 }
auto vacuum_fixlen_rows(const FragmentInfo &fragment, const std::shared_ptr< Chunk_NS::Chunk > &chunk, const std::vector< uint64_t > &frag_offsets)
int32_t StringOffsetT
Definition: sqltypes.h:936
void wait_cleanup_threads(std::vector< std::future< void >> &threads)
int64_t get_epoch_seconds_from_days(const int64_t days)
static void set_chunk_stats(const SQLTypeInfo &col_type, int8_t *data_addr, bool &has_null, T &min, T &max)
static void set_chunk_metadata(const Catalog_Namespace::Catalog *catalog, FragmentInfo &fragment, const std::shared_ptr< Chunk_NS::Chunk > &chunk, const size_t nrows_to_keep, UpdelRoll &updel_roll)
FragmentInfo * getFragmentInfo(const int fragment_id) const override
Retrieve the fragment info object for an individual fragment for editing.
auto getChunksForAllColumns(const TableDescriptor *td, const FragmentInfo &fragment, const Data_Namespace::MemoryLevel memory_level)
#define CHECK(condition)
Definition: Logger.h:197
void updateColumnMetadata(const ColumnDescriptor *cd, FragmentInfo &fragment, std::shared_ptr< Chunk_NS::Chunk > chunk, const UpdateValuesStats &update_values_stats, const SQLTypeInfo &rhs_type, UpdelRoll &updel_roll) override
int cpu_threads()
Definition: thread_count.h:24
auto vacuum_varlen_rows(const FragmentInfo &fragment, const std::shared_ptr< Chunk_NS::Chunk > &chunk, const std::vector< uint64_t > &frag_offsets)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Fragmenter_Namespace::InsertOrderFragmenter::conditionallyInstantiateFileMgrWithParams ( )
protected

Definition at line 115 of file InsertOrderFragmenter.cpp.

References catalog_(), Data_Namespace::DISK_LEVEL, File_Namespace::FileMgrParams::max_rollback_epochs, and TableDescriptor::maxRollbackEpochs.

115  {
116  // Somewhat awkward to do this in Fragmenter, but FileMgrs are not instantiated until
117  // first use by Fragmenter, and until maxRollbackEpochs param, no options were set in
118  // storage per table
120  const TableDescriptor* td =
121  catalog_->getMetadataForTable(physicalTableId_, false /*populateFragmenter*/);
122  File_Namespace::FileMgrParams fileMgrParams;
123  fileMgrParams.max_rollback_epochs = td->maxRollbackEpochs;
125  chunkKeyPrefix_[0], chunkKeyPrefix_[1], fileMgrParams);
126  }
127 }
int32_t maxRollbackEpochs
File_Namespace::GlobalFileMgr * getGlobalFileMgr() const
Definition: DataMgr.cpp:543
void setFileMgrParams(const int32_t db_id, const int32_t tb_id, const FileMgrParams &file_mgr_params)
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.

+ Here is the call graph for this function:

FragmentInfo * Fragmenter_Namespace::InsertOrderFragmenter::createNewFragment ( const Data_Namespace::MemoryLevel  memory_level = Data_Namespace::DISK_LEVEL)
protected

creates new fragment, calling createChunk() method of BufferMgr to make a new chunk for each column of the table.

Also unpins the chunks of the previous insert buffer

Definition at line 726 of file InsertOrderFragmenter.cpp.

References Fragmenter_Namespace::anonymous_namespace{InsertOrderFragmenter.cpp}::compute_device_for_fragment(), and Fragmenter_Namespace::FragmentInfo::fragmentId.

727  {
728  // also sets the new fragment as the insertBuffer for each column
729 
730  maxFragmentId_++;
731  auto newFragmentInfo = std::make_unique<FragmentInfo>();
732  newFragmentInfo->fragmentId = maxFragmentId_;
733  newFragmentInfo->shadowNumTuples = 0;
734  newFragmentInfo->setPhysicalNumTuples(0);
735  for (const auto levelSize : dataMgr_->levelSizes_) {
736  newFragmentInfo->deviceIds.push_back(compute_device_for_fragment(
737  physicalTableId_, newFragmentInfo->fragmentId, levelSize));
738  }
739  newFragmentInfo->physicalTableId = physicalTableId_;
740  newFragmentInfo->shard = shard_;
741 
742  for (map<int, Chunk>::iterator colMapIt = columnMap_.begin();
743  colMapIt != columnMap_.end();
744  ++colMapIt) {
745  ChunkKey chunkKey = chunkKeyPrefix_;
746  chunkKey.push_back(colMapIt->second.getColumnDesc()->columnId);
747  chunkKey.push_back(maxFragmentId_);
748  colMapIt->second.createChunkBuffer(
749  dataMgr_,
750  chunkKey,
751  memoryLevel,
752  newFragmentInfo->deviceIds[static_cast<int>(memoryLevel)],
753  pageSize_);
754  colMapIt->second.initEncoder();
755  }
756 
757  mapd_lock_guard<mapd_shared_mutex> writeLock(fragmentInfoMutex_);
758  fragmentInfoVec_.push_back(std::move(newFragmentInfo));
759  return fragmentInfoVec_.back().get();
760 }
std::vector< int > ChunkKey
Definition: types.h:37
std::vector< int > levelSizes_
Definition: DataMgr.h:212
std::deque< std::unique_ptr< FragmentInfo > > fragmentInfoVec_
std::map< int, Chunk_NS::Chunk > columnMap_
int compute_device_for_fragment(const int table_id, const int fragment_id, const int num_devices)

+ Here is the call graph for this function:

void Fragmenter_Namespace::InsertOrderFragmenter::deleteFragments ( const std::vector< int > &  dropFragIds)
protected

Definition at line 252 of file InsertOrderFragmenter.cpp.

References catalog_(), and lockmgr::TableLockMgrImpl< TableDataLockMgr >::getWriteLockForTable().

252  {
253  // Fix a verified loophole on sharded logical table which is locked using logical
254  // tableId while it's its physical tables that can come here when fragments overflow
255  // during COPY. Locks on a logical table and its physical tables never intersect, which
256  // means potential races. It'll be an overkill to resolve a logical table to physical
257  // tables in DBHandler, ParseNode or other higher layers where the logical table is
258  // locked with Table Read/Write locks; it's easier to lock the logical table of its
259  // physical tables. A downside of this approach may be loss of parallel execution of
260  // deleteFragments across physical tables. Because deleteFragments is a short in-memory
261  // operation, the loss seems not a big deal.
262  auto chunkKeyPrefix = chunkKeyPrefix_;
263  if (shard_ >= 0) {
264  chunkKeyPrefix[1] = catalog_->getLogicalTableId(chunkKeyPrefix[1]);
265  }
266 
267  // need to keep lock seq as TableLock >> fragmentInfoMutex_ or
268  // SELECT and COPY may enter a deadlock
269  const auto delete_lock =
271 
272  mapd_unique_lock<mapd_shared_mutex> writeLock(fragmentInfoMutex_);
273 
274  for (const auto fragId : dropFragIds) {
275  for (const auto& col : columnMap_) {
276  int colId = col.first;
277  vector<int> fragPrefix = chunkKeyPrefix_;
278  fragPrefix.push_back(colId);
279  fragPrefix.push_back(fragId);
280  dataMgr_->deleteChunksWithPrefix(fragPrefix);
281  }
282  }
283 }
static WriteLock getWriteLockForTable(const Catalog_Namespace::Catalog &cat, const std::string &table_name)
Definition: LockMgrImpl.h:155
int getLogicalTableId(const int physicalTableId) const
Definition: Catalog.cpp:4138
void deleteChunksWithPrefix(const ChunkKey &keyPrefix)
Definition: DataMgr.cpp:436
std::map< int, Chunk_NS::Chunk > columnMap_

+ Here is the call graph for this function:

void Fragmenter_Namespace::InsertOrderFragmenter::dropColumns ( const std::vector< int > &  columnIds)
overridevirtual

Implements Fragmenter_Namespace::AbstractFragmenter.

Definition at line 540 of file InsertOrderFragmenter.cpp.

540  {
541  // prevent concurrent insert rows and drop column
542  mapd_unique_lock<mapd_shared_mutex> insertLock(insertMutex_);
543  // synchronize concurrent accesses to fragmentInfoVec_
544  mapd_unique_lock<mapd_shared_mutex> writeLock(fragmentInfoMutex_);
545  for (auto const& fragmentInfo : fragmentInfoVec_) {
546  fragmentInfo->shadowChunkMetadataMap = fragmentInfo->getChunkMetadataMapPhysical();
547  }
548 
549  for (const auto columnId : columnIds) {
550  auto cit = columnMap_.find(columnId);
551  if (columnMap_.end() != cit) {
552  columnMap_.erase(cit);
553  }
554 
555  vector<int> fragPrefix = chunkKeyPrefix_;
556  fragPrefix.push_back(columnId);
557  dataMgr_->deleteChunksWithPrefix(fragPrefix);
558 
559  for (const auto& fragmentInfo : fragmentInfoVec_) {
560  auto cmdit = fragmentInfo->shadowChunkMetadataMap.find(columnId);
561  if (fragmentInfo->shadowChunkMetadataMap.end() != cmdit) {
562  fragmentInfo->shadowChunkMetadataMap.erase(cmdit);
563  }
564  }
565  }
566  for (const auto& fragmentInfo : fragmentInfoVec_) {
567  fragmentInfo->setChunkMetadataMap(fragmentInfo->shadowChunkMetadataMap);
568  }
569 }
std::deque< std::unique_ptr< FragmentInfo > > fragmentInfoVec_
void deleteChunksWithPrefix(const ChunkKey &keyPrefix)
Definition: DataMgr.cpp:436
std::map< int, Chunk_NS::Chunk > columnMap_
void Fragmenter_Namespace::InsertOrderFragmenter::dropFragmentsToSize ( const size_t  maxRows)
overridevirtual

Will truncate table to less than maxRows by dropping fragments.

Implements Fragmenter_Namespace::AbstractFragmenter.

Definition at line 220 of file InsertOrderFragmenter.cpp.

220  {
221  mapd_unique_lock<mapd_shared_mutex> insert_lock(insertMutex_);
223 }
void dropFragmentsToSizeNoInsertLock(const size_t max_rows)
void Fragmenter_Namespace::InsertOrderFragmenter::dropFragmentsToSizeNoInsertLock ( const size_t  max_rows)
private

Definition at line 225 of file InsertOrderFragmenter.cpp.

References CHECK_GE, CHECK_GT, DROP_FRAGMENT_FACTOR, logger::INFO, and LOG.

225  {
226  // not safe to call from outside insertData
227  // b/c depends on insertLock around numTuples_
228 
229  // don't ever drop the only fragment!
230  if (numTuples_ == fragmentInfoVec_.back()->getPhysicalNumTuples()) {
231  return;
232  }
233 
234  if (numTuples_ > max_rows) {
235  size_t preNumTuples = numTuples_;
236  vector<int> dropFragIds;
237  size_t targetRows = max_rows * DROP_FRAGMENT_FACTOR;
238  while (numTuples_ > targetRows) {
239  CHECK_GT(fragmentInfoVec_.size(), size_t(0));
240  size_t numFragTuples = fragmentInfoVec_[0]->getPhysicalNumTuples();
241  dropFragIds.push_back(fragmentInfoVec_[0]->fragmentId);
242  fragmentInfoVec_.pop_front();
243  CHECK_GE(numTuples_, numFragTuples);
244  numTuples_ -= numFragTuples;
245  }
246  deleteFragments(dropFragIds);
247  LOG(INFO) << "dropFragmentsToSize, numTuples pre: " << preNumTuples
248  << " post: " << numTuples_ << " maxRows: " << max_rows;
249  }
250 }
std::deque< std::unique_ptr< FragmentInfo > > fragmentInfoVec_
#define LOG(tag)
Definition: Logger.h:188
#define CHECK_GE(x, y)
Definition: Logger.h:210
#define CHECK_GT(x, y)
Definition: Logger.h:209
void deleteFragments(const std::vector< int > &dropFragIds)
#define DROP_FRAGMENT_FACTOR
std::vector<int> Fragmenter_Namespace::InsertOrderFragmenter::getChunkKeyPrefix ( ) const
inline

Definition at line 108 of file InsertOrderFragmenter.h.

References chunkKeyPrefix_.

void Fragmenter_Namespace::InsertOrderFragmenter::getChunkMetadata ( )
protected

Definition at line 129 of file InsertOrderFragmenter.cpp.

References CHECK, CHECK_GE, Fragmenter_Namespace::anonymous_namespace{InsertOrderFragmenter.cpp}::compute_device_for_fragment(), Data_Namespace::DISK_LEVEL, logger::FATAL, LOG, gpu_enabled::sort(), and to_string().

129  {
130  if (uses_foreign_storage_ ||
132  // memory-resident tables won't have anything on disk
133  ChunkMetadataVector chunk_metadata;
135 
136  // data comes like this - database_id, table_id, column_id, fragment_id
137  // but lets sort by database_id, table_id, fragment_id, column_id
138 
139  int fragment_subkey_index = 3;
140  std::sort(chunk_metadata.begin(),
141  chunk_metadata.end(),
142  [&](const auto& pair1, const auto& pair2) {
143  return pair1.first[3] < pair2.first[3];
144  });
145 
146  for (auto chunk_itr = chunk_metadata.begin(); chunk_itr != chunk_metadata.end();
147  ++chunk_itr) {
148  int cur_column_id = chunk_itr->first[2];
149  int cur_fragment_id = chunk_itr->first[fragment_subkey_index];
150 
151  if (fragmentInfoVec_.empty() ||
152  cur_fragment_id != fragmentInfoVec_.back()->fragmentId) {
153  auto new_fragment_info = std::make_unique<Fragmenter_Namespace::FragmentInfo>();
154  CHECK(new_fragment_info);
155  maxFragmentId_ = cur_fragment_id;
156  new_fragment_info->fragmentId = cur_fragment_id;
157  new_fragment_info->setPhysicalNumTuples(chunk_itr->second->numElements);
158  numTuples_ += new_fragment_info->getPhysicalNumTuples();
159  for (const auto level_size : dataMgr_->levelSizes_) {
160  new_fragment_info->deviceIds.push_back(
161  compute_device_for_fragment(physicalTableId_, cur_fragment_id, level_size));
162  }
163  new_fragment_info->shadowNumTuples = new_fragment_info->getPhysicalNumTuples();
164  new_fragment_info->physicalTableId = physicalTableId_;
165  new_fragment_info->shard = shard_;
166  fragmentInfoVec_.emplace_back(std::move(new_fragment_info));
167  } else {
168  if (chunk_itr->second->numElements !=
169  fragmentInfoVec_.back()->getPhysicalNumTuples()) {
170  LOG(FATAL) << "Inconsistency in num tuples within fragment for table " +
171  std::to_string(physicalTableId_) + ", Column " +
172  std::to_string(cur_column_id) + ". Fragment Tuples: " +
174  fragmentInfoVec_.back()->getPhysicalNumTuples()) +
175  ", Chunk Tuples: " +
176  std::to_string(chunk_itr->second->numElements);
177  }
178  }
179  CHECK(fragmentInfoVec_.back().get());
180  fragmentInfoVec_.back().get()->setChunkMetadata(cur_column_id, chunk_itr->second);
181  }
182  }
183 
184  size_t maxFixedColSize = 0;
185 
186  for (auto colIt = columnMap_.begin(); colIt != columnMap_.end(); ++colIt) {
187  auto size = colIt->second.getColumnDesc()->columnType.get_size();
188  if (size == -1) { // variable length
189  varLenColInfo_.insert(std::make_pair(colIt->first, 0));
190  size = 8; // b/c we use this for string and array indices - gross to have magic
191  // number here
192  }
193  CHECK_GE(size, 0);
194  maxFixedColSize = std::max(maxFixedColSize, static_cast<size_t>(size));
195  }
196 
197  // this is maximum number of rows assuming everything is fixed length
198  maxFragmentRows_ = std::min(maxFragmentRows_, maxChunkSize_ / maxFixedColSize);
199 
200  if (!uses_foreign_storage_ && fragmentInfoVec_.size() > 0) {
201  // Now need to get the insert buffers for each column - should be last
202  // fragment
203  int lastFragmentId = fragmentInfoVec_.back()->fragmentId;
204  // TODO: add accessor here for safe indexing
205  int deviceId =
206  fragmentInfoVec_.back()->deviceIds[static_cast<int>(defaultInsertLevel_)];
207  for (auto colIt = columnMap_.begin(); colIt != columnMap_.end(); ++colIt) {
208  ChunkKey insertKey = chunkKeyPrefix_; // database_id and table_id
209  insertKey.push_back(colIt->first); // column id
210  insertKey.push_back(lastFragmentId); // fragment id
211  colIt->second.getChunkBuffer(dataMgr_, insertKey, defaultInsertLevel_, deviceId);
212  auto varLenColInfoIt = varLenColInfo_.find(colIt->first);
213  if (varLenColInfoIt != varLenColInfo_.end()) {
214  varLenColInfoIt->second = colIt->second.getBuffer()->size();
215  }
216  }
217  }
218 }
std::vector< int > ChunkKey
Definition: types.h:37
std::vector< int > levelSizes_
Definition: DataMgr.h:212
std::deque< std::unique_ptr< FragmentInfo > > fragmentInfoVec_
#define LOG(tag)
Definition: Logger.h:188
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
#define CHECK_GE(x, y)
Definition: Logger.h:210
std::string to_string(char const *&&v)
void getChunkMetadataVecForKeyPrefix(ChunkMetadataVector &chunkMetadataVec, const ChunkKey &keyPrefix)
Definition: DataMgr.cpp:410
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
std::unordered_map< int, size_t > varLenColInfo_
#define CHECK(condition)
Definition: Logger.h:197
std::map< int, Chunk_NS::Chunk > columnMap_
int compute_device_for_fragment(const int table_id, const int fragment_id, const int num_devices)

+ Here is the call graph for this function:

auto Fragmenter_Namespace::InsertOrderFragmenter::getChunksForAllColumns ( const TableDescriptor td,
const FragmentInfo fragment,
const Data_Namespace::MemoryLevel  memory_level 
)

Definition at line 1007 of file UpdelStorage.cpp.

References catalog_, CHECK, Catalog_Namespace::DBMetadata::dbId, Fragmenter_Namespace::FragmentInfo::fragmentId, Chunk_NS::Chunk::getChunk(), Fragmenter_Namespace::FragmentInfo::getChunkMetadataMapPhysical(), Catalog_Namespace::Catalog::getCurrentDB(), Catalog_Namespace::Catalog::getDataMgr(), Catalog_Namespace::Catalog::getMetadataForColumn(), TableDescriptor::nColumns, and TableDescriptor::tableId.

Referenced by compactRows().

1010  {
1011  std::vector<std::shared_ptr<Chunk_NS::Chunk>> chunks;
1012  // coming from updateColumn (on '$delete$' column) we dont have chunks for all columns
1013  for (int col_id = 1, ncol = 0; ncol < td->nColumns; ++col_id) {
1014  if (const auto cd = catalog_->getMetadataForColumn(td->tableId, col_id)) {
1015  ++ncol;
1016  if (!cd->isVirtualCol) {
1017  auto chunk_meta_it = fragment.getChunkMetadataMapPhysical().find(col_id);
1018  CHECK(chunk_meta_it != fragment.getChunkMetadataMapPhysical().end());
1019  ChunkKey chunk_key{
1020  catalog_->getCurrentDB().dbId, td->tableId, col_id, fragment.fragmentId};
1021  auto chunk = Chunk_NS::Chunk::getChunk(cd,
1022  &catalog_->getDataMgr(),
1023  chunk_key,
1024  memory_level,
1025  0,
1026  chunk_meta_it->second->numBytes,
1027  chunk_meta_it->second->numElements);
1028  chunks.push_back(chunk);
1029  }
1030  }
1031  }
1032  return chunks;
1033 }
std::vector< int > ChunkKey
Definition: types.h:37
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:222
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:221
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
static std::shared_ptr< Chunk > getChunk(const ColumnDescriptor *cd, DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId, const size_t num_bytes, const size_t num_elems)
Definition: Chunk.cpp:28
#define CHECK(condition)
Definition: Logger.h:197

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

int Fragmenter_Namespace::InsertOrderFragmenter::getFragmenterId ( )
inlineoverridevirtual

get fragmenter's id

Implements Fragmenter_Namespace::AbstractFragmenter.

Definition at line 107 of file InsertOrderFragmenter.h.

References chunkKeyPrefix_.

107 { return chunkKeyPrefix_.back(); }
std::string Fragmenter_Namespace::InsertOrderFragmenter::getFragmenterType ( )
inlineoverridevirtual

get fragmenter's type (as string

Implements Fragmenter_Namespace::AbstractFragmenter.

Definition at line 112 of file InsertOrderFragmenter.h.

References fragmenterType_.

FragmentInfo * Fragmenter_Namespace::InsertOrderFragmenter::getFragmentInfo ( const int  fragment_id) const
overridevirtual

Retrieve the fragment info object for an individual fragment for editing.

Implements Fragmenter_Namespace::AbstractFragmenter.

Definition at line 389 of file InsertOrderFragmenter.cpp.

References CHECK.

Referenced by compactRows(), updateColumn(), and updateColumns().

389  {
390  auto fragment_it = std::find_if(fragmentInfoVec_.begin(),
391  fragmentInfoVec_.end(),
392  [fragment_id](const auto& fragment) -> bool {
393  return fragment->fragmentId == fragment_id;
394  });
395  CHECK(fragment_it != fragmentInfoVec_.end());
396  return fragment_it->get();
397 }
std::deque< std::unique_ptr< FragmentInfo > > fragmentInfoVec_
#define CHECK(condition)
Definition: Logger.h:197

+ Here is the caller graph for this function:

FragmentInfo& Fragmenter_Namespace::InsertOrderFragmenter::getFragmentInfoFromId ( const int  fragment_id)
protected
TableInfo Fragmenter_Namespace::InsertOrderFragmenter::getFragmentsForQuery ( )
overridevirtual

returns (inside QueryInfo) object all ids and row sizes of fragments

Implements Fragmenter_Namespace::AbstractFragmenter.

Definition at line 762 of file InsertOrderFragmenter.cpp.

References Fragmenter_Namespace::TableInfo::chunkKeyPrefix, Fragmenter_Namespace::FragmentInfo::deviceIds, Fragmenter_Namespace::FragmentInfo::fragmentId, Fragmenter_Namespace::TableInfo::fragments, Fragmenter_Namespace::TableInfo::getPhysicalNumTuples(), Fragmenter_Namespace::FragmentInfo::physicalTableId, Fragmenter_Namespace::FragmentInfo::setPhysicalNumTuples(), Fragmenter_Namespace::TableInfo::setPhysicalNumTuples(), Fragmenter_Namespace::FragmentInfo::shadowNumTuples, and Fragmenter_Namespace::FragmentInfo::shard.

762  {
763  mapd_shared_lock<mapd_shared_mutex> readLock(fragmentInfoMutex_);
764  TableInfo queryInfo;
765  queryInfo.chunkKeyPrefix = chunkKeyPrefix_;
766  // right now we don't test predicate, so just return (copy of) all fragments
767  bool fragmentsExist = false;
768  if (fragmentInfoVec_.empty()) {
769  // If we have no fragments add a dummy empty fragment to make the executor
770  // not have separate logic for 0-row tables
771  int maxFragmentId = 0;
772  FragmentInfo emptyFragmentInfo;
773  emptyFragmentInfo.fragmentId = maxFragmentId;
774  emptyFragmentInfo.shadowNumTuples = 0;
775  emptyFragmentInfo.setPhysicalNumTuples(0);
776  emptyFragmentInfo.deviceIds.resize(dataMgr_->levelSizes_.size());
777  emptyFragmentInfo.physicalTableId = physicalTableId_;
778  emptyFragmentInfo.shard = shard_;
779  queryInfo.fragments.push_back(emptyFragmentInfo);
780  } else {
781  fragmentsExist = true;
782  std::for_each(
783  fragmentInfoVec_.begin(),
784  fragmentInfoVec_.end(),
785  [&queryInfo](const auto& fragment_owned_ptr) {
786  queryInfo.fragments.emplace_back(*fragment_owned_ptr); // makes a copy
787  });
788  }
789  readLock.unlock();
790  queryInfo.setPhysicalNumTuples(0);
791  auto partIt = queryInfo.fragments.begin();
792  if (fragmentsExist) {
793  while (partIt != queryInfo.fragments.end()) {
794  if (partIt->getPhysicalNumTuples() == 0) {
795  // this means that a concurrent insert query inserted tuples into a new fragment
796  // but when the query came in we didn't have this fragment. To make sure we
797  // don't mess up the executor we delete this fragment from the metadatamap
798  // (fixes earlier bug found 2015-05-08)
799  partIt = queryInfo.fragments.erase(partIt);
800  } else {
801  queryInfo.setPhysicalNumTuples(queryInfo.getPhysicalNumTuples() +
802  partIt->getPhysicalNumTuples());
803  ++partIt;
804  }
805  }
806  } else {
807  // We added a dummy fragment and know the table is empty
808  queryInfo.setPhysicalNumTuples(0);
809  }
810  return queryInfo;
811 }
std::vector< int > levelSizes_
Definition: DataMgr.h:212
std::deque< std::unique_ptr< FragmentInfo > > fragmentInfoVec_

+ Here is the call graph for this function:

size_t Fragmenter_Namespace::InsertOrderFragmenter::getNumRows ( )
inlineoverridevirtual
const std::vector< uint64_t > Fragmenter_Namespace::InsertOrderFragmenter::getVacuumOffsets ( const std::shared_ptr< Chunk_NS::Chunk > &  chunk)
overridevirtual

Implements Fragmenter_Namespace::AbstractFragmenter.

Definition at line 1036 of file UpdelStorage.cpp.

References CHECK, cpu_threads(), i, test_fsi::r, and Fragmenter_Namespace::wait_cleanup_threads().

Referenced by updateColumn().

1037  {
1038  const auto data_buffer = chunk->getBuffer();
1039  const auto data_addr = data_buffer->getMemoryPtr();
1040  const size_t nrows_in_chunk = data_buffer->size();
1041  const size_t ncore = cpu_threads();
1042  const size_t segsz = (nrows_in_chunk + ncore - 1) / ncore;
1043  std::vector<std::vector<uint64_t>> deleted_offsets;
1044  deleted_offsets.resize(ncore);
1045  std::vector<std::future<void>> threads;
1046  for (size_t rbegin = 0; rbegin < nrows_in_chunk; rbegin += segsz) {
1047  threads.emplace_back(std::async(std::launch::async, [=, &deleted_offsets] {
1048  const auto rend = std::min<size_t>(rbegin + segsz, nrows_in_chunk);
1049  const auto ithread = rbegin / segsz;
1050  CHECK(ithread < deleted_offsets.size());
1051  deleted_offsets[ithread].reserve(segsz);
1052  for (size_t r = rbegin; r < rend; ++r) {
1053  if (data_addr[r]) {
1054  deleted_offsets[ithread].push_back(r);
1055  }
1056  }
1057  }));
1058  }
1059  wait_cleanup_threads(threads);
1060  std::vector<uint64_t> all_deleted_offsets;
1061  for (size_t i = 0; i < ncore; ++i) {
1062  all_deleted_offsets.insert(
1063  all_deleted_offsets.end(), deleted_offsets[i].begin(), deleted_offsets[i].end());
1064  }
1065  return all_deleted_offsets;
1066 }
tuple r
Definition: test_fsi.py:16
void wait_cleanup_threads(std::vector< std::future< void >> &threads)
#define CHECK(condition)
Definition: Logger.h:197
int cpu_threads()
Definition: thread_count.h:24

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

bool Fragmenter_Namespace::InsertOrderFragmenter::hasDeletedRows ( const int  delete_column_id)
overridevirtual

Iterates through chunk metadata to return whether any rows have been deleted.

Implements Fragmenter_Namespace::AbstractFragmenter.

Definition at line 571 of file InsertOrderFragmenter.cpp.

References CHECK.

571  {
572  mapd_shared_lock<mapd_shared_mutex> read_lock(fragmentInfoMutex_);
573 
574  for (auto const& fragment : fragmentInfoVec_) {
575  auto chunk_meta_it = fragment->getChunkMetadataMapPhysical().find(delete_column_id);
576  CHECK(chunk_meta_it != fragment->getChunkMetadataMapPhysical().end());
577  const auto& chunk_stats = chunk_meta_it->second->chunkStats;
578  if (chunk_stats.max.tinyintval == 1) {
579  return true;
580  }
581  }
582  return false;
583 }
std::deque< std::unique_ptr< FragmentInfo > > fragmentInfoVec_
mapd_shared_lock< mapd_shared_mutex > read_lock
#define CHECK(condition)
Definition: Logger.h:197
void Fragmenter_Namespace::InsertOrderFragmenter::insertData ( InsertData insert_data_struct)
overridevirtual

appends data onto the most recently occuring fragment, creating a new one if necessary

Todo:
be able to fill up current fragment in multi-row insert before creating new fragment

Implements Fragmenter_Namespace::AbstractFragmenter.

Reimplemented in Fragmenter_Namespace::SortedOrderFragmenter.

Definition at line 414 of file InsertOrderFragmenter.cpp.

References catalog_(), Fragmenter_Namespace::InsertData::databaseId, Data_Namespace::DISK_LEVEL, and Fragmenter_Namespace::InsertData::tableId.

Referenced by Fragmenter_Namespace::SortedOrderFragmenter::insertData().

414  {
415  // TODO: this local lock will need to be centralized when ALTER COLUMN is added, bc
416  try {
417  // prevent two threads from trying to insert into the same table simultaneously
418  mapd_unique_lock<mapd_shared_mutex> insertLock(insertMutex_);
419  if (!isAddingNewColumns(insert_data_struct)) {
420  insertDataImpl(insert_data_struct);
421  } else {
422  addColumns(insert_data_struct);
423  }
424  if (defaultInsertLevel_ ==
425  Data_Namespace::DISK_LEVEL) { // only checkpoint if data is resident on disk
427  chunkKeyPrefix_[0],
428  chunkKeyPrefix_[1]); // need to checkpoint here to remove window for corruption
429  }
430  } catch (...) {
431  auto table_epochs = catalog_->getTableEpochs(insert_data_struct.databaseId,
432  insert_data_struct.tableId);
433  // the statement below deletes *this* object!
434  // relying on exception propagation at this stage
435  // until we can sort this out in a cleaner fashion
436  catalog_->setTableEpochs(insert_data_struct.databaseId, table_epochs);
437  throw;
438  }
439 }
void checkpoint(const int db_id, const int tb_id)
Definition: DataMgr.cpp:489
void addColumns(const InsertData &insertDataStruct)
bool isAddingNewColumns(const InsertData &insert_data) const
void setTableEpochs(const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
Definition: Catalog.cpp:3019
std::vector< TableEpochInfo > getTableEpochs(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:2991

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Fragmenter_Namespace::InsertOrderFragmenter::insertDataImpl ( InsertData insert_data)
protected

Definition at line 585 of file InsertOrderFragmenter.cpp.

References CHECK, CHECK_GT, CHECK_LE, Fragmenter_Namespace::InsertData::columnIds, Fragmenter_Namespace::InsertData::data, i, Fragmenter_Namespace::InsertData::is_default, DataBlockPtr::numbersPtr, and Fragmenter_Namespace::InsertData::numRows.

585  {
586  // populate deleted system column if it should exist, as it will not come from client
587  std::unique_ptr<int8_t[]> data_for_deleted_column;
588  for (const auto& cit : columnMap_) {
589  if (cit.second.getColumnDesc()->isDeletedCol) {
590  data_for_deleted_column.reset(new int8_t[insert_data.numRows]);
591  memset(data_for_deleted_column.get(), 0, insert_data.numRows);
592  insert_data.data.emplace_back(DataBlockPtr{data_for_deleted_column.get()});
593  insert_data.columnIds.push_back(cit.second.getColumnDesc()->columnId);
594  insert_data.is_default.push_back(false);
595  break;
596  }
597  }
598  CHECK(insert_data.is_default.size() == insert_data.columnIds.size());
599  std::unordered_map<int, int> inverseInsertDataColIdMap;
600  for (size_t insertId = 0; insertId < insert_data.columnIds.size(); ++insertId) {
601  inverseInsertDataColIdMap.insert(
602  std::make_pair(insert_data.columnIds[insertId], insertId));
603  }
604 
605  size_t numRowsLeft = insert_data.numRows;
606  size_t numRowsInserted = 0;
607  vector<DataBlockPtr> dataCopy =
608  insert_data.data; // bc append data will move ptr forward and this violates
609  // constness of InsertData
610  if (numRowsLeft <= 0) {
611  return;
612  }
613 
614  FragmentInfo* currentFragment{nullptr};
615 
616  // Access to fragmentInfoVec_ is protected as we are under the insertMutex_ lock but it
617  // feels fragile
618  if (fragmentInfoVec_.empty()) { // if no fragments exist for table
619  currentFragment = createNewFragment(defaultInsertLevel_);
620  } else {
621  currentFragment = fragmentInfoVec_.back().get();
622  }
623  CHECK(currentFragment);
624 
625  size_t startFragment = fragmentInfoVec_.size() - 1;
626 
627  while (numRowsLeft > 0) { // may have to create multiple fragments for bulk insert
628  // loop until done inserting all rows
629  CHECK_LE(currentFragment->shadowNumTuples, maxFragmentRows_);
630  size_t rowsLeftInCurrentFragment =
631  maxFragmentRows_ - currentFragment->shadowNumTuples;
632  size_t numRowsToInsert = min(rowsLeftInCurrentFragment, numRowsLeft);
633  if (rowsLeftInCurrentFragment != 0) {
634  for (auto& varLenColInfoIt : varLenColInfo_) {
635  CHECK_LE(varLenColInfoIt.second, maxChunkSize_);
636  size_t bytesLeft = maxChunkSize_ - varLenColInfoIt.second;
637  auto insertIdIt = inverseInsertDataColIdMap.find(varLenColInfoIt.first);
638  if (insertIdIt != inverseInsertDataColIdMap.end()) {
639  auto colMapIt = columnMap_.find(varLenColInfoIt.first);
640  numRowsToInsert = std::min(numRowsToInsert,
641  colMapIt->second.getNumElemsForBytesInsertData(
642  dataCopy[insertIdIt->second],
643  numRowsToInsert,
644  numRowsInserted,
645  bytesLeft,
646  insert_data.is_default[insertIdIt->second]));
647  }
648  }
649  }
650 
651  if (rowsLeftInCurrentFragment == 0 || numRowsToInsert == 0) {
652  currentFragment = createNewFragment(defaultInsertLevel_);
653  if (numRowsInserted == 0) {
654  startFragment++;
655  }
656  rowsLeftInCurrentFragment = maxFragmentRows_;
657  for (auto& varLenColInfoIt : varLenColInfo_) {
658  varLenColInfoIt.second = 0; // reset byte counter
659  }
660  numRowsToInsert = min(rowsLeftInCurrentFragment, numRowsLeft);
661  for (auto& varLenColInfoIt : varLenColInfo_) {
662  CHECK_LE(varLenColInfoIt.second, maxChunkSize_);
663  size_t bytesLeft = maxChunkSize_ - varLenColInfoIt.second;
664  auto insertIdIt = inverseInsertDataColIdMap.find(varLenColInfoIt.first);
665  if (insertIdIt != inverseInsertDataColIdMap.end()) {
666  auto colMapIt = columnMap_.find(varLenColInfoIt.first);
667  numRowsToInsert = std::min(numRowsToInsert,
668  colMapIt->second.getNumElemsForBytesInsertData(
669  dataCopy[insertIdIt->second],
670  numRowsToInsert,
671  numRowsInserted,
672  bytesLeft,
673  insert_data.is_default[insertIdIt->second]));
674  }
675  }
676  }
677 
678  CHECK_GT(numRowsToInsert, size_t(0)); // would put us into an endless loop as we'd
679  // never be able to insert anything
680 
681  {
682  mapd_unique_lock<mapd_shared_mutex> writeLock(fragmentInfoMutex_);
683  // for each column, append the data in the appropriate insert buffer
684  for (size_t i = 0; i < insert_data.columnIds.size(); ++i) {
685  int columnId = insert_data.columnIds[i];
686  auto colMapIt = columnMap_.find(columnId);
687  CHECK(colMapIt != columnMap_.end());
688  currentFragment->shadowChunkMetadataMap[columnId] = colMapIt->second.appendData(
689  dataCopy[i], numRowsToInsert, numRowsInserted, insert_data.is_default[i]);
690  auto varLenColInfoIt = varLenColInfo_.find(columnId);
691  if (varLenColInfoIt != varLenColInfo_.end()) {
692  varLenColInfoIt->second = colMapIt->second.getBuffer()->size();
693  }
694  }
695  if (hasMaterializedRowId_) {
696  size_t startId = maxFragmentRows_ * currentFragment->fragmentId +
697  currentFragment->shadowNumTuples;
698  auto row_id_data = std::make_unique<int64_t[]>(numRowsToInsert);
699  for (size_t i = 0; i < numRowsToInsert; ++i) {
700  row_id_data[i] = i + startId;
701  }
702  DataBlockPtr rowIdBlock;
703  rowIdBlock.numbersPtr = reinterpret_cast<int8_t*>(row_id_data.get());
704  auto colMapIt = columnMap_.find(rowIdColId_);
705  currentFragment->shadowChunkMetadataMap[rowIdColId_] =
706  colMapIt->second.appendData(rowIdBlock, numRowsToInsert, numRowsInserted);
707  }
708 
709  currentFragment->shadowNumTuples =
710  fragmentInfoVec_.back()->getPhysicalNumTuples() + numRowsToInsert;
711  numRowsLeft -= numRowsToInsert;
712  numRowsInserted += numRowsToInsert;
713  for (auto partIt = fragmentInfoVec_.begin() + startFragment;
714  partIt != fragmentInfoVec_.end();
715  ++partIt) {
716  auto fragment_ptr = partIt->get();
717  fragment_ptr->setPhysicalNumTuples(fragment_ptr->shadowNumTuples);
718  fragment_ptr->setChunkMetadataMap(fragment_ptr->shadowChunkMetadataMap);
719  }
720  }
721  }
722  numTuples_ += insert_data.numRows;
724 }
void dropFragmentsToSizeNoInsertLock(const size_t max_rows)
std::deque< std::unique_ptr< FragmentInfo > > fragmentInfoVec_
#define CHECK_GT(x, y)
Definition: Logger.h:209
FragmentInfo * createNewFragment(const Data_Namespace::MemoryLevel memory_level=Data_Namespace::DISK_LEVEL)
creates new fragment, calling createChunk() method of BufferMgr to make a new chunk for each column o...
#define CHECK_LE(x, y)
Definition: Logger.h:208
std::unordered_map< int, size_t > varLenColInfo_
#define CHECK(condition)
Definition: Logger.h:197
std::map< int, Chunk_NS::Chunk > columnMap_
int8_t * numbersPtr
Definition: sqltypes.h:220
void Fragmenter_Namespace::InsertOrderFragmenter::insertDataNoCheckpoint ( InsertData insert_data_struct)
overridevirtual

Given data wrapped in an InsertData struct, inserts it into the correct partitions No locks and checkpoints taken needs to be managed externally.

Implements Fragmenter_Namespace::AbstractFragmenter.

Reimplemented in Fragmenter_Namespace::SortedOrderFragmenter.

Definition at line 441 of file InsertOrderFragmenter.cpp.

Referenced by Fragmenter_Namespace::SortedOrderFragmenter::insertDataNoCheckpoint(), and updateColumns().

441  {
442  // TODO: this local lock will need to be centralized when ALTER COLUMN is added, bc
443  mapd_unique_lock<mapd_shared_mutex> insertLock(
444  insertMutex_); // prevent two threads from trying to insert into the same table
445  // simultaneously
446  if (!isAddingNewColumns(insert_data_struct)) {
447  insertDataImpl(insert_data_struct);
448  } else {
449  addColumns(insert_data_struct);
450  }
451 }
void addColumns(const InsertData &insertDataStruct)
bool isAddingNewColumns(const InsertData &insert_data) const

+ Here is the caller graph for this function:

bool Fragmenter_Namespace::InsertOrderFragmenter::isAddingNewColumns ( const InsertData insert_data) const
private

Definition at line 399 of file InsertOrderFragmenter.cpp.

References CHECK, and Fragmenter_Namespace::InsertData::columnIds.

399  {
400  bool all_columns_already_exist = true, all_columns_are_new = true;
401  for (const auto column_id : insert_data.columnIds) {
402  if (columnMap_.find(column_id) == columnMap_.end()) {
403  all_columns_already_exist = false;
404  } else {
405  all_columns_are_new = false;
406  }
407  }
408  // only one should be TRUE
409  bool either_all_exist_or_all_new = all_columns_already_exist ^ all_columns_are_new;
410  CHECK(either_all_exist_or_all_new);
411  return all_columns_are_new;
412 }
#define CHECK(condition)
Definition: Logger.h:197
std::map< int, Chunk_NS::Chunk > columnMap_
void Fragmenter_Namespace::InsertOrderFragmenter::lockInsertCheckpointData ( const InsertData insertDataStruct)
protected
InsertOrderFragmenter& Fragmenter_Namespace::InsertOrderFragmenter::operator= ( const InsertOrderFragmenter )
protected
void Fragmenter_Namespace::InsertOrderFragmenter::setNumRows ( const size_t  numTuples)
inlineoverridevirtual
void Fragmenter_Namespace::InsertOrderFragmenter::updateChunkStats ( const ColumnDescriptor cd,
std::unordered_map< int, ChunkStats > &  stats_map,
std::optional< Data_Namespace::MemoryLevel memory_level 
)
overridevirtual

Update chunk stats.

WARNING: This method is entirely unlocked. Higher level locks are expected to prevent any table read or write during a chunk metadata update, since we need to modify various buffers and metadata maps.

Implements Fragmenter_Namespace::AbstractFragmenter.

Definition at line 298 of file InsertOrderFragmenter.cpp.

References catalog_(), CHECK, ColumnDescriptor::columnId, ColumnDescriptor::columnType, DatumToString(), Data_Namespace::DISK_LEVEL, get_logical_type_info(), Chunk_NS::Chunk::getChunk(), SQLTypeInfo::is_dict_encoded_string(), kBIGINT, LOG, show_chunk(), VLOG, and logger::WARNING.

301  {
302  // synchronize concurrent accesses to fragmentInfoVec_
303  mapd_unique_lock<mapd_shared_mutex> writeLock(fragmentInfoMutex_);
309  if (shard_ >= 0) {
310  LOG(WARNING) << "Skipping chunk stats update for logical table " << physicalTableId_;
311  }
312 
313  CHECK(cd);
314  const auto column_id = cd->columnId;
315  const auto col_itr = columnMap_.find(column_id);
316  CHECK(col_itr != columnMap_.end());
317 
318  for (auto const& fragment : fragmentInfoVec_) {
319  auto stats_itr = stats_map.find(fragment->fragmentId);
320  if (stats_itr != stats_map.end()) {
321  auto chunk_meta_it = fragment->getChunkMetadataMapPhysical().find(column_id);
322  CHECK(chunk_meta_it != fragment->getChunkMetadataMapPhysical().end());
323  ChunkKey chunk_key{catalog_->getCurrentDB().dbId,
325  column_id,
326  fragment->fragmentId};
327  auto chunk = Chunk_NS::Chunk::getChunk(cd,
328  &catalog_->getDataMgr(),
329  chunk_key,
330  memory_level.value_or(defaultInsertLevel_),
331  0,
332  chunk_meta_it->second->numBytes,
333  chunk_meta_it->second->numElements);
334  auto buf = chunk->getBuffer();
335  CHECK(buf);
336  if (!buf->hasEncoder()) {
337  throw std::runtime_error("No encoder for chunk " + show_chunk(chunk_key));
338  }
339  auto encoder = buf->getEncoder();
340 
341  auto chunk_stats = stats_itr->second;
342 
343  auto old_chunk_metadata = std::make_shared<ChunkMetadata>();
344  encoder->getMetadata(old_chunk_metadata);
345  auto& old_chunk_stats = old_chunk_metadata->chunkStats;
346 
347  const bool didResetStats = encoder->resetChunkStats(chunk_stats);
348  // Use the logical type to display data, since the encoding should be ignored
349  const auto logical_ti = cd->columnType.is_dict_encoded_string()
351  : get_logical_type_info(cd->columnType);
352  if (!didResetStats) {
353  VLOG(3) << "Skipping chunk stats reset for " << show_chunk(chunk_key);
354  VLOG(3) << "Max: " << DatumToString(old_chunk_stats.max, logical_ti) << " -> "
355  << DatumToString(chunk_stats.max, logical_ti);
356  VLOG(3) << "Min: " << DatumToString(old_chunk_stats.min, logical_ti) << " -> "
357  << DatumToString(chunk_stats.min, logical_ti);
358  VLOG(3) << "Nulls: " << (chunk_stats.has_nulls ? "True" : "False");
359  continue; // move to next fragment
360  }
361 
362  VLOG(2) << "Resetting chunk stats for " << show_chunk(chunk_key);
363  VLOG(2) << "Max: " << DatumToString(old_chunk_stats.max, logical_ti) << " -> "
364  << DatumToString(chunk_stats.max, logical_ti);
365  VLOG(2) << "Min: " << DatumToString(old_chunk_stats.min, logical_ti) << " -> "
366  << DatumToString(chunk_stats.min, logical_ti);
367  VLOG(2) << "Nulls: " << (chunk_stats.has_nulls ? "True" : "False");
368 
369  // Reset fragment metadata map and set buffer to dirty
370  auto new_metadata = std::make_shared<ChunkMetadata>();
371  // Run through fillChunkStats to ensure any transformations to the raw metadata
372  // values get applied (e.g. for date in days)
373  encoder->getMetadata(new_metadata);
374 
375  fragment->setChunkMetadata(column_id, new_metadata);
376  fragment->shadowChunkMetadataMap =
377  fragment->getChunkMetadataMap(); // TODO(adb): needed?
379  buf->setDirty();
380  }
381  } else {
382  LOG(WARNING) << "No chunk stats update found for fragment " << fragment->fragmentId
383  << ", table " << physicalTableId_ << ", "
384  << ", column " << column_id;
385  }
386  }
387 }
std::vector< int > ChunkKey
Definition: types.h:37
std::string DatumToString(Datum d, const SQLTypeInfo &ti)
Definition: Datum.cpp:356
std::deque< std::unique_ptr< FragmentInfo > > fragmentInfoVec_
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:222
#define LOG(tag)
Definition: Logger.h:188
SQLTypeInfo get_logical_type_info(const SQLTypeInfo &type_info)
Definition: sqltypes.h:910
std::string show_chunk(const ChunkKey &key)
Definition: types.h:85
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:221
static std::shared_ptr< Chunk > getChunk(const ColumnDescriptor *cd, DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId, const size_t num_bytes, const size_t num_elems)
Definition: Chunk.cpp:28
#define CHECK(condition)
Definition: Logger.h:197
bool is_dict_encoded_string() const
Definition: sqltypes.h:525
SQLTypeInfo columnType
std::map< int, Chunk_NS::Chunk > columnMap_
#define VLOG(n)
Definition: Logger.h:291

+ Here is the call graph for this function:

std::optional< ChunkUpdateStats > Fragmenter_Namespace::InsertOrderFragmenter::updateColumn ( const Catalog_Namespace::Catalog catalog,
const TableDescriptor td,
const ColumnDescriptor cd,
const int  fragment_id,
const std::vector< uint64_t > &  frag_offsets,
const std::vector< ScalarTargetValue > &  rhs_values,
const SQLTypeInfo rhs_type,
const Data_Namespace::MemoryLevel  memory_level,
UpdelRoll updel_roll 
)
overridevirtual

Implements Fragmenter_Namespace::AbstractFragmenter.

Definition at line 638 of file UpdelStorage.cpp.

References UpdelRoll::catalog, CHECK, CHECK_GT, Fragmenter_Namespace::ChunkUpdateStats::chunk, ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, compactRows(), Data_Namespace::CPU_LEVEL, cpu_threads(), Catalog_Namespace::DBMetadata::dbId, anonymous_namespace{TypedDataAccessors.h}::decimal_to_double(), UpdelRoll::dirtyChunkeys, UpdelRoll::dirtyChunks, Fragmenter_Namespace::ChunkUpdateStats::fragment_rows_count, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), anonymous_namespace{ResultSetReductionInterpreter.cpp}::get_element_size(), DateConverters::get_epoch_seconds_from_days(), SQLTypeInfo::get_scale(), Chunk_NS::Chunk::getChunk(), Catalog_Namespace::Catalog::getCurrentDB(), Catalog_Namespace::Catalog::getDataMgr(), getFragmentInfo(), Catalog_Namespace::Catalog::getLogicalTableId(), Catalog_Namespace::Catalog::getMetadataForColumn(), Catalog_Namespace::Catalog::getMetadataForDict(), getVacuumOffsets(), SQLTypeInfo::is_boolean(), SQLTypeInfo::is_decimal(), SQLTypeInfo::is_fp(), Fragmenter_Namespace::is_integral(), SQLTypeInfo::is_string(), SQLTypeInfo::is_time(), ColumnDescriptor::isDeletedCol, kENCODING_DICT, UpdelRoll::logicalTableId, UpdelRoll::memoryLevel, UpdelRoll::mutex, Fragmenter_Namespace::ChunkUpdateStats::new_values_stats, Fragmenter_Namespace::ChunkUpdateStats::old_values_stats, anonymous_namespace{TypedDataAccessors.h}::put_null(), test_fsi::r, shard_, ColumnDescriptor::tableId, TableDescriptor::tableId, temp_mutex_, to_string(), Fragmenter_Namespace::FragmentInfo::unconditionalVacuum_, UNREACHABLE, Fragmenter_Namespace::anonymous_namespace{UpdelStorage.cpp}::update_metadata(), foreign_storage::update_stats(), updateColumnMetadata(), Fragmenter_Namespace::ChunkUpdateStats::updated_rows_count, NullAwareValidator< INNER_VALIDATOR >::validate(), and Fragmenter_Namespace::wait_cleanup_threads().

Referenced by updateColumn().

647  {
648  updel_roll.catalog = catalog;
649  updel_roll.logicalTableId = catalog->getLogicalTableId(td->tableId);
650  updel_roll.memoryLevel = memory_level;
651 
652  const size_t ncore = cpu_threads();
653  const auto nrow = frag_offsets.size();
654  const auto n_rhs_values = rhs_values.size();
655  if (0 == nrow) {
656  return {};
657  }
658  CHECK(nrow == n_rhs_values || 1 == n_rhs_values);
659 
660  auto fragment_ptr = getFragmentInfo(fragment_id);
661  auto& fragment = *fragment_ptr;
662  auto chunk_meta_it = fragment.getChunkMetadataMapPhysical().find(cd->columnId);
663  CHECK(chunk_meta_it != fragment.getChunkMetadataMapPhysical().end());
664  ChunkKey chunk_key{
665  catalog->getCurrentDB().dbId, td->tableId, cd->columnId, fragment.fragmentId};
666  auto chunk = Chunk_NS::Chunk::getChunk(cd,
667  &catalog->getDataMgr(),
668  chunk_key,
670  0,
671  chunk_meta_it->second->numBytes,
672  chunk_meta_it->second->numElements);
673 
674  std::vector<ChunkUpdateStats> update_stats_per_thread(ncore);
675 
676  // parallel update elements
677  std::vector<std::future<void>> threads;
678 
679  const auto segsz = (nrow + ncore - 1) / ncore;
680  auto dbuf = chunk->getBuffer();
681  auto dbuf_addr = dbuf->getMemoryPtr();
682  dbuf->setUpdated();
683  {
684  std::lock_guard<std::mutex> lck(updel_roll.mutex);
685  if (updel_roll.dirtyChunks.count(chunk.get()) == 0) {
686  updel_roll.dirtyChunks.emplace(chunk.get(), chunk);
687  }
688 
689  ChunkKey chunkey{updel_roll.catalog->getCurrentDB().dbId,
690  cd->tableId,
691  cd->columnId,
692  fragment.fragmentId};
693  updel_roll.dirtyChunkeys.insert(chunkey);
694  }
695  for (size_t rbegin = 0, c = 0; rbegin < nrow; ++c, rbegin += segsz) {
696  threads.emplace_back(std::async(
697  std::launch::async, [=, &update_stats_per_thread, &frag_offsets, &rhs_values] {
698  SQLTypeInfo lhs_type = cd->columnType;
699 
700  // !! not sure if this is a undocumented convention or a bug, but for a sharded
701  // table the dictionary id of a encoded string column is not specified by
702  // comp_param in physical table but somehow in logical table :) comp_param in
703  // physical table is always 0, so need to adapt accordingly...
704  auto cdl = (shard_ < 0)
705  ? cd
706  : catalog->getMetadataForColumn(
707  catalog->getLogicalTableId(td->tableId), cd->columnId);
708  CHECK(cdl);
709  DecimalOverflowValidator decimalOverflowValidator(lhs_type);
710  NullAwareValidator<DecimalOverflowValidator> nullAwareDecimalOverflowValidator(
711  lhs_type, &decimalOverflowValidator);
712  DateDaysOverflowValidator dateDaysOverflowValidator(lhs_type);
713  NullAwareValidator<DateDaysOverflowValidator> nullAwareDateOverflowValidator(
714  lhs_type, &dateDaysOverflowValidator);
715 
716  StringDictionary* stringDict{nullptr};
717  if (lhs_type.is_string()) {
718  CHECK(kENCODING_DICT == lhs_type.get_compression());
719  auto dictDesc = const_cast<DictDescriptor*>(
720  catalog->getMetadataForDict(cdl->columnType.get_comp_param()));
721  CHECK(dictDesc);
722  stringDict = dictDesc->stringDict.get();
723  CHECK(stringDict);
724  }
725 
726  for (size_t r = rbegin; r < std::min(rbegin + segsz, nrow); r++) {
727  const auto roffs = frag_offsets[r];
728  auto data_ptr = dbuf_addr + roffs * get_element_size(lhs_type);
729  auto sv = &rhs_values[1 == n_rhs_values ? 0 : r];
730  ScalarTargetValue sv2;
731 
732  // Subtle here is on the two cases of string-to-string assignments, when
733  // upstream passes RHS string as a string index instead of a preferred "real
734  // string".
735  // case #1. For "SET str_col = str_literal", it is hard to resolve temp str
736  // index
737  // in this layer, so if upstream passes a str idx here, an
738  // exception is thrown.
739  // case #2. For "SET str_col1 = str_col2", RHS str idx is converted to LHS
740  // str idx.
741  if (rhs_type.is_string()) {
742  if (const auto vp = boost::get<int64_t>(sv)) {
743  auto dictDesc = const_cast<DictDescriptor*>(
744  catalog->getMetadataForDict(rhs_type.get_comp_param()));
745  if (nullptr == dictDesc) {
746  throw std::runtime_error(
747  "UPDATE does not support cast from string literal to string "
748  "column.");
749  }
750  auto stringDict = dictDesc->stringDict.get();
751  CHECK(stringDict);
752  sv2 = NullableString(stringDict->getString(*vp));
753  sv = &sv2;
754  }
755  }
756 
757  if (const auto vp = boost::get<int64_t>(sv)) {
758  auto v = *vp;
759  if (lhs_type.is_string()) {
760  throw std::runtime_error("UPDATE does not support cast to string.");
761  }
762  int64_t old_val;
763  get_scalar<int64_t>(data_ptr, lhs_type, old_val);
764  // Handle special case where date column with date in days encoding stores
765  // metadata in epoch seconds.
766  if (lhs_type.is_date_in_days()) {
768  }
769  put_scalar<int64_t>(data_ptr, lhs_type, v, cd->columnName, &rhs_type);
770  if (lhs_type.is_decimal()) {
771  nullAwareDecimalOverflowValidator.validate<int64_t>(v);
772  int64_t decimal_val;
773  get_scalar<int64_t>(data_ptr, lhs_type, decimal_val);
774  int64_t target_value = (v == inline_int_null_value<int64_t>() &&
775  lhs_type.get_notnull() == false)
776  ? v
777  : decimal_val;
779  lhs_type, update_stats_per_thread[c], target_value, old_val);
780  auto const positive_v_and_negative_d = (v >= 0) && (decimal_val < 0);
781  auto const negative_v_and_positive_d = (v < 0) && (decimal_val >= 0);
782  if (positive_v_and_negative_d || negative_v_and_positive_d) {
783  throw std::runtime_error(
784  "Data conversion overflow on " + std::to_string(v) +
785  " from DECIMAL(" + std::to_string(rhs_type.get_dimension()) + ", " +
786  std::to_string(rhs_type.get_scale()) + ") to (" +
787  std::to_string(lhs_type.get_dimension()) + ", " +
788  std::to_string(lhs_type.get_scale()) + ")");
789  }
790  } else if (is_integral(lhs_type)) {
791  if (lhs_type.is_date_in_days()) {
792  // Store meta values in seconds
793  if (lhs_type.get_size() == 2) {
794  nullAwareDateOverflowValidator.validate<int16_t>(v);
795  } else {
796  nullAwareDateOverflowValidator.validate<int32_t>(v);
797  }
798  int64_t days;
799  get_scalar<int64_t>(data_ptr, lhs_type, days);
800  const auto seconds = DateConverters::get_epoch_seconds_from_days(days);
801  int64_t target_value = (v == inline_int_null_value<int64_t>() &&
802  lhs_type.get_notnull() == false)
803  ? NullSentinelSupplier()(lhs_type, v)
804  : seconds;
806  lhs_type, update_stats_per_thread[c], target_value, old_val);
807  } else {
808  int64_t target_value;
809  if (rhs_type.is_decimal()) {
810  target_value = round(decimal_to_double(rhs_type, v));
811  } else {
812  target_value = v;
813  }
815  lhs_type, update_stats_per_thread[c], target_value, old_val);
816  }
817  } else {
818  if (rhs_type.is_decimal()) {
819  update_metadata(lhs_type,
820  update_stats_per_thread[c],
821  decimal_to_double(rhs_type, v),
822  double(old_val));
823  } else {
824  update_metadata(lhs_type, update_stats_per_thread[c], v, old_val);
825  }
826  }
827  } else if (const auto vp = boost::get<double>(sv)) {
828  auto v = *vp;
829  if (lhs_type.is_string()) {
830  throw std::runtime_error("UPDATE does not support cast to string.");
831  }
832  double old_val;
833  get_scalar<double>(data_ptr, lhs_type, old_val);
834  put_scalar<double>(data_ptr, lhs_type, v, cd->columnName);
835  if (lhs_type.is_integer()) {
837  lhs_type, update_stats_per_thread[c], int64_t(v), int64_t(old_val));
838  } else if (lhs_type.is_fp()) {
840  lhs_type, update_stats_per_thread[c], double(v), double(old_val));
841  } else {
842  UNREACHABLE() << "Unexpected combination of a non-floating or integer "
843  "LHS with a floating RHS.";
844  }
845  } else if (const auto vp = boost::get<float>(sv)) {
846  auto v = *vp;
847  if (lhs_type.is_string()) {
848  throw std::runtime_error("UPDATE does not support cast to string.");
849  }
850  float old_val;
851  get_scalar<float>(data_ptr, lhs_type, old_val);
852  put_scalar<float>(data_ptr, lhs_type, v, cd->columnName);
853  if (lhs_type.is_integer()) {
855  lhs_type, update_stats_per_thread[c], int64_t(v), int64_t(old_val));
856  } else {
857  update_metadata(lhs_type, update_stats_per_thread[c], double(v), old_val);
858  }
859  } else if (const auto vp = boost::get<NullableString>(sv)) {
860  const auto s = boost::get<std::string>(vp);
861  const auto sval = s ? *s : std::string("");
862  if (lhs_type.is_string()) {
863  decltype(stringDict->getOrAdd(sval)) sidx;
864  {
865  std::unique_lock<std::mutex> lock(temp_mutex_);
866  sidx = stringDict->getOrAdd(sval);
867  }
868  int64_t old_val;
869  get_scalar<int64_t>(data_ptr, lhs_type, old_val);
870  put_scalar<int64_t>(data_ptr, lhs_type, sidx, cd->columnName);
872  lhs_type, update_stats_per_thread[c], int64_t(sidx), old_val);
873  } else if (sval.size() > 0) {
874  auto dval = std::atof(sval.data());
875  if (lhs_type.is_boolean()) {
876  dval = sval == "t" || sval == "true" || sval == "T" || sval == "True";
877  } else if (lhs_type.is_time()) {
878  throw std::runtime_error(
879  "Date/Time/Timestamp update not supported through translated "
880  "string path.");
881  }
882  if (lhs_type.is_fp() || lhs_type.is_decimal()) {
883  double old_val;
884  get_scalar<double>(data_ptr, lhs_type, old_val);
885  put_scalar<double>(data_ptr, lhs_type, dval, cd->columnName);
887  lhs_type, update_stats_per_thread[c], double(dval), old_val);
888  } else {
889  int64_t old_val;
890  get_scalar<int64_t>(data_ptr, lhs_type, old_val);
891  put_scalar<int64_t>(data_ptr, lhs_type, dval, cd->columnName);
893  lhs_type, update_stats_per_thread[c], int64_t(dval), old_val);
894  }
895  } else {
896  put_null(data_ptr, lhs_type, cd->columnName);
897  update_stats_per_thread[c].new_values_stats.has_null = true;
898  }
899  } else {
900  CHECK(false);
901  }
902  }
903  }));
904  if (threads.size() >= (size_t)cpu_threads()) {
905  wait_cleanup_threads(threads);
906  }
907  }
908  wait_cleanup_threads(threads);
909 
910  // for unit test
912  if (cd->isDeletedCol) {
913  const auto deleted_offsets = getVacuumOffsets(chunk);
914  if (deleted_offsets.size() > 0) {
915  compactRows(catalog, td, fragment_id, deleted_offsets, memory_level, updel_roll);
916  return {};
917  }
918  }
919  }
920  ChunkUpdateStats update_stats;
921  for (size_t c = 0; c < ncore; ++c) {
922  update_metadata(update_stats.new_values_stats,
923  update_stats_per_thread[c].new_values_stats);
924  update_metadata(update_stats.old_values_stats,
925  update_stats_per_thread[c].old_values_stats);
926  }
927 
928  CHECK_GT(fragment.shadowNumTuples, size_t(0));
930  cd, fragment, chunk, update_stats.new_values_stats, cd->columnType, updel_roll);
931  update_stats.updated_rows_count = nrow;
932  update_stats.fragment_rows_count = fragment.shadowNumTuples;
933  update_stats.chunk = chunk;
934  return update_stats;
935 }
Data_Namespace::MemoryLevel memoryLevel
Definition: UpdelRoll.h:65
std::vector< int > ChunkKey
Definition: types.h:37
void update_stats(Encoder *encoder, const SQLTypeInfo &column_type, DataBlockPtr data_block, const size_t row_count)
double decimal_to_double(const SQLTypeInfo &otype, int64_t oval)
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:222
std::map< Chunk_NS::Chunk *, std::shared_ptr< Chunk_NS::Chunk > > dirtyChunks
Definition: UpdelRoll.h:52
HOST DEVICE int get_scale() const
Definition: sqltypes.h:319
tuple r
Definition: test_fsi.py:16
std::set< ChunkKey > dirtyChunkeys
Definition: UpdelRoll.h:53
#define UNREACHABLE()
Definition: Logger.h:241
#define CHECK_GT(x, y)
Definition: Logger.h:209
std::string to_string(char const *&&v)
void wait_cleanup_threads(std::vector< std::future< void >> &threads)
int64_t get_epoch_seconds_from_days(const int64_t days)
const Catalog_Namespace::Catalog * catalog
Definition: UpdelRoll.h:63
const std::vector< uint64_t > getVacuumOffsets(const std::shared_ptr< Chunk_NS::Chunk > &chunk) override
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:221
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
static std::shared_ptr< Chunk > getChunk(const ColumnDescriptor *cd, DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId, const size_t num_bytes, const size_t num_elems)
Definition: Chunk.cpp:28
int getLogicalTableId(const int physicalTableId) const
Definition: Catalog.cpp:4138
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1444
void put_null(void *ndptr, const SQLTypeInfo &ntype, const std::string col_name)
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:322
void compactRows(const Catalog_Namespace::Catalog *catalog, const TableDescriptor *td, const int fragment_id, const std::vector< uint64_t > &frag_offsets, const Data_Namespace::MemoryLevel memory_level, UpdelRoll &updel_roll) override
FragmentInfo * getFragmentInfo(const int fragment_id) const override
Retrieve the fragment info object for an individual fragment for editing.
void update_metadata(SQLTypeInfo const &ti, ChunkUpdateStats &update_stats, int64_t const updated_val, int64_t const old_val, NullSentinelSupplier s=NullSentinelSupplier())
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:316
boost::variant< std::string, void * > NullableString
Definition: TargetValue.h:155
int logicalTableId
Definition: UpdelRoll.h:64
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:323
#define CHECK(condition)
Definition: Logger.h:197
void updateColumnMetadata(const ColumnDescriptor *cd, FragmentInfo &fragment, std::shared_ptr< Chunk_NS::Chunk > chunk, const UpdateValuesStats &update_values_stats, const SQLTypeInfo &rhs_type, UpdelRoll &updel_roll) override
Descriptor for a dictionary for a string columne.
SQLTypeInfo columnType
bool is_string() const
Definition: sqltypes.h:488
int cpu_threads()
Definition: thread_count.h:24
bool is_decimal() const
Definition: sqltypes.h:491
std::string columnName
std::mutex mutex
Definition: UpdelRoll.h:49
bool is_integral(const SQLTypeInfo &t)
boost::variant< int64_t, double, float, NullableString > ScalarTargetValue
Definition: TargetValue.h:156

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Fragmenter_Namespace::InsertOrderFragmenter::updateColumn ( const Catalog_Namespace::Catalog catalog,
const TableDescriptor td,
const ColumnDescriptor cd,
const int  fragment_id,
const std::vector< uint64_t > &  frag_offsets,
const ScalarTargetValue rhs_value,
const SQLTypeInfo rhs_type,
const Data_Namespace::MemoryLevel  memory_level,
UpdelRoll updel_roll 
)
overridevirtual

Implements Fragmenter_Namespace::AbstractFragmenter.

Definition at line 54 of file UpdelStorage.cpp.

References updateColumn().

62  {
63  updateColumn(catalog,
64  td,
65  cd,
66  fragment_id,
67  frag_offsets,
68  std::vector<ScalarTargetValue>(1, rhs_value),
69  rhs_type,
70  memory_level,
71  updel_roll);
72 }
std::optional< ChunkUpdateStats > updateColumn(const Catalog_Namespace::Catalog *catalog, const TableDescriptor *td, const ColumnDescriptor *cd, const int fragment_id, const std::vector< uint64_t > &frag_offsets, const std::vector< ScalarTargetValue > &rhs_values, const SQLTypeInfo &rhs_type, const Data_Namespace::MemoryLevel memory_level, UpdelRoll &updel_roll) override

+ Here is the call graph for this function:

void Fragmenter_Namespace::InsertOrderFragmenter::updateColumnChunkMetadata ( const ColumnDescriptor cd,
const int  fragment_id,
const std::shared_ptr< ChunkMetadata metadata 
)
overridevirtual

Updates the metadata for a column chunk.

Parameters
cd- ColumnDescriptor for the column
fragment_id- Fragment id of the chunk within the column
metadata- shared_ptr of the metadata to update column chunk with

Implements Fragmenter_Namespace::AbstractFragmenter.

Definition at line 285 of file InsertOrderFragmenter.cpp.

References CHECK, and ColumnDescriptor::columnId.

288  {
289  // synchronize concurrent accesses to fragmentInfoVec_
290  mapd_unique_lock<mapd_shared_mutex> writeLock(fragmentInfoMutex_);
291 
292  CHECK(metadata.get());
293  auto fragment_info = getFragmentInfo(fragment_id);
294  CHECK(fragment_info);
295  fragment_info->setChunkMetadata(cd->columnId, metadata);
296 }
FragmentInfo * getFragmentInfo(const int fragment_id) const override
Retrieve the fragment info object for an individual fragment for editing.
#define CHECK(condition)
Definition: Logger.h:197
void Fragmenter_Namespace::InsertOrderFragmenter::updateColumnMetadata ( const ColumnDescriptor cd,
FragmentInfo fragment,
std::shared_ptr< Chunk_NS::Chunk chunk,
const UpdateValuesStats update_values_stats,
const SQLTypeInfo rhs_type,
UpdelRoll updel_roll 
)
overridevirtual

Implements Fragmenter_Namespace::AbstractFragmenter.

Definition at line 937 of file UpdelStorage.cpp.

References UpdelRoll::catalog, UpdelRoll::chunkMetadata, ColumnDescriptor::columnId, ColumnDescriptor::columnType, Fragmenter_Namespace::FragmentInfo::getChunkMetadataMapPhysical(), Catalog_Namespace::Catalog::getMetadataForTable(), Fragmenter_Namespace::UpdateValuesStats::has_null, SQLTypeInfo::is_decimal(), Fragmenter_Namespace::is_integral(), kENCODING_DICT, Fragmenter_Namespace::UpdateValuesStats::max_double, Fragmenter_Namespace::UpdateValuesStats::max_int64t, Fragmenter_Namespace::UpdateValuesStats::min_double, Fragmenter_Namespace::UpdateValuesStats::min_int64t, UpdelRoll::mutex, UpdelRoll::numTuples, Fragmenter_Namespace::FragmentInfo::shadowNumTuples, ColumnDescriptor::tableId, and foreign_storage::update_stats().

Referenced by compactRows(), and updateColumn().

943  {
944  auto td = updel_roll.catalog->getMetadataForTable(cd->tableId);
945  auto key = std::make_pair(td, &fragment);
946  std::lock_guard<std::mutex> lck(updel_roll.mutex);
947  if (0 == updel_roll.chunkMetadata.count(key)) {
948  updel_roll.chunkMetadata[key] = fragment.getChunkMetadataMapPhysical();
949  }
950  if (0 == updel_roll.numTuples.count(key)) {
951  updel_roll.numTuples[key] = fragment.shadowNumTuples;
952  }
953  auto& chunkMetadata = updel_roll.chunkMetadata[key];
954 
955  auto buffer = chunk->getBuffer();
956  const auto& lhs_type = cd->columnType;
957 
958  auto encoder = buffer->getEncoder();
959  auto update_stats = [&encoder](auto min, auto max, auto has_null) {
960  static_assert(std::is_same<decltype(min), decltype(max)>::value,
961  "Type mismatch on min/max");
962  if (has_null) {
963  encoder->updateStats(decltype(min)(), true);
964  }
965  if (max < min) {
966  return;
967  }
968  encoder->updateStats(min, false);
969  encoder->updateStats(max, false);
970  };
971 
972  if (is_integral(lhs_type) || (lhs_type.is_decimal() && rhs_type.is_decimal())) {
973  update_stats(new_values_stats.min_int64t,
974  new_values_stats.max_int64t,
975  new_values_stats.has_null);
976  } else if (lhs_type.is_fp()) {
977  update_stats(new_values_stats.min_double,
978  new_values_stats.max_double,
979  new_values_stats.has_null);
980  } else if (lhs_type.is_decimal()) {
981  update_stats((int64_t)(new_values_stats.min_double * pow(10, lhs_type.get_scale())),
982  (int64_t)(new_values_stats.max_double * pow(10, lhs_type.get_scale())),
983  new_values_stats.has_null);
984  } else if (!lhs_type.is_array() && !lhs_type.is_geometry() &&
985  !(lhs_type.is_string() && kENCODING_DICT != lhs_type.get_compression())) {
986  update_stats(new_values_stats.min_int64t,
987  new_values_stats.max_int64t,
988  new_values_stats.has_null);
989  }
990  buffer->getEncoder()->getMetadata(chunkMetadata[cd->columnId]);
991 }
void update_stats(Encoder *encoder, const SQLTypeInfo &column_type, DataBlockPtr data_block, const size_t row_count)
const Catalog_Namespace::Catalog * catalog
Definition: UpdelRoll.h:63
std::map< MetaDataKey, ChunkMetadataMap > chunkMetadata
Definition: UpdelRoll.h:59
SQLTypeInfo columnType
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
bool is_decimal() const
Definition: sqltypes.h:491
std::mutex mutex
Definition: UpdelRoll.h:49
bool is_integral(const SQLTypeInfo &t)
std::map< MetaDataKey, size_t > numTuples
Definition: UpdelRoll.h:56

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Fragmenter_Namespace::InsertOrderFragmenter::updateColumns ( const Catalog_Namespace::Catalog catalog,
const TableDescriptor td,
const int  fragmentId,
const std::vector< TargetMetaInfo sourceMetaInfo,
const std::vector< const ColumnDescriptor * >  columnDescriptors,
const RowDataProvider sourceDataProvider,
const size_t  indexOffFragmentOffsetColumn,
const Data_Namespace::MemoryLevel  memoryLevel,
UpdelRoll updelRoll,
Executor executor 
)
overridevirtual

Implements Fragmenter_Namespace::AbstractFragmenter.

Definition at line 267 of file UpdelStorage.cpp.

References UpdelRoll::catalog, CHECK, checked_get(), Fragmenter_Namespace::InsertData::columnIds, cpu_threads(), TargetValueConverterFactory::create(), Fragmenter_Namespace::InsertData::databaseId, Catalog_Namespace::DBMetadata::dbId, UpdelRoll::dirtyChunkeys, UpdelRoll::dirtyChunks, g_enable_experimental_string_functions, Fragmenter_Namespace::get_chunks(), get_logical_type_info(), SQLTypeInfo::get_size(), Catalog_Namespace::Catalog::getCurrentDB(), Fragmenter_Namespace::RowDataProvider::getEntryAt(), Fragmenter_Namespace::RowDataProvider::getEntryCount(), getFragmentInfo(), Fragmenter_Namespace::RowDataProvider::getLiteralDictionary(), Catalog_Namespace::Catalog::getLogicalTableId(), Fragmenter_Namespace::RowDataProvider::getRowCount(), i, insertDataNoCheckpoint(), Fragmenter_Namespace::InsertData::is_default, SQLTypeInfo::is_string(), UpdelRoll::is_varlen_update, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, UpdelRoll::logicalTableId, UpdelRoll::memoryLevel, Fragmenter_Namespace::InsertData::numRows, TableDescriptor::tableId, and Fragmenter_Namespace::InsertData::tableId.

277  {
278  updelRoll.is_varlen_update = true;
279  updelRoll.catalog = catalog;
280  updelRoll.logicalTableId = catalog->getLogicalTableId(td->tableId);
281  updelRoll.memoryLevel = memoryLevel;
282 
283  size_t num_entries = sourceDataProvider.getEntryCount();
284  size_t num_rows = sourceDataProvider.getRowCount();
285 
286  if (0 == num_rows) {
287  // bail out early
288  return;
289  }
290 
292 
293  auto fragment_ptr = getFragmentInfo(fragmentId);
294  auto& fragment = *fragment_ptr;
295  std::vector<std::shared_ptr<Chunk_NS::Chunk>> chunks;
296  get_chunks(catalog, td, fragment, memoryLevel, chunks);
297  std::vector<std::unique_ptr<TargetValueConverter>> sourceDataConverters(
298  columnDescriptors.size());
299  std::vector<std::unique_ptr<ChunkToInsertDataConverter>> chunkConverters;
300  size_t indexOfDeletedColumn{0};
301  std::shared_ptr<Chunk_NS::Chunk> deletedChunk;
302  for (size_t indexOfChunk = 0; indexOfChunk < chunks.size(); indexOfChunk++) {
303  auto chunk = chunks[indexOfChunk];
304  const auto chunk_cd = chunk->getColumnDesc();
305 
306  if (chunk_cd->isDeletedCol) {
307  indexOfDeletedColumn = chunk_cd->columnId;
308  deletedChunk = chunk;
309  continue;
310  }
311 
312  auto targetColumnIt = std::find_if(columnDescriptors.begin(),
313  columnDescriptors.end(),
314  [=](const ColumnDescriptor* cd) -> bool {
315  return cd->columnId == chunk_cd->columnId;
316  });
317 
318  if (targetColumnIt != columnDescriptors.end()) {
319  auto indexOfTargetColumn = std::distance(columnDescriptors.begin(), targetColumnIt);
320 
321  auto sourceDataMetaInfo = sourceMetaInfo[indexOfTargetColumn];
322  auto targetDescriptor = columnDescriptors[indexOfTargetColumn];
323 
325  num_rows,
326  *catalog,
327  sourceDataMetaInfo,
328  targetDescriptor,
329  targetDescriptor->columnType,
330  !targetDescriptor->columnType.get_notnull(),
331  sourceDataProvider.getLiteralDictionary(),
333  ? executor->getStringDictionaryProxy(
334  sourceDataMetaInfo.get_type_info().get_comp_param(),
335  executor->getRowSetMemoryOwner(),
336  true)
337  : nullptr};
338  auto converter = factory.create(param);
339  sourceDataConverters[indexOfTargetColumn] = std::move(converter);
340 
341  if (targetDescriptor->columnType.is_geometry()) {
342  // geometry columns are composites
343  // need to skip chunks, depending on geo type
344  switch (targetDescriptor->columnType.get_type()) {
345  case kMULTIPOLYGON:
346  indexOfChunk += 5;
347  break;
348  case kPOLYGON:
349  indexOfChunk += 4;
350  break;
351  case kLINESTRING:
352  indexOfChunk += 2;
353  break;
354  case kPOINT:
355  indexOfChunk += 1;
356  break;
357  default:
358  CHECK(false); // not supported
359  }
360  }
361  } else {
362  if (chunk_cd->columnType.is_varlen() || chunk_cd->columnType.is_fixlen_array()) {
363  std::unique_ptr<ChunkToInsertDataConverter> converter;
364 
365  if (chunk_cd->columnType.is_fixlen_array()) {
366  converter =
367  std::make_unique<FixedLenArrayChunkConverter>(num_rows, chunk.get());
368  } else if (chunk_cd->columnType.is_string()) {
369  converter = std::make_unique<StringChunkConverter>(num_rows, chunk.get());
370  } else if (chunk_cd->columnType.is_geometry()) {
371  // the logical geo column is a string column
372  converter = std::make_unique<StringChunkConverter>(num_rows, chunk.get());
373  } else {
374  converter = std::make_unique<ArrayChunkConverter>(num_rows, chunk.get());
375  }
376 
377  chunkConverters.push_back(std::move(converter));
378 
379  } else if (chunk_cd->columnType.is_date_in_days()) {
380  /* Q: Why do we need this?
381  A: In variable length updates path we move the chunk content of column
382  without decoding. Since it again passes through DateDaysEncoder
383  the expected value should be in seconds, but here it will be in days.
384  Therefore, using DateChunkConverter chunk values are being scaled to
385  seconds which then ultimately encoded in days in DateDaysEncoder.
386  */
387  std::unique_ptr<ChunkToInsertDataConverter> converter;
388  const size_t physical_size = chunk_cd->columnType.get_size();
389  if (physical_size == 2) {
390  converter =
391  std::make_unique<DateChunkConverter<int16_t>>(num_rows, chunk.get());
392  } else if (physical_size == 4) {
393  converter =
394  std::make_unique<DateChunkConverter<int32_t>>(num_rows, chunk.get());
395  } else {
396  CHECK(false);
397  }
398  chunkConverters.push_back(std::move(converter));
399  } else {
400  std::unique_ptr<ChunkToInsertDataConverter> converter;
401  SQLTypeInfo logical_type = get_logical_type_info(chunk_cd->columnType);
402  int logical_size = logical_type.get_size();
403  int physical_size = chunk_cd->columnType.get_size();
404 
405  if (logical_type.is_string()) {
406  // for dicts -> logical = physical
407  logical_size = physical_size;
408  }
409 
410  if (8 == physical_size) {
411  converter = std::make_unique<ScalarChunkConverter<int64_t, int64_t>>(
412  num_rows, chunk.get());
413  } else if (4 == physical_size) {
414  if (8 == logical_size) {
415  converter = std::make_unique<ScalarChunkConverter<int32_t, int64_t>>(
416  num_rows, chunk.get());
417  } else {
418  converter = std::make_unique<ScalarChunkConverter<int32_t, int32_t>>(
419  num_rows, chunk.get());
420  }
421  } else if (2 == chunk_cd->columnType.get_size()) {
422  if (8 == logical_size) {
423  converter = std::make_unique<ScalarChunkConverter<int16_t, int64_t>>(
424  num_rows, chunk.get());
425  } else if (4 == logical_size) {
426  converter = std::make_unique<ScalarChunkConverter<int16_t, int32_t>>(
427  num_rows, chunk.get());
428  } else {
429  converter = std::make_unique<ScalarChunkConverter<int16_t, int16_t>>(
430  num_rows, chunk.get());
431  }
432  } else if (1 == chunk_cd->columnType.get_size()) {
433  if (8 == logical_size) {
434  converter = std::make_unique<ScalarChunkConverter<int8_t, int64_t>>(
435  num_rows, chunk.get());
436  } else if (4 == logical_size) {
437  converter = std::make_unique<ScalarChunkConverter<int8_t, int32_t>>(
438  num_rows, chunk.get());
439  } else if (2 == logical_size) {
440  converter = std::make_unique<ScalarChunkConverter<int8_t, int16_t>>(
441  num_rows, chunk.get());
442  } else {
443  converter = std::make_unique<ScalarChunkConverter<int8_t, int8_t>>(
444  num_rows, chunk.get());
445  }
446  } else {
447  CHECK(false); // unknown
448  }
449 
450  chunkConverters.push_back(std::move(converter));
451  }
452  }
453  }
454 
455  static boost_variant_accessor<ScalarTargetValue> SCALAR_TARGET_VALUE_ACCESSOR;
456  static boost_variant_accessor<int64_t> OFFSET_VALUE__ACCESSOR;
457 
458  updelRoll.dirtyChunks[deletedChunk.get()] = deletedChunk;
459  ChunkKey chunkey{updelRoll.catalog->getCurrentDB().dbId,
460  deletedChunk->getColumnDesc()->tableId,
461  deletedChunk->getColumnDesc()->columnId,
462  fragment.fragmentId};
463  updelRoll.dirtyChunkeys.insert(chunkey);
464  bool* deletedChunkBuffer =
465  reinterpret_cast<bool*>(deletedChunk->getBuffer()->getMemoryPtr());
466 
467  std::atomic<size_t> row_idx{0};
468 
469  auto row_converter = [&sourceDataProvider,
470  &sourceDataConverters,
471  &indexOffFragmentOffsetColumn,
472  &chunkConverters,
473  &deletedChunkBuffer,
474  &row_idx](size_t indexOfEntry) -> void {
475  // convert the source data
476  const auto row = sourceDataProvider.getEntryAt(indexOfEntry);
477  if (row.empty()) {
478  return;
479  }
480 
481  size_t indexOfRow = row_idx.fetch_add(1);
482 
483  for (size_t col = 0; col < sourceDataConverters.size(); col++) {
484  if (sourceDataConverters[col]) {
485  const auto& mapd_variant = row[col];
486  sourceDataConverters[col]->convertToColumnarFormat(indexOfRow, &mapd_variant);
487  }
488  }
489 
490  auto scalar = checked_get(
491  indexOfRow, &row[indexOffFragmentOffsetColumn], SCALAR_TARGET_VALUE_ACCESSOR);
492  auto indexInChunkBuffer = *checked_get(indexOfRow, scalar, OFFSET_VALUE__ACCESSOR);
493 
494  // convert the remaining chunks
495  for (size_t idx = 0; idx < chunkConverters.size(); idx++) {
496  chunkConverters[idx]->convertToColumnarFormat(indexOfRow, indexInChunkBuffer);
497  }
498 
499  // now mark the row as deleted
500  deletedChunkBuffer[indexInChunkBuffer] = true;
501  };
502 
503  bool can_go_parallel = num_rows > 20000;
504 
505  if (can_go_parallel) {
506  const size_t num_worker_threads = cpu_threads();
507  std::vector<std::future<void>> worker_threads;
508  for (size_t i = 0,
509  start_entry = 0,
510  stride = (num_entries + num_worker_threads - 1) / num_worker_threads;
511  i < num_worker_threads && start_entry < num_entries;
512  ++i, start_entry += stride) {
513  const auto end_entry = std::min(start_entry + stride, num_rows);
514  worker_threads.push_back(std::async(
515  std::launch::async,
516  [&row_converter](const size_t start, const size_t end) {
517  for (size_t indexOfRow = start; indexOfRow < end; ++indexOfRow) {
518  row_converter(indexOfRow);
519  }
520  },
521  start_entry,
522  end_entry));
523  }
524 
525  for (auto& child : worker_threads) {
526  child.wait();
527  }
528 
529  } else {
530  for (size_t entryIdx = 0; entryIdx < num_entries; entryIdx++) {
531  row_converter(entryIdx);
532  }
533  }
534 
536  insert_data.databaseId = catalog->getCurrentDB().dbId;
537  insert_data.tableId = td->tableId;
538 
539  for (size_t i = 0; i < chunkConverters.size(); i++) {
540  chunkConverters[i]->addDataBlocksToInsertData(insert_data);
541  continue;
542  }
543 
544  for (size_t i = 0; i < sourceDataConverters.size(); i++) {
545  if (sourceDataConverters[i]) {
546  sourceDataConverters[i]->addDataBlocksToInsertData(insert_data);
547  }
548  continue;
549  }
550 
551  insert_data.numRows = num_rows;
552  insert_data.is_default.resize(insert_data.columnIds.size(), false);
553  insertDataNoCheckpoint(insert_data);
554 
555  // update metdata for deleted chunk as we are doing special handling
556  auto chunk_meta_it = fragment.getChunkMetadataMap().find(indexOfDeletedColumn);
557  CHECK(chunk_meta_it != fragment.getChunkMetadataMap().end());
558  chunk_meta_it->second->chunkStats.max.boolval = 1;
559 
560  // Im not completely sure that we need to do this in fragmented and on the buffer
561  // but leaving this alone for now
562  if (!deletedChunk->getBuffer()->hasEncoder()) {
563  deletedChunk->initEncoder();
564  }
565  deletedChunk->getBuffer()->getEncoder()->updateStats(static_cast<int64_t>(true), false);
566 
567  if (fragment.shadowNumTuples > deletedChunk->getBuffer()->getEncoder()->getNumElems()) {
568  // An append to the same fragment will increase shadowNumTuples.
569  // Update NumElems in this case. Otherwise, use existing NumElems.
570  deletedChunk->getBuffer()->getEncoder()->setNumElems(fragment.shadowNumTuples);
571  }
572  deletedChunk->getBuffer()->setUpdated();
573 }
Data_Namespace::MemoryLevel memoryLevel
Definition: UpdelRoll.h:65
bool is_varlen_update
Definition: UpdelRoll.h:67
std::vector< int > ChunkKey
Definition: types.h:37
HOST DEVICE int get_size() const
Definition: sqltypes.h:324
std::map< Chunk_NS::Chunk *, std::shared_ptr< Chunk_NS::Chunk > > dirtyChunks
Definition: UpdelRoll.h:52
std::set< ChunkKey > dirtyChunkeys
Definition: UpdelRoll.h:53
SQLTypeInfo get_logical_type_info(const SQLTypeInfo &type_info)
Definition: sqltypes.h:910
std::vector< bool > is_default
Definition: Fragmenter.h:66
int tableId
identifies the database into which the data is being inserted
Definition: Fragmenter.h:61
size_t numRows
a vector of column ids for the row(s) being inserted
Definition: Fragmenter.h:63
static int get_chunks(const Catalog_Namespace::Catalog *catalog, const TableDescriptor *td, const FragmentInfo &fragment, const Data_Namespace::MemoryLevel memory_level, std::vector< std::shared_ptr< Chunk_NS::Chunk >> &chunks)
const Catalog_Namespace::Catalog * catalog
Definition: UpdelRoll.h:63
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:221
virtual size_t const getEntryCount() const =0
int getLogicalTableId(const int physicalTableId) const
Definition: Catalog.cpp:4138
specifies the content in-memory of a row in the column metadata table
const RETURN_TYPE * checked_get(size_t row, const SOURCE_TYPE *boost_variant, boost_variant_accessor< RETURN_TYPE > &accessor)
FragmentInfo * getFragmentInfo(const int fragment_id) const override
Retrieve the fragment info object for an individual fragment for editing.
bool g_enable_experimental_string_functions
virtual StringDictionaryProxy * getLiteralDictionary() const =0
int logicalTableId
Definition: UpdelRoll.h:64
#define CHECK(condition)
Definition: Logger.h:197
The data to be inserted using the fragment manager.
Definition: Fragmenter.h:59
void insertDataNoCheckpoint(InsertData &insert_data_struct) override
Given data wrapped in an InsertData struct, inserts it into the correct partitions No locks and check...
bool is_string() const
Definition: sqltypes.h:488
virtual std::vector< TargetValue > getEntryAt(const size_t index) const =0
int cpu_threads()
Definition: thread_count.h:24
std::vector< int > columnIds
identifies the table into which the data is being inserted
Definition: Fragmenter.h:62
std::unique_ptr< TargetValueConverter > create(ConverterCreateParameter param)
virtual size_t const getRowCount() const =0

+ Here is the call graph for this function:

void Fragmenter_Namespace::InsertOrderFragmenter::updateMetadata ( const Catalog_Namespace::Catalog catalog,
const MetaDataKey key,
UpdelRoll updel_roll 
)
overridevirtual

Implements Fragmenter_Namespace::AbstractFragmenter.

Definition at line 993 of file UpdelStorage.cpp.

References UpdelRoll::chunkMetadata, fragmentInfoMutex_, and UpdelRoll::numTuples.

995  {
996  mapd_unique_lock<mapd_shared_mutex> writeLock(fragmentInfoMutex_);
997  if (updel_roll.chunkMetadata.count(key)) {
998  auto& fragmentInfo = *key.second;
999  const auto& chunkMetadata = updel_roll.chunkMetadata[key];
1000  fragmentInfo.shadowChunkMetadataMap = chunkMetadata;
1001  fragmentInfo.setChunkMetadataMap(chunkMetadata);
1002  fragmentInfo.shadowNumTuples = updel_roll.numTuples[key];
1003  fragmentInfo.setPhysicalNumTuples(fragmentInfo.shadowNumTuples);
1004  }
1005 }
std::map< MetaDataKey, ChunkMetadataMap > chunkMetadata
Definition: UpdelRoll.h:59
std::map< MetaDataKey, size_t > numTuples
Definition: UpdelRoll.h:56
auto Fragmenter_Namespace::InsertOrderFragmenter::vacuum_fixlen_rows ( const FragmentInfo fragment,
const std::shared_ptr< Chunk_NS::Chunk > &  chunk,
const std::vector< uint64_t > &  frag_offsets 
)
protected

Definition at line 1108 of file UpdelStorage.cpp.

References anonymous_namespace{ResultSetReductionInterpreter.cpp}::get_element_size(), and Fragmenter_Namespace::FragmentInfo::getPhysicalNumTuples().

Referenced by compactRows().

1111  {
1112  const auto cd = chunk->getColumnDesc();
1113  const auto& col_type = cd->columnType;
1114  auto data_buffer = chunk->getBuffer();
1115  auto data_addr = data_buffer->getMemoryPtr();
1116  auto element_size =
1117  col_type.is_fixlen_array() ? col_type.get_size() : get_element_size(col_type);
1118  int64_t irow_of_blk_to_keep = 0; // head of next row block to keep
1119  int64_t irow_of_blk_to_fill = 0; // row offset to fit the kept block
1120  size_t nbytes_fix_data_to_keep = 0;
1121  auto nrows_to_vacuum = frag_offsets.size();
1122  auto nrows_in_fragment = fragment.getPhysicalNumTuples();
1123  for (size_t irow = 0; irow <= nrows_to_vacuum; irow++) {
1124  auto is_last_one = irow == nrows_to_vacuum;
1125  auto irow_to_vacuum = is_last_one ? nrows_in_fragment : frag_offsets[irow];
1126  auto maddr_to_vacuum = data_addr;
1127  int64_t nrows_to_keep = irow_to_vacuum - irow_of_blk_to_keep;
1128  if (nrows_to_keep > 0) {
1129  auto nbytes_to_keep = nrows_to_keep * element_size;
1130  if (irow_of_blk_to_fill != irow_of_blk_to_keep) {
1131  // move curr fixlen row block toward front
1132  memmove(maddr_to_vacuum + irow_of_blk_to_fill * element_size,
1133  maddr_to_vacuum + irow_of_blk_to_keep * element_size,
1134  nbytes_to_keep);
1135  }
1136  irow_of_blk_to_fill += nrows_to_keep;
1137  nbytes_fix_data_to_keep += nbytes_to_keep;
1138  }
1139  irow_of_blk_to_keep = irow_to_vacuum + 1;
1140  }
1141  return nbytes_fix_data_to_keep;
1142 }

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

auto Fragmenter_Namespace::InsertOrderFragmenter::vacuum_varlen_rows ( const FragmentInfo fragment,
const std::shared_ptr< Chunk_NS::Chunk > &  chunk,
const std::vector< uint64_t > &  frag_offsets 
)
protected

Definition at line 1222 of file UpdelStorage.cpp.

References CHECK, Fragmenter_Namespace::get_buffer_offset(), Fragmenter_Namespace::get_null_padding(), Fragmenter_Namespace::get_var_len_null_array_indexes(), Fragmenter_Namespace::FragmentInfo::getPhysicalNumTuples(), and i.

Referenced by compactRows().

1225  {
1226  auto is_varlen_array = chunk->getColumnDesc()->columnType.is_varlen_array();
1227  auto data_buffer = chunk->getBuffer();
1228  CHECK(data_buffer);
1229  auto index_buffer = chunk->getIndexBuf();
1230  CHECK(index_buffer);
1231  auto data_addr = data_buffer->getMemoryPtr();
1232  auto indices_addr = index_buffer->getMemoryPtr();
1233  CHECK(indices_addr);
1234  auto index_array = (StringOffsetT*)indices_addr;
1235  int64_t irow_of_blk_to_keep = 0; // head of next row block to keep
1236  int64_t irow_of_blk_to_fill = 0; // row offset to fit the kept block
1237  size_t nbytes_fix_data_to_keep = 0;
1238  auto nrows_in_fragment = fragment.getPhysicalNumTuples();
1239  size_t null_padding =
1240  get_null_padding(is_varlen_array, frag_offsets, index_array, nrows_in_fragment);
1241  size_t nbytes_var_data_to_keep = null_padding;
1242  auto null_array_indexes = get_var_len_null_array_indexes(
1243  chunk->getColumnDesc()->columnType, frag_offsets, index_array, nrows_in_fragment);
1244  auto nrows_to_vacuum = frag_offsets.size();
1245  for (size_t irow = 0; irow <= nrows_to_vacuum; irow++) {
1246  auto is_last_one = irow == nrows_to_vacuum;
1247  auto irow_to_vacuum = is_last_one ? nrows_in_fragment : frag_offsets[irow];
1248  auto maddr_to_vacuum = data_addr;
1249  int64_t nrows_to_keep = irow_to_vacuum - irow_of_blk_to_keep;
1250  if (nrows_to_keep > 0) {
1251  auto ibyte_var_data_to_keep = nbytes_var_data_to_keep;
1252  auto deleted_row_start_offset =
1253  get_buffer_offset(is_varlen_array, index_array, irow_to_vacuum);
1254  auto kept_row_start_offset =
1255  get_buffer_offset(is_varlen_array, index_array, irow_of_blk_to_keep);
1256  auto nbytes_to_keep =
1257  (is_last_one ? data_buffer->size() : deleted_row_start_offset) -
1258  kept_row_start_offset;
1259  if (irow_of_blk_to_fill != irow_of_blk_to_keep) {
1260  if (nbytes_to_keep > 0) {
1261  CHECK(data_addr);
1262  // move curr varlen row block toward front
1263  memmove(data_addr + ibyte_var_data_to_keep,
1264  data_addr + kept_row_start_offset,
1265  nbytes_to_keep);
1266  }
1267 
1268  const auto base_offset = kept_row_start_offset;
1269  for (int64_t i = 0; i < nrows_to_keep; ++i) {
1270  auto update_index = irow_of_blk_to_keep + i;
1271  auto offset = get_buffer_offset(is_varlen_array, index_array, update_index);
1272  index_array[update_index] = ibyte_var_data_to_keep + (offset - base_offset);
1273  }
1274  }
1275  nbytes_var_data_to_keep += nbytes_to_keep;
1276  maddr_to_vacuum = indices_addr;
1277 
1278  constexpr static auto index_element_size = sizeof(StringOffsetT);
1279  nbytes_to_keep = nrows_to_keep * index_element_size;
1280  if (irow_of_blk_to_fill != irow_of_blk_to_keep) {
1281  // move curr fixlen row block toward front
1282  memmove(maddr_to_vacuum + irow_of_blk_to_fill * index_element_size,
1283  maddr_to_vacuum + irow_of_blk_to_keep * index_element_size,
1284  nbytes_to_keep);
1285  }
1286  irow_of_blk_to_fill += nrows_to_keep;
1287  nbytes_fix_data_to_keep += nbytes_to_keep;
1288  }
1289  irow_of_blk_to_keep = irow_to_vacuum + 1;
1290  }
1291 
1292  // Set expected null padding, last offset, and negative values for null array offsets.
1293  index_array[0] = null_padding;
1294  auto post_vacuum_row_count = nrows_in_fragment - nrows_to_vacuum;
1295  index_array[post_vacuum_row_count] = nbytes_var_data_to_keep;
1296  if (!is_varlen_array) {
1297  CHECK(null_array_indexes.empty());
1298  }
1299  for (auto index : null_array_indexes) {
1300  index_array[index + 1] = -1 * std::abs(index_array[index + 1]);
1301  }
1302  return nbytes_var_data_to_keep;
1303 }
std::set< size_t > get_var_len_null_array_indexes(const SQLTypeInfo sql_type_info, const std::vector< uint64_t > &frag_offsets, const StringOffsetT *index_array, size_t fragment_row_count)
int32_t StringOffsetT
Definition: sqltypes.h:936
#define CHECK(condition)
Definition: Logger.h:197
StringOffsetT get_buffer_offset(bool is_varlen_array, const StringOffsetT *index_array, size_t index)
size_t get_null_padding(bool is_varlen_array, const std::vector< uint64_t > &frag_offsets, const StringOffsetT *index_array, size_t fragment_row_count)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Member Data Documentation

Catalog_Namespace::Catalog* Fragmenter_Namespace::InsertOrderFragmenter::catalog_
protected
std::vector<int> Fragmenter_Namespace::InsertOrderFragmenter::chunkKeyPrefix_
protected

Definition at line 178 of file InsertOrderFragmenter.h.

Referenced by getChunkKeyPrefix(), and getFragmenterId().

std::map<int, Chunk_NS::Chunk> Fragmenter_Namespace::InsertOrderFragmenter::columnMap_
protected

stores a map of column id to metadata about that column

Definition at line 180 of file InsertOrderFragmenter.h.

Data_Namespace::DataMgr* Fragmenter_Namespace::InsertOrderFragmenter::dataMgr_
protected

Definition at line 184 of file InsertOrderFragmenter.h.

Data_Namespace::MemoryLevel Fragmenter_Namespace::InsertOrderFragmenter::defaultInsertLevel_
protected

Definition at line 201 of file InsertOrderFragmenter.h.

std::string Fragmenter_Namespace::InsertOrderFragmenter::fragmenterType_
protected

Definition at line 195 of file InsertOrderFragmenter.h.

Referenced by getFragmenterType().

mapd_shared_mutex Fragmenter_Namespace::InsertOrderFragmenter::fragmentInfoMutex_
protected

Definition at line 197 of file InsertOrderFragmenter.h.

Referenced by updateMetadata().

std::deque<std::unique_ptr<FragmentInfo> > Fragmenter_Namespace::InsertOrderFragmenter::fragmentInfoVec_
protected

data about each fragment stored - id and number of rows

Definition at line 182 of file InsertOrderFragmenter.h.

bool Fragmenter_Namespace::InsertOrderFragmenter::hasMaterializedRowId_
protected

Definition at line 203 of file InsertOrderFragmenter.h.

mapd_shared_mutex Fragmenter_Namespace::InsertOrderFragmenter::insertMutex_
protected

Definition at line 199 of file InsertOrderFragmenter.h.

size_t Fragmenter_Namespace::InsertOrderFragmenter::maxChunkSize_
protected

Definition at line 193 of file InsertOrderFragmenter.h.

int Fragmenter_Namespace::InsertOrderFragmenter::maxFragmentId_
protected

Definition at line 192 of file InsertOrderFragmenter.h.

size_t Fragmenter_Namespace::InsertOrderFragmenter::maxFragmentRows_
protected

Definition at line 188 of file InsertOrderFragmenter.h.

size_t Fragmenter_Namespace::InsertOrderFragmenter::maxRows_
protected

Definition at line 194 of file InsertOrderFragmenter.h.

std::shared_ptr<std::mutex> Fragmenter_Namespace::InsertOrderFragmenter::mutex_access_inmem_states
protected

Definition at line 206 of file InsertOrderFragmenter.h.

size_t Fragmenter_Namespace::InsertOrderFragmenter::numTuples_
protected

Definition at line 191 of file InsertOrderFragmenter.h.

Referenced by getNumRows(), and setNumRows().

size_t Fragmenter_Namespace::InsertOrderFragmenter::pageSize_
protected

Definition at line 189 of file InsertOrderFragmenter.h.

const int Fragmenter_Namespace::InsertOrderFragmenter::physicalTableId_
protected
int Fragmenter_Namespace::InsertOrderFragmenter::rowIdColId_
protected

Definition at line 204 of file InsertOrderFragmenter.h.

const int Fragmenter_Namespace::InsertOrderFragmenter::shard_
protected

Definition at line 187 of file InsertOrderFragmenter.h.

Referenced by updateColumn().

std::mutex Fragmenter_Namespace::InsertOrderFragmenter::temp_mutex_
mutableprotected

Definition at line 230 of file InsertOrderFragmenter.h.

Referenced by updateColumn().

const bool Fragmenter_Namespace::InsertOrderFragmenter::uses_foreign_storage_
protected

Definition at line 202 of file InsertOrderFragmenter.h.

std::unordered_map<int, size_t> Fragmenter_Namespace::InsertOrderFragmenter::varLenColInfo_
protected

Definition at line 205 of file InsertOrderFragmenter.h.


The documentation for this class was generated from the following files: