OmniSciDB  c1a53651b2
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Fragmenter_Namespace::anonymous_namespace{InsertOrderFragmenter.cpp} Namespace Reference

Functions

int compute_device_for_fragment (const int table_id, const int fragment_id, const int num_devices)
 
size_t get_num_rows_to_insert (const size_t rows_left_in_current_fragment, const size_t num_rows_left, const size_t num_rows_inserted, const std::unordered_map< int, size_t > &var_len_col_info, const size_t max_chunk_size, const InsertChunks &insert_chunks, std::map< int, Chunk_NS::Chunk > &column_map, const std::vector< size_t > &valid_row_indices)
 

Function Documentation

int Fragmenter_Namespace::anonymous_namespace{InsertOrderFragmenter.cpp}::compute_device_for_fragment ( const int  table_id,
const int  fragment_id,
const int  num_devices 
)

Offset the fragment ID by the table ID, meaning single fragment tables end up balanced across multiple GPUs instead of all falling to GPU 0.

Definition at line 103 of file InsertOrderFragmenter.cpp.

References g_use_table_device_offset.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::createNewFragment(), and Fragmenter_Namespace::InsertOrderFragmenter::getChunkMetadata().

105  {
107  return (table_id + fragment_id) % num_devices;
108  } else {
109  return fragment_id % num_devices;
110  }
111 }
bool g_use_table_device_offset

+ Here is the caller graph for this function:

size_t Fragmenter_Namespace::anonymous_namespace{InsertOrderFragmenter.cpp}::get_num_rows_to_insert ( const size_t  rows_left_in_current_fragment,
const size_t  num_rows_left,
const size_t  num_rows_inserted,
const std::unordered_map< int, size_t > &  var_len_col_info,
const size_t  max_chunk_size,
const InsertChunks &  insert_chunks,
std::map< int, Chunk_NS::Chunk > &  column_map,
const std::vector< size_t > &  valid_row_indices 
)

Definition at line 113 of file InsertOrderFragmenter.cpp.

References CHECK, CHECK_LE, and Fragmenter_Namespace::InsertChunks::chunks.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::insertChunksImpl().

120  {
121  size_t num_rows_to_insert = min(rows_left_in_current_fragment, num_rows_left);
122  if (rows_left_in_current_fragment != 0) {
123  for (const auto& var_len_col_info_it : var_len_col_info) {
124  CHECK_LE(var_len_col_info_it.second, max_chunk_size);
125  size_t bytes_left = max_chunk_size - var_len_col_info_it.second;
126  auto find_it = insert_chunks.chunks.find(var_len_col_info_it.first);
127  if (find_it == insert_chunks.chunks.end()) {
128  continue;
129  }
130  const auto& chunk = find_it->second;
131  auto column_type = chunk->getColumnDesc()->columnType;
132  const int8_t* index_buffer_ptr =
133  column_type.is_varlen_indeed() ? chunk->getIndexBuf()->getMemoryPtr() : nullptr;
134  CHECK(column_type.is_varlen());
135 
136  auto col_map_it = column_map.find(var_len_col_info_it.first);
137  num_rows_to_insert =
138  std::min(num_rows_to_insert,
139  col_map_it->second.getNumElemsForBytesEncodedDataAtIndices(
140  index_buffer_ptr, valid_row_indices, bytes_left));
141  }
142  }
143  return num_rows_to_insert;
144 }
#define CHECK_LE(x, y)
Definition: Logger.h:304
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the caller graph for this function: