Fragmenter_Namespace::anonymous_namespace{InsertOrderFragmenter.cpp} Namespace Reference

Functions
int	compute_device_for_fragment (const int table_id, const int fragment_id, const int num_devices)
size_t	get_num_rows_to_insert (const size_t rows_left_in_current_fragment, const size_t num_rows_left, const size_t num_rows_inserted, const std::unordered_map< int, size_t > &var_len_col_info, const size_t max_chunk_size, const InsertChunks &insert_chunks, std::map< int, Chunk_NS::Chunk > &column_map, const std::vector< size_t > &valid_row_indices)

Function Documentation

int Fragmenter_Namespace::anonymous_namespace{InsertOrderFragmenter.cpp}::compute_device_for_fragment	(	const int	table_id,
		const int	fragment_id,
		const int	num_devices
	)

Offset the fragment ID by the table ID, meaning single fragment tables end up balanced across multiple GPUs instead of all falling to GPU 0.

Definition at line 103 of file InsertOrderFragmenter.cpp.

References g_use_table_device_offset.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::createNewFragment(), and Fragmenter_Namespace::InsertOrderFragmenter::getChunkMetadata().

                                                       {
  if (g_use_table_device_offset) {
    return (table_id + fragment_id) % num_devices;
  } else {
    return fragment_id % num_devices;
  }
}

Here is the caller graph for this function:

size_t Fragmenter_Namespace::anonymous_namespace{InsertOrderFragmenter.cpp}::get_num_rows_to_insert	(	const size_t	rows_left_in_current_fragment,
		const size_t	num_rows_left,
		const size_t	num_rows_inserted,
		const std::unordered_map< int, size_t > &	var_len_col_info,
		const size_t	max_chunk_size,
		const InsertChunks &	insert_chunks,
		std::map< int, Chunk_NS::Chunk > &	column_map,
		const std::vector< size_t > &	valid_row_indices
	)

Definition at line 113 of file InsertOrderFragmenter.cpp.

References CHECK, CHECK_LE, and Fragmenter_Namespace::InsertChunks::chunks.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::insertChunksImpl().

                                                                          {
  size_t num_rows_to_insert = min(rows_left_in_current_fragment, num_rows_left);
  if (rows_left_in_current_fragment != 0) {
    for (const auto& var_len_col_info_it : var_len_col_info) {
      CHECK_LE(var_len_col_info_it.second, max_chunk_size);
      size_t bytes_left = max_chunk_size - var_len_col_info_it.second;
      auto find_it = insert_chunks.chunks.find(var_len_col_info_it.first);
      if (find_it == insert_chunks.chunks.end()) {
        continue;
      }
      const auto& chunk = find_it->second;
      auto column_type = chunk->getColumnDesc()->columnType;
      const int8_t* index_buffer_ptr =
          column_type.is_varlen_indeed() ? chunk->getIndexBuf()->getMemoryPtr() : nullptr;
      CHECK(column_type.is_varlen());

      auto col_map_it = column_map.find(var_len_col_info_it.first);
      num_rows_to_insert =
          std::min(num_rows_to_insert,
                   col_map_it->second.getNumElemsForBytesEncodedDataAtIndices(
                       index_buffer_ptr, valid_row_indices, bytes_left));
    }
  }
  return num_rows_to_insert;
}

Here is the caller graph for this function: