anonymous_namespace{QueryMemoryInitializer.cpp} Namespace Reference

Functions
void	check_total_bitmap_memory (const QueryMemoryDescriptor &query_mem_desc)
std::pair< int64_t *, bool >	alloc_group_by_buffer (const size_t numBytes, RenderAllocatorMap *render_allocator_map, const size_t thread_idx, RowSetMemoryOwner *mem_owner, const bool reuse_existing_buffer_for_thread)
int64_t	get_consistent_frag_size (const std::vector< uint64_t > &frag_offsets)
std::vector< int64_t >	get_consistent_frags_sizes (const std::vector< std::vector< uint64_t >> &frag_offsets)
std::vector< int64_t >	get_consistent_frags_sizes (const std::vector< Analyzer::Expr * > &target_exprs, const std::vector< int64_t > &table_frag_sizes)
std::vector< std::vector < int64_t > >	get_col_frag_offsets (const std::vector< Analyzer::Expr * > &target_exprs, const std::vector< std::vector< uint64_t >> &table_frag_offsets)
int	get_input_idx (RelAlgExecutionUnit const &ra_exe_unit, const shared::TableKey &outer_table_key)
void	check_count_distinct_expr_metadata (const QueryMemoryDescriptor &query_mem_desc, const RelAlgExecutionUnit &ra_exe_unit)
QueryMemoryInitializer::TargetAggOpsMetadata	collect_target_expr_metadata (const QueryMemoryDescriptor &query_mem_desc, const RelAlgExecutionUnit &ra_exe_unit)
template<typename T >
int8_t *	initColumnarBuffer (T *buffer_ptr, const T init_val, const uint32_t entry_count)
void	eachAggregateTargetIdxOfType (std::vector< Analyzer::Expr * > const &target_exprs, SQLAgg const agg_type, std::function< void(Analyzer::AggExpr const *, size_t)> lambda)
void	compact_projection_buffer_for_cpu_columnar (const QueryMemoryDescriptor &query_mem_desc, int8_t *projection_buffer, const size_t projection_count)

Function Documentation

std::pair<int64_t*, bool> anonymous_namespace{QueryMemoryInitializer.cpp}::alloc_group_by_buffer	(	const size_t	numBytes,
		RenderAllocatorMap *	render_allocator_map,
		const size_t	thread_idx,
		RowSetMemoryOwner *	mem_owner,
		const bool	reuse_existing_buffer_for_thread
	)

Definition at line 59 of file QueryMemoryInitializer.cpp.

References RowSetMemoryOwner::allocate(), RowSetMemoryOwner::allocateCachedGroupByBuffer(), and RenderAllocatorMap::getRenderAllocator().

Referenced by QueryMemoryInitializer::QueryMemoryInitializer(), and thread_idx_().

                                                 {
  if (render_allocator_map) {
    // NOTE(adb): If we got here, we are performing an in-situ rendering query and are not
    // using CUDA buffers. Therefore we need to allocate result set storage using CPU
    // memory.
    const auto gpu_idx = 0;  // Only 1 GPU supported in CUDA-disabled rendering mode
    auto render_allocator_ptr = render_allocator_map->getRenderAllocator(gpu_idx);
    return std::make_pair(
        reinterpret_cast<int64_t*>(render_allocator_ptr->alloc(numBytes)), false);
  } else if (reuse_existing_buffer_for_thread) {
    return mem_owner->allocateCachedGroupByBuffer(numBytes, thread_idx);
  }
  return std::make_pair(
      reinterpret_cast<int64_t*>(mem_owner->allocate(numBytes, thread_idx)), false);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void anonymous_namespace{QueryMemoryInitializer.cpp}::check_count_distinct_expr_metadata	(	const QueryMemoryDescriptor &	query_mem_desc,
		const RelAlgExecutionUnit &	ra_exe_unit
	)

Definition at line 167 of file QueryMemoryInitializer.cpp.

References CHECK, CHECK_EQ, CHECK_GE, CHECK_LT, g_bigint_count, get_target_info(), QueryMemoryDescriptor::getCountDistinctDescriptor(), QueryMemoryDescriptor::getLogicalSlotWidthBytes(), QueryMemoryDescriptor::getSlotCount(), QueryMemoryDescriptor::getSlotIndexForSingleSlotCol(), Invalid, is_distinct_target(), kAPPROX_COUNT_DISTINCT, kCOUNT, kCOUNT_IF, and RelAlgExecutionUnit::target_exprs.

Referenced by QueryMemoryInitializer::QueryMemoryInitializer().

                                                                                {
  const size_t agg_col_count{query_mem_desc.getSlotCount()};
  CHECK_GE(agg_col_count, ra_exe_unit.target_exprs.size());
  for (size_t target_idx = 0; target_idx < ra_exe_unit.target_exprs.size();
       ++target_idx) {
    const auto target_expr = ra_exe_unit.target_exprs[target_idx];
    const auto agg_info = get_target_info(target_expr, g_bigint_count);
    if (is_distinct_target(agg_info)) {
      CHECK(agg_info.is_agg &&
            (agg_info.agg_kind == kCOUNT || agg_info.agg_kind == kCOUNT_IF ||
             agg_info.agg_kind == kAPPROX_COUNT_DISTINCT));
      CHECK(!agg_info.sql_type.is_varlen());
      const size_t agg_col_idx = query_mem_desc.getSlotIndexForSingleSlotCol(target_idx);
      CHECK_LT(static_cast<size_t>(agg_col_idx), agg_col_count);
      CHECK_EQ(static_cast<size_t>(query_mem_desc.getLogicalSlotWidthBytes(agg_col_idx)),
               sizeof(int64_t));
      const auto& count_distinct_desc =
          query_mem_desc.getCountDistinctDescriptor(target_idx);
      CHECK(count_distinct_desc.impl_type_ != CountDistinctImplType::Invalid);
    }
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void anonymous_namespace{QueryMemoryInitializer.cpp}::check_total_bitmap_memory ( const QueryMemoryDescriptor &  query_mem_desc )

inline

Definition at line 32 of file QueryMemoryInitializer.cpp.

References Bitmap, CountDistinctDescriptor::bitmapPaddedSizeBytes(), g_bitmap_memory_limit, QueryMemoryDescriptor::getCountDistinctDescriptor(), QueryMemoryDescriptor::getCountDistinctDescriptorsSize(), and QueryMemoryDescriptor::getEntryCount().

Referenced by QueryMemoryInitializer::QueryMemoryInitializer().

                                                                                   {
  const size_t groups_buffer_entry_count = query_mem_desc.getEntryCount();
  checked_int64_t total_bytes_per_group = 0;
  const size_t num_count_distinct_descs =
      query_mem_desc.getCountDistinctDescriptorsSize();
  for (size_t i = 0; i < num_count_distinct_descs; i++) {
    const auto count_distinct_desc = query_mem_desc.getCountDistinctDescriptor(i);
    if (count_distinct_desc.impl_type_ != CountDistinctImplType::Bitmap) {
      continue;
    }
    total_bytes_per_group += count_distinct_desc.bitmapPaddedSizeBytes();
  }
  int64_t total_bytes{0};
  // Using OutOfHostMemory until we can verify that SlabTooBig would also be properly
  // caught
  try {
    total_bytes = static_cast<int64_t>(total_bytes_per_group * groups_buffer_entry_count);
  } catch (...) {
    // Absurd amount of memory, merely computing the number of bits overflows int64_t.
    // Don't bother to report the real amount, this is unlikely to ever happen.
    throw OutOfHostMemory(std::numeric_limits<int64_t>::max() / 8);
  }
  if (total_bytes >= g_bitmap_memory_limit) {
    throw OutOfHostMemory(total_bytes);
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

QueryMemoryInitializer::TargetAggOpsMetadata anonymous_namespace{QueryMemoryInitializer.cpp}::collect_target_expr_metadata	(	const QueryMemoryDescriptor &	query_mem_desc,
		const RelAlgExecutionUnit &	ra_exe_unit
	)

Definition at line 191 of file QueryMemoryInitializer.cpp.

References QueryMemoryDescriptor::countDistinctDescriptorsLogicallyEmpty(), QueryMemoryInitializer::TargetAggOpsMetadata::has_count_distinct, QueryMemoryInitializer::TargetAggOpsMetadata::has_mode, QueryMemoryInitializer::TargetAggOpsMetadata::has_tdigest, kAPPROX_QUANTILE, kMODE, and RelAlgExecutionUnit::target_exprs.

Referenced by QueryMemoryInitializer::QueryMemoryInitializer().

                                            {
  QueryMemoryInitializer::TargetAggOpsMetadata agg_op_metadata;
  if (!query_mem_desc.countDistinctDescriptorsLogicallyEmpty()) {
    agg_op_metadata.has_count_distinct = true;
  }
  std::for_each(
      ra_exe_unit.target_exprs.begin(),
      ra_exe_unit.target_exprs.end(),
      [&agg_op_metadata](const Analyzer::Expr* expr) {
        if (auto const* agg_expr = dynamic_cast<Analyzer::AggExpr const*>(expr)) {
          if (agg_expr->get_aggtype() == kMODE) {
            agg_op_metadata.has_mode = true;
          } else if (agg_expr->get_aggtype() == kAPPROX_QUANTILE) {
            agg_op_metadata.has_tdigest = true;
          }
        }
      });
  return agg_op_metadata;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void anonymous_namespace{QueryMemoryInitializer.cpp}::compact_projection_buffer_for_cpu_columnar	(	const QueryMemoryDescriptor &	query_mem_desc,
		int8_t *	projection_buffer,
		const size_t	projection_count
	)

Definition at line 1262 of file QueryMemoryInitializer.cpp.

References align_to_int64(), CHECK, QueryMemoryDescriptor::getColOffInBytes(), QueryMemoryDescriptor::getEntryCount(), QueryMemoryDescriptor::getPaddedSlotWidthBytes(), and QueryMemoryDescriptor::getSlotCount().

Referenced by QueryMemoryInitializer::compactProjectionBuffersCpu().

                                   {
  // the first column (row indices) remains unchanged.
  CHECK(projection_count <= query_mem_desc.getEntryCount());
  constexpr size_t row_index_width = sizeof(int64_t);
  size_t buffer_offset1{projection_count * row_index_width};
  // other columns are actual non-lazy columns for the projection:
  for (size_t i = 0; i < query_mem_desc.getSlotCount(); i++) {
    if (query_mem_desc.getPaddedSlotWidthBytes(i) > 0) {
      auto column_proj_size =
          projection_count * query_mem_desc.getPaddedSlotWidthBytes(i);
      auto buffer_offset2 = query_mem_desc.getColOffInBytes(i);
      if (buffer_offset1 + column_proj_size >= buffer_offset2) {
        // overlapping
        std::memmove(projection_buffer + buffer_offset1,
                     projection_buffer + buffer_offset2,
                     column_proj_size);
      } else {
        std::memcpy(projection_buffer + buffer_offset1,
                    projection_buffer + buffer_offset2,
                    column_proj_size);
      }
      buffer_offset1 += align_to_int64(column_proj_size);
    }
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void anonymous_namespace{QueryMemoryInitializer.cpp}::eachAggregateTargetIdxOfType	(	std::vector< Analyzer::Expr * > const &	target_exprs,
		SQLAgg const	agg_type,
		std::function< void(Analyzer::AggExpr const *, size_t)>	lambda
	)

Definition at line 904 of file QueryMemoryInitializer.cpp.

Referenced by QueryMemoryInitializer::allocateModeBuffer(), QueryMemoryInitializer::allocateTDigestsBuffer(), QueryMemoryInitializer::initializeModeIndexSet(), and QueryMemoryInitializer::initializeQuantileParams().

                                                              {
  for (size_t target_idx = 0; target_idx < target_exprs.size(); ++target_idx) {
    auto const target_expr = target_exprs[target_idx];
    if (auto const* agg_expr = dynamic_cast<Analyzer::AggExpr const*>(target_expr)) {
      if (agg_expr->get_aggtype() == agg_type) {
        lambda(agg_expr, target_idx);
      }
    }
  }
}

Here is the caller graph for this function:

std::vector<std::vector<int64_t> > anonymous_namespace{QueryMemoryInitializer.cpp}::get_col_frag_offsets ( const std::vector< Analyzer::Expr * > &  target_exprs, const std::vector< std::vector< uint64_t >> &  table_frag_offsets  )

inline

Definition at line 130 of file QueryMemoryInitializer.cpp.

References CHECK_EQ, and CHECK_LT.

Referenced by QueryMemoryInitializer::QueryMemoryInitializer(), and thread_idx_().

                                                              {
  std::vector<std::vector<int64_t>> col_frag_offsets;
  for (auto& table_offsets : table_frag_offsets) {
    std::vector<int64_t> col_offsets;
    for (auto expr : target_exprs) {
      if (const auto col_var = dynamic_cast<Analyzer::ColumnVar*>(expr)) {
        if (col_var->get_rte_idx() < 0) {
          CHECK_EQ(-1, col_var->get_rte_idx());
          col_offsets.push_back(int64_t(-1));
        } else {
          CHECK_LT(static_cast<size_t>(col_var->get_rte_idx()), table_offsets.size());
          col_offsets.push_back(
              static_cast<int64_t>(table_offsets[col_var->get_rte_idx()]));
        }
      } else {
        col_offsets.push_back(int64_t(-1));
      }
    }
    col_frag_offsets.push_back(col_offsets);
  }
  return col_frag_offsets;
}

Here is the caller graph for this function:

int64_t anonymous_namespace{QueryMemoryInitializer.cpp}::get_consistent_frag_size ( const std::vector< uint64_t > &  frag_offsets )

inline

Definition at line 80 of file QueryMemoryInitializer.cpp.

Referenced by get_consistent_frags_sizes().

                                                                                 {
  if (frag_offsets.size() < 2) {
    return int64_t(-1);
  }
  const auto frag_size = frag_offsets[1] - frag_offsets[0];
  for (size_t i = 2; i < frag_offsets.size(); ++i) {
    const auto curr_size = frag_offsets[i] - frag_offsets[i - 1];
    if (curr_size != frag_size) {
      return int64_t(-1);
    }
  }
  return !frag_size ? std::numeric_limits<int64_t>::max()
                    : static_cast<int64_t>(frag_size);
}

Here is the caller graph for this function:

std::vector<int64_t> anonymous_namespace{QueryMemoryInitializer.cpp}::get_consistent_frags_sizes ( const std::vector< std::vector< uint64_t >> &  frag_offsets )

inline

Definition at line 95 of file QueryMemoryInitializer.cpp.

References get_consistent_frag_size().

Referenced by QueryMemoryInitializer::QueryMemoryInitializer(), and thread_idx_().

                                                        {
  if (frag_offsets.empty()) {
    return {};
  }
  std::vector<int64_t> frag_sizes;
  for (size_t tab_idx = 0; tab_idx < frag_offsets[0].size(); ++tab_idx) {
    std::vector<uint64_t> tab_offs;
    for (auto& offsets : frag_offsets) {
      tab_offs.push_back(offsets[tab_idx]);
    }
    frag_sizes.push_back(get_consistent_frag_size(tab_offs));
  }
  return frag_sizes;
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::vector<int64_t> anonymous_namespace{QueryMemoryInitializer.cpp}::get_consistent_frags_sizes ( const std::vector< Analyzer::Expr * > &  target_exprs, const std::vector< int64_t > &  table_frag_sizes  )

inline

Definition at line 111 of file QueryMemoryInitializer.cpp.

References CHECK_EQ.

                                                {
  std::vector<int64_t> col_frag_sizes;
  for (auto expr : target_exprs) {
    if (const auto col_var = dynamic_cast<Analyzer::ColumnVar*>(expr)) {
      if (col_var->get_rte_idx() < 0) {
        CHECK_EQ(-1, col_var->get_rte_idx());
        col_frag_sizes.push_back(int64_t(-1));
      } else {
        col_frag_sizes.push_back(table_frag_sizes[col_var->get_rte_idx()]);
      }
    } else {
      col_frag_sizes.push_back(int64_t(-1));
    }
  }
  return col_frag_sizes;
}

int anonymous_namespace{QueryMemoryInitializer.cpp}::get_input_idx	(	RelAlgExecutionUnit const &	ra_exe_unit,
		const shared::TableKey &	outer_table_key
	)

Definition at line 157 of file QueryMemoryInitializer.cpp.

References RelAlgExecutionUnit::input_descs.

Referenced by QueryPlanDagExtractor::handleLeftDeepJoinTree(), and QueryMemoryInitializer::QueryMemoryInitializer().

                                                         {
  auto match_table_key = [=](auto& desc) {
    return outer_table_key == desc.getTableKey();
  };
  auto& input_descs = ra_exe_unit.input_descs;
  auto itr = std::find_if(input_descs.begin(), input_descs.end(), match_table_key);
  return itr == input_descs.end() ? 0 : itr->getNestLevel();
}

Here is the caller graph for this function:

template<typename T >

int8_t* anonymous_namespace{QueryMemoryInitializer.cpp}::initColumnarBuffer	(	T *	buffer_ptr,
		const T	init_val,
		const uint32_t	entry_count
	)

Definition at line 667 of file QueryMemoryInitializer.cpp.

References heavydb.dtypes::T.

                                                                                        {
  static_assert(sizeof(T) <= sizeof(int64_t), "Unsupported template type");
  for (uint32_t i = 0; i < entry_count; ++i) {
    buffer_ptr[i] = init_val;
  }
  return reinterpret_cast<int8_t*>(buffer_ptr + entry_count);
}