StringDictionaryProxy Class Reference

#include <StringDictionaryProxy.h>

Classes
struct	HeterogeneousStringEqual
struct	HeterogeneousStringHash
class	IdMap

Public Types
using	TransientMap = robin_hood::unordered_node_map< std::string, int32_t, HeterogeneousStringHash, HeterogeneousStringEqual >

Public Member Functions
	StringDictionaryProxy (StringDictionaryProxy const &)=delete
StringDictionaryProxy const &	operator= (StringDictionaryProxy const &)=delete
	StringDictionaryProxy (std::shared_ptr< StringDictionary > sd, const int32_t string_dict_id, const int64_t generation)
int32_t	getDictId () const noexcept
bool	operator== (StringDictionaryProxy const &) const
bool	operator!= (StringDictionaryProxy const &) const
int32_t	getOrAdd (const std::string &str) noexcept
StringDictionary *	getDictionary () const noexcept
int64_t	getGeneration () const noexcept
std::vector< int32_t >	getTransientBulk (const std::vector< std::string > &strings) const
	Executes read-only lookup of a vector of strings and returns a vector of their integer ids. More...
int32_t	getOrAddTransient (const std::string &str)
std::vector< int32_t >	getOrAddTransientBulk (const std::vector< std::string > &strings)
int32_t	getIdOfString (const std::string &str) const
int32_t	getIdOfStringNoGeneration (const std::string &str) const
std::string	getString (int32_t string_id) const
std::vector< std::string >	getStrings (const std::vector< int32_t > &string_ids) const
std::pair< const char *, size_t >	getStringBytes (int32_t string_id) const noexcept
IdMap	initIdMap () const
IdMap	buildIntersectionTranslationMapToOtherProxy (const StringDictionaryProxy *dest_proxy, const std::vector< StringOps_Namespace::StringOpInfo > &string_op_infos) const
	Builds a vectorized string_id translation map from this proxy to dest_proxy. More...
IdMap	buildUnionTranslationMapToOtherProxy (StringDictionaryProxy *dest_proxy, const std::vector< StringOps_Namespace::StringOpInfo > &string_op_types) const
size_t	storageEntryCount () const
	Returns the number of string entries in the underlying string dictionary, at this proxy's generation_ if it is set/valid, otherwise just the current size of the dictionary. More...
size_t	transientEntryCount () const
	Returns the number of transient string entries for this proxy,. More...
size_t	entryCount () const
	Returns the number of total string entries for this proxy, both stored in the underlying dictionary and in the transient map. Equal to storageEntryCount() + transientEntryCount() More...
void	updateGeneration (const int64_t generation) noexcept
std::vector< int32_t >	getLike (const std::string &pattern, const bool icase, const bool is_simple, const char escape) const
std::vector< int32_t >	getCompare (const std::string &pattern, const std::string &comp_operator) const
std::vector< int32_t >	getRegexpLike (const std::string &pattern, const char escape) const
const std::vector< std::string const * > &	getTransientVector () const
void	eachStringSerially (StringDictionary::StringCallback &) const
IdMap	transientUnion (StringDictionaryProxy const &)

Static Public Member Functions
static unsigned	transientIdToIndex (int32_t const id)
static int32_t	transientIndexToId (unsigned const index)

Private Member Functions
std::string	getStringUnlocked (const int32_t string_id) const
size_t	transientEntryCountUnlocked () const
size_t	entryCountUnlocked () const
size_t	persistedC () const
template<typename String >
int32_t	lookupTransientStringUnlocked (const String &lookup_string) const
size_t	getTransientBulkImpl (const std::vector< std::string > &strings, int32_t *string_ids, const bool take_read_lock) const
template<typename String >
size_t	transientLookupBulk (const std::vector< String > &lookup_strings, int32_t *string_ids, const bool take_read_lock) const
template<typename String >
size_t	transientLookupBulkUnlocked (const std::vector< String > &lookup_strings, int32_t *string_ids) const
template<typename String >
size_t	transientLookupBulkParallelUnlocked (const std::vector< String > &lookup_strings, int32_t *string_ids) const
IdMap	buildIntersectionTranslationMapToOtherProxyUnlocked (const StringDictionaryProxy *dest_proxy, const std::vector< StringOps_Namespace::StringOpInfo > &string_op_infos) const
template<typename String >
int32_t	getIdOfStringFromClient (String const &) const
template<typename String >
int32_t	getOrAddTransientUnlocked (String const &)

Private Attributes
std::shared_ptr< StringDictionary >	string_dict_
const int32_t	string_dict_id_
TransientMap	transient_str_to_int_
std::vector< std::string const * >	transient_string_vec_
int64_t	generation_
std::shared_mutex	rw_mutex_

Friends
class	StringLocalCallback
class	StringNetworkCallback

Detailed Description

Definition at line 38 of file StringDictionaryProxy.h.

Member Typedef Documentation

using StringDictionaryProxy::TransientMap = robin_hood::unordered_node_map<std::string, int32_t, HeterogeneousStringHash, HeterogeneousStringEqual>

Definition at line 226 of file StringDictionaryProxy.h.

Constructor & Destructor Documentation

StringDictionaryProxy::StringDictionaryProxy ( StringDictionaryProxy const &  )

delete

StringDictionaryProxy::StringDictionaryProxy	(	std::shared_ptr< StringDictionary >	sd,
		const int32_t	string_dict_id,
		const int64_t	generation
	)

Definition at line 39 of file StringDictionaryProxy.cpp.

    : string_dict_(sd), string_dict_id_(string_dict_id), generation_(generation) {}

Member Function Documentation

StringDictionaryProxy::IdMap StringDictionaryProxy::buildIntersectionTranslationMapToOtherProxy	(	const StringDictionaryProxy *	dest_proxy,
		const std::vector< StringOps_Namespace::StringOpInfo > &	string_op_infos
	)		const

Builds a vectorized string_id translation map from this proxy to dest_proxy.

Parameters

dest_proxy

StringDictionaryProxy that we are to map this proxy's string ids to

Returns

An IdMap which encapsulates a std::vector<int32_t> of string ids for both transient and non-transient strings, mapping to their translated string_ids. offset_ is defined to be the number of transient entries + 1. The ordering of values in the vector_map_ is:

• the transient ids (there are offset_-1 of these)
• INVALID_STR_ID (=-1)
• the non-transient string ids For example if there are 3 transient entries in this proxy and 20 in the underlying string dictionary, then vector_map_ will be of size() == 24 and offset_=3+1. The formula to translate ids is new_id = vector_map_[offset_ + old_id]. It is always the case that vector_map_[offset_-1]==-1 so that INVALID_STR_ID maps to INVALID_STR_ID.

Definition at line 302 of file StringDictionaryProxy.cpp.

References buildIntersectionTranslationMapToOtherProxyUnlocked(), getDictId(), order_translation_locks(), and rw_mutex_.

Referenced by RowSetMemoryOwner::addStringProxyIntersectionTranslationMap().

                                                                             {
  const auto source_dict_id = getDictId();
  const auto dest_dict_id = dest_proxy->getDictId();

  std::shared_lock<std::shared_mutex> source_proxy_read_lock(rw_mutex_, std::defer_lock);
  std::unique_lock<std::shared_mutex> dest_proxy_write_lock(dest_proxy->rw_mutex_,
                                                            std::defer_lock);
  order_translation_locks(
      source_dict_id, dest_dict_id, source_proxy_read_lock, dest_proxy_write_lock);
  return buildIntersectionTranslationMapToOtherProxyUnlocked(dest_proxy, string_op_infos);
}

Here is the call graph for this function:

Here is the caller graph for this function:

StringDictionaryProxy::IdMap StringDictionaryProxy::buildIntersectionTranslationMapToOtherProxyUnlocked ( const StringDictionaryProxy *  dest_proxy, const std::vector< StringOps_Namespace::StringOpInfo > &  string_op_infos  ) const

private

Definition at line 183 of file StringDictionaryProxy.cpp.

References CHECK_GT, CHECK_LE, StringDictionaryProxy::IdMap::data(), DEBUG_TIMER, StringDictionaryProxy::IdMap::empty(), entryCountUnlocked(), generation_, getTransientBulkImpl(), StringDictionaryProxy::IdMap::getVectorMap(), initIdMap(), StringDictionary::INVALID_STR_ID, lookupTransientStringUnlocked(), StringDictionaryProxy::IdMap::numTransients(), StringDictionaryProxy::IdMap::setNumUntranslatedStrings(), StringDictionaryProxy::IdMap::setRangeEnd(), StringDictionaryProxy::IdMap::setRangeStart(), StringDictionaryProxy::IdMap::storageData(), storageEntryCount(), string_dict_, shared::transform(), transient_string_vec_, transientEntryCountUnlocked(), and VLOG.

Referenced by buildIntersectionTranslationMapToOtherProxy(), and buildUnionTranslationMapToOtherProxy().

                                                                             {
  auto timer = DEBUG_TIMER(__func__);
  IdMap id_map = initIdMap();

  if (id_map.empty()) {
    return id_map;
  }

  const StringOps_Namespace::StringOps string_ops(string_op_infos);

  // First map transient strings, store at front of vector map
  const size_t num_transient_entries = id_map.numTransients();
  size_t num_transient_strings_not_translated = 0UL;
  if (num_transient_entries) {
    std::vector<std::string> transient_lookup_strings(num_transient_entries);
    if (string_ops.size()) {
      std::transform(transient_string_vec_.cbegin(),
                     transient_string_vec_.cend(),
                     transient_lookup_strings.rbegin(),
                     [&](std::string const* ptr) { return string_ops(*ptr); });
    } else {
      std::transform(transient_string_vec_.cbegin(),
                     transient_string_vec_.cend(),
                     transient_lookup_strings.rbegin(),
                     [](std::string const* ptr) { return *ptr; });
    }

    // This lookup may have a different snapshot of
    // dest_proxy transients and dictionary than what happens under
    // the below dest_proxy_read_lock. We may need an unlocked version of
    // getTransientBulk to ensure consistency (I don't believe
    // current behavior would cause crashes/races, verify this though)

    // Todo(mattp): Consider variant of getTransientBulkImp that can take
    // a vector of pointer-to-strings so we don't have to materialize
    // transient_string_vec_ into transient_lookup_strings.

    num_transient_strings_not_translated =
        dest_proxy->getTransientBulkImpl(transient_lookup_strings, id_map.data(), false);
  }

  // Now map strings in dictionary
  // We place non-transient strings after the transient strings
  // if they exist, otherwise at index 0
  int32_t* translation_map_stored_entries_ptr = id_map.storageData();

  auto dest_transient_lookup_callback = [dest_proxy, translation_map_stored_entries_ptr](
                                            const std::string_view& source_string,
                                            const int32_t source_string_id) {
    translation_map_stored_entries_ptr[source_string_id] =
        dest_proxy->lookupTransientStringUnlocked(source_string);
    return translation_map_stored_entries_ptr[source_string_id] ==
           StringDictionary::INVALID_STR_ID;
  };

  const size_t num_dest_transients = dest_proxy->transientEntryCountUnlocked();
  const size_t num_persisted_strings_not_translated =
      generation_ > 0 ? string_dict_->buildDictionaryTranslationMap(
                            dest_proxy->string_dict_.get(),
                            translation_map_stored_entries_ptr,
                            generation_,
                            dest_proxy->generation_,
                            num_dest_transients > 0UL,
                            dest_transient_lookup_callback,
                            string_op_infos)
                      : 0UL;

  const size_t num_dest_entries = dest_proxy->entryCountUnlocked();
  const size_t num_total_entries =
      id_map.getVectorMap().size() - 1UL /* account for skipped entry -1 */;
  CHECK_GT(num_total_entries, 0UL);
  const size_t num_strings_not_translated =
      num_transient_strings_not_translated + num_persisted_strings_not_translated;
  CHECK_LE(num_strings_not_translated, num_total_entries);
  id_map.setNumUntranslatedStrings(num_strings_not_translated);

  // Below is a conservative setting of range based on the size of the destination proxy,
  // but probably not worth a scan over the data (or inline computation as we translate)
  // to compute the actual ranges

  id_map.setRangeStart(
      num_dest_transients > 0 ? -1 - static_cast<int32_t>(num_dest_transients) : 0);
  id_map.setRangeEnd(dest_proxy->storageEntryCount());

  const size_t num_entries_translated = num_total_entries - num_strings_not_translated;
  const float match_pct =
      100.0 * static_cast<float>(num_entries_translated) / num_total_entries;
  VLOG(1) << std::fixed << std::setprecision(2) << match_pct << "% ("
          << num_entries_translated << " entries) from dictionary ("
          << string_dict_->getDbId() << ", " << string_dict_->getDictId() << ") with "
          << num_total_entries << " total entries ( " << num_transient_entries
          << " literals)"
          << " translated to dictionary (" << dest_proxy->string_dict_->getDbId() << ", "
          << dest_proxy->string_dict_->getDictId() << ") with " << num_dest_entries
          << " total entries (" << dest_proxy->transientEntryCountUnlocked()
          << " literals).";

  return id_map;
}

Here is the call graph for this function:

Here is the caller graph for this function:

StringDictionaryProxy::IdMap StringDictionaryProxy::buildUnionTranslationMapToOtherProxy	(	StringDictionaryProxy *	dest_proxy,
		const std::vector< StringOps_Namespace::StringOpInfo > &	string_op_types
	)		const

Definition at line 316 of file StringDictionaryProxy.cpp.

References buildIntersectionTranslationMapToOtherProxyUnlocked(), DEBUG_TIMER, getDictId(), getOrAddTransientUnlocked(), getStringUnlocked(), StringDictionary::INVALID_STR_ID, order_translation_locks(), rw_mutex_, string_dict_, to_string(), and transientEntryCountUnlocked().

Referenced by RowSetMemoryOwner::addStringProxyUnionTranslationMap().

                                                                             {
  auto timer = DEBUG_TIMER(__func__);

  const auto source_dict_id = getDictId();
  const auto dest_dict_id = dest_proxy->getDictId();
  std::shared_lock<std::shared_mutex> source_proxy_read_lock(rw_mutex_, std::defer_lock);
  std::unique_lock<std::shared_mutex> dest_proxy_write_lock(dest_proxy->rw_mutex_,
                                                            std::defer_lock);
  order_translation_locks(
      source_dict_id, dest_dict_id, source_proxy_read_lock, dest_proxy_write_lock);

  auto id_map =
      buildIntersectionTranslationMapToOtherProxyUnlocked(dest_proxy, string_op_infos);
  if (id_map.empty()) {
    return id_map;
  }
  const auto num_untranslated_strings = id_map.numUntranslatedStrings();
  if (num_untranslated_strings > 0) {
    const size_t total_post_translation_dest_transients =
        num_untranslated_strings + dest_proxy->transientEntryCountUnlocked();
    constexpr size_t max_allowed_transients =
        static_cast<size_t>(std::numeric_limits<int32_t>::max() -
                            2); /* -2 accounts for INVALID_STR_ID and NULL value */
    if (total_post_translation_dest_transients > max_allowed_transients) {
      throw std::runtime_error("Union translation to dictionary" +
                               std::to_string(getDictId()) + " would result in " +
                               std::to_string(total_post_translation_dest_transients) +
                               " transient entries, which is more than limit of " +
                               std::to_string(max_allowed_transients) + " transients.");
    }
    const int32_t map_domain_start = id_map.domainStart();
    const int32_t map_domain_end = id_map.domainEnd();

    const StringOps_Namespace::StringOps string_ops(string_op_infos);
    const bool has_string_ops = string_ops.size();

    // First iterate over transient strings and add to dest map
    // Todo (todd): Add call to fetch string_views (local) or strings (distributed)
    // for all non-translated ids to avoid string-by-string fetch

    for (int32_t source_string_id = map_domain_start; source_string_id < -1;
         ++source_string_id) {
      if (id_map[source_string_id] == StringDictionary::INVALID_STR_ID) {
        const auto source_string = getStringUnlocked(source_string_id);
        const auto dest_string_id = dest_proxy->getOrAddTransientUnlocked(
            has_string_ops ? string_ops(source_string) : source_string);
        id_map[source_string_id] = dest_string_id;
      }
    }
    // Now iterate over stored strings
    for (int32_t source_string_id = 0; source_string_id < map_domain_end;
         ++source_string_id) {
      if (id_map[source_string_id] == StringDictionary::INVALID_STR_ID) {
        const auto source_string = string_dict_->getString(source_string_id);
        const auto dest_string_id = dest_proxy->getOrAddTransientUnlocked(
            has_string_ops ? string_ops(source_string) : source_string);
        id_map[source_string_id] = dest_string_id;
      }
    }
  }
  // We may have added transients to the destination proxy, use this to update
  // our id map range (used downstream for ExpressionRange)

  const size_t num_dest_transients = dest_proxy->transientEntryCountUnlocked();
  id_map.setRangeStart(
      num_dest_transients > 0 ? -1 - static_cast<int32_t>(num_dest_transients) : 0);
  return id_map;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void StringDictionaryProxy::eachStringSerially

(

StringDictionary::StringCallback &

serial_callback

)

const

Definition at line 532 of file StringDictionaryProxy.cpp.

References generation_, string_dict_, and transient_string_vec_.

Referenced by transientUnion().

                                                           {
  constexpr int32_t max_transient_id = -2;
  // Iterate over transient strings.
  for (unsigned index = 0; index < transient_string_vec_.size(); ++index) {
    std::string const& str = *transient_string_vec_[index];
    int32_t const string_id = max_transient_id - index;
    serial_callback(str, string_id);
  }
  // Iterate over non-transient strings.
  string_dict_->eachStringSerially(generation_, serial_callback);
}

Here is the caller graph for this function:

size_t StringDictionaryProxy::entryCount

(

)

const

Returns the number of total string entries for this proxy, both stored in the underlying dictionary and in the transient map. Equal to storageEntryCount() + transientEntryCount()

Returns

size_t Number of total string entries for this proxy

Definition at line 526 of file StringDictionaryProxy.cpp.

References entryCountUnlocked(), and rw_mutex_.

                                               {
  std::shared_lock<std::shared_mutex> read_lock(rw_mutex_);
  return entryCountUnlocked();
}

Here is the call graph for this function:

size_t StringDictionaryProxy::entryCountUnlocked ( ) const

private

Definition at line 522 of file StringDictionaryProxy.cpp.

References storageEntryCount(), and transientEntryCountUnlocked().

Referenced by buildIntersectionTranslationMapToOtherProxyUnlocked(), and entryCount().

                                                       {
  return storageEntryCount() + transientEntryCountUnlocked();
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::vector< int32_t > StringDictionaryProxy::getCompare	(	const std::string &	pattern,
		const std::string &	comp_operator
	)		const

Definition at line 451 of file StringDictionaryProxy.cpp.

References CHECK_GE, anonymous_namespace{StringDictionaryProxy.cpp}::do_compare(), generation_, run_benchmark_import::result, string_dict_, transient_string_vec_, and transientIndexToId().

Referenced by anonymous_namespace{StringOpsIR.cpp}::get_compared_ids().

                                          {
  CHECK_GE(generation_, 0);
  auto result = string_dict_->getCompare(pattern, comp_operator, generation_);
  for (unsigned index = 0; index < transient_string_vec_.size(); ++index) {
    if (do_compare(*transient_string_vec_[index], pattern, comp_operator)) {
      result.push_back(transientIndexToId(index));
    }
  }
  return result;
}

Here is the call graph for this function:

Here is the caller graph for this function:

int32_t StringDictionaryProxy::getDictId ( ) const

inlinenoexcept

Definition at line 46 of file StringDictionaryProxy.h.

References string_dict_id_.

Referenced by buildIntersectionTranslationMapToOtherProxy(), buildUnionTranslationMapToOtherProxy(), TransientStringLiteralsVisitor::visitStringOper(), and TransientStringLiteralsVisitor::visitUOper().

{ return string_dict_id_; };

Here is the caller graph for this function:

StringDictionary * StringDictionaryProxy::getDictionary ( ) const

noexcept

Definition at line 730 of file StringDictionaryProxy.cpp.

References string_dict_.

Referenced by RowSetMemoryOwner::addStringProxyIntersectionTranslationMap(), and RowSetMemoryOwner::addStringProxyUnionTranslationMap().

                                                                      {
  return string_dict_.get();
}

Here is the caller graph for this function:

int64_t StringDictionaryProxy::getGeneration ( ) const

noexcept

Definition at line 734 of file StringDictionaryProxy.cpp.

References generation_.

                                                            {
  return generation_;
}

int32_t StringDictionaryProxy::getIdOfString

(

const std::string &

str

)

const

Definition at line 110 of file StringDictionaryProxy.cpp.

References getIdOfStringFromClient(), StringDictionary::INVALID_STR_ID, rw_mutex_, and transient_str_to_int_.

Referenced by anonymous_namespace{RelAlgTranslator.cpp}::fill_dictionary_encoded_in_vals(), and Executor::serializeLiterals().

                                                                       {
  std::shared_lock<std::shared_mutex> read_lock(rw_mutex_);
  auto const str_id = getIdOfStringFromClient(str);
  if (str_id != StringDictionary::INVALID_STR_ID || transient_str_to_int_.empty()) {
    return str_id;
  }
  auto it = transient_str_to_int_.find(str);
  return it != transient_str_to_int_.end() ? it->second
                                           : StringDictionary::INVALID_STR_ID;
}

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename String >

int32_t StringDictionaryProxy::getIdOfStringFromClient ( String const &  str ) const

private

Definition at line 122 of file StringDictionaryProxy.cpp.

References CHECK_GE, generation_, string_dict_, and truncate_to_generation().

Referenced by getIdOfString(), and getOrAddTransient().

                                                                              {
  CHECK_GE(generation_, 0);
  return truncate_to_generation(string_dict_->getIdOfString(str), generation_);
}

Here is the call graph for this function:

Here is the caller graph for this function:

int32_t StringDictionaryProxy::getIdOfStringNoGeneration

(

const std::string &

str

)

const

Definition at line 127 of file StringDictionaryProxy.cpp.

References StringDictionary::INVALID_STR_ID, rw_mutex_, string_dict_, and transient_str_to_int_.

                                                                                   {
  std::shared_lock<std::shared_mutex> read_lock(rw_mutex_);
  auto str_id = string_dict_->getIdOfString(str);
  if (str_id != StringDictionary::INVALID_STR_ID || transient_str_to_int_.empty()) {
    return str_id;
  }
  auto it = transient_str_to_int_.find(str);
  return it != transient_str_to_int_.end() ? it->second
                                           : StringDictionary::INVALID_STR_ID;
}

std::vector< int32_t > StringDictionaryProxy::getLike	(	const std::string &	pattern,
		const bool	icase,
		const bool	is_simple,
		const char	escape
	)		const

Definition at line 413 of file StringDictionaryProxy.cpp.

References CHECK_GE, generation_, anonymous_namespace{StringDictionary.cpp}::is_like(), run_benchmark_import::result, string_dict_, transient_string_vec_, and transientIndexToId().

                                                                             {
  CHECK_GE(generation_, 0);
  auto result = string_dict_->getLike(pattern, icase, is_simple, escape, generation_);
  for (unsigned index = 0; index < transient_string_vec_.size(); ++index) {
    if (is_like(*transient_string_vec_[index], pattern, icase, is_simple, escape)) {
      result.push_back(transientIndexToId(index));
    }
  }
  return result;
}

Here is the call graph for this function:

int32_t StringDictionaryProxy::getOrAdd ( const std::string &  str )

noexcept

Definition at line 486 of file StringDictionaryProxy.cpp.

Referenced by DictionaryValueConverter< TARGET_TYPE >::convertTransientStringIdToPermanentId().

                                                                     {
  return string_dict_->getOrAdd(str);
}

Here is the caller graph for this function:

int32_t StringDictionaryProxy::getOrAddTransient

(

const std::string &

str

)

Definition at line 101 of file StringDictionaryProxy.cpp.

References getIdOfStringFromClient(), getOrAddTransientUnlocked(), StringDictionary::INVALID_STR_ID, and rw_mutex_.

Referenced by apply_string_ops_and_encode(), Executor::serializeLiterals(), string_compress(), TransientStringLiteralsVisitor::visitConstant(), and TransientStringLiteralsVisitor::visitStringOper().

                                                                     {
  auto const string_id = getIdOfStringFromClient(str);
  if (string_id != StringDictionary::INVALID_STR_ID) {
    return string_id;
  }
  std::lock_guard<std::shared_mutex> write_lock(rw_mutex_);
  return getOrAddTransientUnlocked(str);
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::vector< int32_t > StringDictionaryProxy::getOrAddTransientBulk

(

const std::vector< std::string > &

strings

)

Definition at line 60 of file StringDictionaryProxy.cpp.

References CHECK_GE, generation_, getOrAddTransientUnlocked(), StringDictionary::INVALID_STR_ID, rw_mutex_, and string_dict_.

Referenced by supported_ml_frameworks__cpu_().

                                         {
  CHECK_GE(generation_, 0);
  const size_t num_strings = strings.size();
  std::vector<int32_t> string_ids(num_strings);
  if (num_strings == 0) {
    return string_ids;
  }
  // Since new strings added to a StringDictionaryProxy are not materialized in the
  // proxy's underlying StringDictionary, we can use the fast parallel
  // StringDictionary::getBulk method to fetch ids from the underlying dictionary (which
  // will return StringDictionary::INVALID_STR_ID for strings that don't exist)

  // Don't need to be under lock here as the string ids for strings in the underlying
  // materialized dictionary are immutable
  const size_t num_strings_not_found =
      string_dict_->getBulk(strings, string_ids.data(), generation_);
  if (num_strings_not_found > 0) {
    std::lock_guard<std::shared_mutex> write_lock(rw_mutex_);
    for (size_t string_idx = 0; string_idx < num_strings; ++string_idx) {
      if (string_ids[string_idx] == StringDictionary::INVALID_STR_ID) {
        string_ids[string_idx] = getOrAddTransientUnlocked(strings[string_idx]);
      }
    }
  }
  return string_ids;
}

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename String >

int32_t StringDictionaryProxy::getOrAddTransientUnlocked ( String const &  str )

private

Definition at line 89 of file StringDictionaryProxy.cpp.

References transient_str_to_int_, transient_string_vec_, and transientIndexToId().

Referenced by buildUnionTranslationMapToOtherProxy(), getOrAddTransient(), getOrAddTransientBulk(), StringLocalCallback::operator()(), and StringNetworkCallback::operator()().

                                                                          {
  unsigned const new_index = transient_str_to_int_.size();
  auto transient_id = transientIndexToId(new_index);
  auto const emplaced = transient_str_to_int_.emplace(str, transient_id);
  if (emplaced.second) {  // (str, transient_id) was added to transient_str_to_int_.
    transient_string_vec_.push_back(&emplaced.first->first);
  } else {  // str already exists in transient_str_to_int_. Return existing transient_id.
    transient_id = emplaced.first->second;
  }
  return transient_id;
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::vector< int32_t > StringDictionaryProxy::getRegexpLike	(	const std::string &	pattern,
		const char	escape
	)		const

Definition at line 474 of file StringDictionaryProxy.cpp.

References CHECK_GE, generation_, anonymous_namespace{StringDictionary.cpp}::is_regexp_like(), run_benchmark_import::result, string_dict_, transient_string_vec_, and transientIndexToId().

                                                                                   {
  CHECK_GE(generation_, 0);
  auto result = string_dict_->getRegexpLike(pattern, escape, generation_);
  for (unsigned index = 0; index < transient_string_vec_.size(); ++index) {
    if (is_regexp_like(*transient_string_vec_[index], pattern, escape)) {
      result.push_back(transientIndexToId(index));
    }
  }
  return result;
}

Here is the call graph for this function:

std::string StringDictionaryProxy::getString

(

int32_t

string_id

)

const

Definition at line 138 of file StringDictionaryProxy.cpp.

References getStringUnlocked(), and rw_mutex_.

Referenced by anonymous_namespace{ResultSetIteration.cpp}::build_string_array_target_value(), StringValueConverter::convertToColumnarFormatFromDict(), DictionaryValueConverter< TARGET_TYPE >::convertTransientStringIdToPermanentId(), anonymous_namespace{RelAlgTranslator.cpp}::fill_dictionary_encoded_in_vals(), intersect_translate_string_id_to_other_dict(), and union_translate_string_id_to_other_dict().

                                                                  {
  if (inline_int_null_value<int32_t>() == string_id) {
    return "";
  }
  std::shared_lock<std::shared_mutex> read_lock(rw_mutex_);
  return getStringUnlocked(string_id);
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::pair< const char *, size_t > StringDictionaryProxy::getStringBytes ( int32_t  string_id ) const

noexcept

Definition at line 490 of file StringDictionaryProxy.cpp.

References CHECK_LT.

Referenced by anonymous_namespace{ExternalExecutor.cpp}::decode_string(), and string_decompress().

                                      {
  if (string_id >= 0) {
    return string_dict_.get()->getStringBytes(string_id);
  }
  unsigned const string_index = transientIdToIndex(string_id);
  CHECK_LT(string_index, transient_string_vec_.size());
  std::string const* const str_ptr = transient_string_vec_[string_index];
  return {str_ptr->c_str(), str_ptr->size()};
}

Here is the caller graph for this function:

std::vector< std::string > StringDictionaryProxy::getStrings

(

const std::vector< int32_t > &

string_ids

)

const

Definition at line 155 of file StringDictionaryProxy.cpp.

References string_dict_, transient_string_vec_, and transientIdToIndex().

                                                {
  std::vector<std::string> strings;
  if (!string_ids.empty()) {
    strings.reserve(string_ids.size());
    for (const auto string_id : string_ids) {
      if (string_id >= 0) {
        strings.emplace_back(string_dict_->getString(string_id));
      } else if (inline_int_null_value<int32_t>() == string_id) {
        strings.emplace_back("");
      } else {
        unsigned const string_index = transientIdToIndex(string_id);
        strings.emplace_back(*transient_string_vec_[string_index]);
      }
    }
  }
  return strings;
}

Here is the call graph for this function:

std::string StringDictionaryProxy::getStringUnlocked ( const int32_t  string_id ) const

private

Definition at line 146 of file StringDictionaryProxy.cpp.

References CHECK_LT, storageEntryCount(), string_dict_, transient_string_vec_, and transientIdToIndex().

Referenced by buildUnionTranslationMapToOtherProxy(), and getString().

                                                                                {
  if (string_id >= 0 && storageEntryCount() > 0) {
    return string_dict_->getString(string_id);
  }
  unsigned const string_index = transientIdToIndex(string_id);
  CHECK_LT(string_index, transient_string_vec_.size());
  return *transient_string_vec_[string_index];
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::vector< int32_t > StringDictionaryProxy::getTransientBulk

(

const std::vector< std::string > &

strings

)

const

Executes read-only lookup of a vector of strings and returns a vector of their integer ids.

This function, unlike getOrAddTransientBulk, will not add strings to the dictionary. Use this function if strings that don't currently exist in the StringDictionaryProxy should not be added to the proxy as transient entries. This method also has performance advantages over getOrAddTransientBulk for read-only use cases, in that it can: 1) Take a read lock instead of a write lock for the transient lookups 2) Use a tbb::parallel_for implementation of the transient string lookups as we are guaranteed that the underlying map of strings to int ids cannot change

Parameters

strings

- Vector of strings to perform string id lookups on

Returns

A vector of string_ids of the same length as strings, containing the id of any strings for which were found in the underlying StringDictionary instance or in the proxy's tranient map, otherwise StringDictionary::INVALID_STR_ID for strings not found.

Definition at line 52 of file StringDictionaryProxy.cpp.

References CHECK_GE, generation_, and getTransientBulkImpl().

                                               {
  CHECK_GE(generation_, 0);
  std::vector<int32_t> string_ids(strings.size());
  getTransientBulkImpl(strings, string_ids.data(), true);
  return string_ids;
}

Here is the call graph for this function:

size_t StringDictionaryProxy::getTransientBulkImpl ( const std::vector< std::string > &  strings, int32_t *  string_ids, const bool  take_read_lock  ) const

private

Definition at line 627 of file StringDictionaryProxy.cpp.

References generation_, string_dict_, and transientLookupBulk().

Referenced by buildIntersectionTranslationMapToOtherProxyUnlocked(), and getTransientBulk().

                                     {
  const size_t num_strings = strings.size();
  if (num_strings == 0) {
    return 0UL;
  }
  // StringDictionary::getBulk returns the number of strings not found
  if (string_dict_->getBulk(strings, string_ids, generation_) == 0UL) {
    return 0UL;
  }

  // If here, dictionary could not find at least 1 target string,
  // now look these up in the transient dictionary
  // transientLookupBulk returns the number of strings not found
  return transientLookupBulk(strings, string_ids, take_read_lock);
}

Here is the call graph for this function:

Here is the caller graph for this function:

const std::vector<std::string const*>& StringDictionaryProxy::getTransientVector ( ) const

inline

Definition at line 228 of file StringDictionaryProxy.h.

References transient_string_vec_.

Referenced by DictionaryValueConverter< TARGET_TYPE >::DictionaryValueConverter(), and DictionaryValueConverter< TARGET_TYPE >::processBuffer().

                                                                {
    return transient_string_vec_;
  }

Here is the caller graph for this function:

IdMap StringDictionaryProxy::initIdMap ( ) const

inline

Definition at line 134 of file StringDictionaryProxy.h.

References generation_, and transient_string_vec_.

Referenced by buildIntersectionTranslationMapToOtherProxyUnlocked(), and transientUnion().

{ return IdMap(transient_string_vec_.size(), generation_); }

Here is the caller graph for this function:

template<typename String >

int32_t StringDictionaryProxy::lookupTransientStringUnlocked ( const String &  lookup_string ) const

private

Definition at line 175 of file StringDictionaryProxy.cpp.

References StringDictionary::INVALID_STR_ID, and transient_str_to_int_.

Referenced by buildIntersectionTranslationMapToOtherProxyUnlocked(), transientLookupBulkParallelUnlocked(), and transientLookupBulkUnlocked().

                                       {
  const auto it = transient_str_to_int_.find(lookup_string);
  return it == transient_str_to_int_.end() ? StringDictionary::INVALID_STR_ID
                                           : it->second;
}

Here is the caller graph for this function:

bool StringDictionaryProxy::operator!=

(

StringDictionaryProxy const &

rhs

)

const

Definition at line 743 of file StringDictionaryProxy.cpp.

References operator==().

                                                                             {
  return !operator==(rhs);
}

Here is the call graph for this function:

StringDictionaryProxy const& StringDictionaryProxy::operator= ( StringDictionaryProxy const &  )

delete

bool StringDictionaryProxy::operator==

(

StringDictionaryProxy const &

rhs

)

const

Definition at line 738 of file StringDictionaryProxy.cpp.

References string_dict_id_, and transient_str_to_int_.

Referenced by operator!=().

                                                                             {
  return string_dict_id_ == rhs.string_dict_id_ &&
         transient_str_to_int_ == rhs.transient_str_to_int_;
}

Here is the caller graph for this function:

size_t StringDictionaryProxy::persistedC ( ) const

private

size_t StringDictionaryProxy::storageEntryCount

(

)

const

Returns the number of string entries in the underlying string dictionary, at this proxy's generation_ if it is set/valid, otherwise just the current size of the dictionary.

Returns

size_t Number of entries in the string dictionary (at this proxy's generation if set)

Definition at line 501 of file StringDictionaryProxy.cpp.

References CHECK_LE, generation_, and string_dict_.

Referenced by buildIntersectionTranslationMapToOtherProxyUnlocked(), entryCountUnlocked(), and getStringUnlocked().

                                                      {
  const size_t num_storage_entries{generation_ == -1 ? string_dict_->storageEntryCount()
                                                     : generation_};
  CHECK_LE(num_storage_entries, static_cast<size_t>(std::numeric_limits<int32_t>::max()));
  return num_storage_entries;
}

Here is the caller graph for this function:

size_t StringDictionaryProxy::transientEntryCount

(

)

const

Returns the number of transient string entries for this proxy,.

Returns

size_t Number of transient string entries for this proxy

Definition at line 517 of file StringDictionaryProxy.cpp.

References rw_mutex_, and transientEntryCountUnlocked().

                                                        {
  std::shared_lock<std::shared_mutex> read_lock(rw_mutex_);
  return transientEntryCountUnlocked();
}

Here is the call graph for this function:

size_t StringDictionaryProxy::transientEntryCountUnlocked ( ) const

private

Definition at line 508 of file StringDictionaryProxy.cpp.

References CHECK_LE, and transient_str_to_int_.

Referenced by buildIntersectionTranslationMapToOtherProxyUnlocked(), buildUnionTranslationMapToOtherProxy(), entryCountUnlocked(), and transientEntryCount().

                                                                {
  // CHECK_LE(num_storage_entries,
  // static_cast<size_t>(std::numeric_limits<int32_t>::max()));
  const size_t num_transient_entries{transient_str_to_int_.size()};
  CHECK_LE(num_transient_entries,
           static_cast<size_t>(std::numeric_limits<int32_t>::max()) - 1);
  return num_transient_entries;
}

Here is the caller graph for this function:

static unsigned StringDictionaryProxy::transientIdToIndex ( int32_t const  id )

inlinestatic

Definition at line 234 of file StringDictionaryProxy.h.

Referenced by getStrings(), getStringUnlocked(), and StringDictionary::populate_string_ids().

                                                       {
    constexpr int max_transient_string_id = -2;
    return static_cast<unsigned>(max_transient_string_id - id);
  }

Here is the caller graph for this function:

static int32_t StringDictionaryProxy::transientIndexToId ( unsigned const  index )

inlinestatic

Definition at line 239 of file StringDictionaryProxy.h.

Referenced by DictionaryValueConverter< TARGET_TYPE >::DictionaryValueConverter(), getCompare(), getLike(), getOrAddTransientUnlocked(), getRegexpLike(), and ArrowResultSetConverter::initializeColumnBuilder().

                                                          {
    constexpr int max_transient_string_id = -2;
    return static_cast<int32_t>(max_transient_string_id - index);
  }

Here is the caller graph for this function:

template<typename String >

size_t StringDictionaryProxy::transientLookupBulk ( const std::vector< String > &  lookup_strings, int32_t *  string_ids, const bool  take_read_lock  ) const

private

Definition at line 647 of file StringDictionaryProxy.cpp.

References rw_mutex_, transient_str_to_int_, transientLookupBulkParallelUnlocked(), and transientLookupBulkUnlocked().

Referenced by getTransientBulkImpl().

                                     {
  const size_t num_strings = lookup_strings.size();
  auto read_lock = take_read_lock ? std::shared_lock<std::shared_mutex>(rw_mutex_)
                                  : std::shared_lock<std::shared_mutex>();

  if (num_strings == static_cast<size_t>(0) || transient_str_to_int_.empty()) {
    return 0UL;
  }
  constexpr size_t tbb_parallel_threshold{20000};
  if (num_strings < tbb_parallel_threshold) {
    return transientLookupBulkUnlocked(lookup_strings, string_ids);
  } else {
    return transientLookupBulkParallelUnlocked(lookup_strings, string_ids);
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename String >

size_t StringDictionaryProxy::transientLookupBulkParallelUnlocked ( const std::vector< String > &  lookup_strings, int32_t *  string_ids  ) const

private

Definition at line 687 of file StringDictionaryProxy.cpp.

References CHECK_GE, StringDictionary::INVALID_STR_ID, lookupTransientStringUnlocked(), ThreadInfo::num_elems_per_thread, ThreadInfo::num_threads, and threading_serial::parallel_for().

Referenced by transientLookupBulk().

                               {
  const size_t num_lookup_strings = lookup_strings.size();
  const size_t target_inputs_per_thread = 20000L;
  ThreadInfo thread_info(
      std::thread::hardware_concurrency(), num_lookup_strings, target_inputs_per_thread);
  CHECK_GE(thread_info.num_threads, 1L);
  CHECK_GE(thread_info.num_elems_per_thread, 1L);

  std::vector<size_t> num_strings_not_found_per_thread(thread_info.num_threads, 0UL);

  tbb::task_arena limited_arena(thread_info.num_threads);
  limited_arena.execute([&] {
    tbb::parallel_for(
        tbb::blocked_range<size_t>(
            0, num_lookup_strings, thread_info.num_elems_per_thread /* tbb grain_size */),
        [&](const tbb::blocked_range<size_t>& r) {
          const size_t start_idx = r.begin();
          const size_t end_idx = r.end();
          size_t num_local_strings_not_found = 0;
          for (size_t string_idx = start_idx; string_idx < end_idx; ++string_idx) {
            if (string_ids[string_idx] != StringDictionary::INVALID_STR_ID) {
              continue;
            }
            string_ids[string_idx] =
                lookupTransientStringUnlocked(lookup_strings[string_idx]);
            if (string_ids[string_idx] == StringDictionary::INVALID_STR_ID) {
              num_local_strings_not_found++;
            }
          }
          const size_t tbb_thread_idx = tbb::this_task_arena::current_thread_index();
          num_strings_not_found_per_thread[tbb_thread_idx] = num_local_strings_not_found;
        },
        tbb::simple_partitioner());
  });
  size_t num_strings_not_found = 0;
  for (int64_t thread_idx = 0; thread_idx < thread_info.num_threads; ++thread_idx) {
    num_strings_not_found += num_strings_not_found_per_thread[thread_idx];
  }
  return num_strings_not_found;
}

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename String >

size_t StringDictionaryProxy::transientLookupBulkUnlocked ( const std::vector< String > &  lookup_strings, int32_t *  string_ids  ) const

private

Definition at line 667 of file StringDictionaryProxy.cpp.

References StringDictionary::INVALID_STR_ID, and lookupTransientStringUnlocked().

Referenced by transientLookupBulk().

                               {
  const size_t num_strings = lookup_strings.size();
  size_t num_strings_not_found = 0;
  for (size_t string_idx = 0; string_idx < num_strings; ++string_idx) {
    if (string_ids[string_idx] != StringDictionary::INVALID_STR_ID) {
      continue;
    }
    // If we're here it means we need to look up this string as we don't
    // have a valid id for it
    string_ids[string_idx] = lookupTransientStringUnlocked(lookup_strings[string_idx]);
    if (string_ids[string_idx] == StringDictionary::INVALID_STR_ID) {
      num_strings_not_found++;
    }
  }
  return num_strings_not_found;
}

Here is the call graph for this function:

Here is the caller graph for this function:

StringDictionaryProxy::IdMap StringDictionaryProxy::transientUnion

(

StringDictionaryProxy const &

sdp_rhs

)

Definition at line 596 of file StringDictionaryProxy.cpp.

References eachStringSerially(), initIdMap(), and string_dict_.

                                          {
  IdMap id_map = sdp_rhs.initIdMap();
  // serial_callback cannot be parallelized due to calling getOrAddTransientUnlocked().
  std::unique_ptr<StringDictionary::StringCallback> serial_callback;
  if (string_dict_->isClient()) {
    serial_callback = std::make_unique<StringNetworkCallback>(this, id_map);
  } else {
    serial_callback = std::make_unique<StringLocalCallback>(this, id_map);
  }
  // Import all non-duplicate strings (transient and non-transient) and add to id_map.
  sdp_rhs.eachStringSerially(*serial_callback);
  return id_map;
}

Here is the call graph for this function:

void StringDictionaryProxy::updateGeneration ( const int64_t  generation )

noexcept

Definition at line 616 of file StringDictionaryProxy.cpp.

References CHECK_EQ.

                                                                              {
  if (generation == -1) {
    return;
  }
  if (generation_ != -1) {
    CHECK_EQ(generation_, generation);
    return;
  }
  generation_ = generation;
}

Friends And Related Function Documentation

friend class StringLocalCallback

friend

Definition at line 290 of file StringDictionaryProxy.h.

friend class StringNetworkCallback

friend

Definition at line 291 of file StringDictionaryProxy.h.

Member Data Documentation

int64_t StringDictionaryProxy::generation_

private

Definition at line 281 of file StringDictionaryProxy.h.

Referenced by buildIntersectionTranslationMapToOtherProxyUnlocked(), eachStringSerially(), getCompare(), getGeneration(), getIdOfStringFromClient(), getLike(), getOrAddTransientBulk(), getRegexpLike(), getTransientBulk(), getTransientBulkImpl(), initIdMap(), and storageEntryCount().

std::shared_mutex StringDictionaryProxy::rw_mutex_

mutableprivate

Definition at line 282 of file StringDictionaryProxy.h.

Referenced by buildIntersectionTranslationMapToOtherProxy(), buildUnionTranslationMapToOtherProxy(), entryCount(), getIdOfString(), getIdOfStringNoGeneration(), getOrAddTransient(), getOrAddTransientBulk(), getString(), transientEntryCount(), and transientLookupBulk().

std::shared_ptr<StringDictionary> StringDictionaryProxy::string_dict_

private

Definition at line 276 of file StringDictionaryProxy.h.

Referenced by buildIntersectionTranslationMapToOtherProxyUnlocked(), buildUnionTranslationMapToOtherProxy(), eachStringSerially(), getCompare(), getDictionary(), getIdOfStringFromClient(), getIdOfStringNoGeneration(), getLike(), getOrAddTransientBulk(), getRegexpLike(), getStrings(), getStringUnlocked(), getTransientBulkImpl(), StringLocalCallback::operator()(), storageEntryCount(), and transientUnion().

const int32_t StringDictionaryProxy::string_dict_id_

private

Definition at line 277 of file StringDictionaryProxy.h.

Referenced by getDictId(), and operator==().

TransientMap StringDictionaryProxy::transient_str_to_int_

private

Definition at line 278 of file StringDictionaryProxy.h.

Referenced by getIdOfString(), getIdOfStringNoGeneration(), getOrAddTransientUnlocked(), lookupTransientStringUnlocked(), operator==(), transientEntryCountUnlocked(), and transientLookupBulk().

std::vector<std::string const*> StringDictionaryProxy::transient_string_vec_

private

Definition at line 280 of file StringDictionaryProxy.h.

Referenced by buildIntersectionTranslationMapToOtherProxyUnlocked(), eachStringSerially(), getCompare(), getLike(), getOrAddTransientUnlocked(), getRegexpLike(), getStrings(), getStringUnlocked(), getTransientVector(), and initIdMap().

---

The documentation for this class was generated from the following files:

• /home/jenkins-slave/workspace/core-os-doxygen/StringDictionary/StringDictionaryProxy.h
• /home/jenkins-slave/workspace/core-os-doxygen/StringDictionary/StringDictionaryProxy.cpp