StringDictionary.cpp File Reference

#include "StringDictionary/StringDictionaryProxy.h"
#include "StringOps/StringOps.h"
#include <tbb/parallel_for.h>
#include <tbb/task_arena.h>
#include <algorithm>
#include <boost/filesystem/operations.hpp>
#include <boost/filesystem/path.hpp>
#include <boost/sort/spreadsort/string_sort.hpp>
#include <functional>
#include <future>
#include <iostream>
#include <string_view>
#include <thread>
#include <type_traits>
#include <sys/fcntl.h>
#include "Logger/Logger.h"
#include "OSDependent/heavyai_fs.h"
#include "Shared/sqltypes.h"
#include "Shared/thread_count.h"
#include "StringDictionaryClient.h"
#include "Utils/Regexp.h"
#include "Utils/StringLike.h"
#include "LeafHostInfo.h"

Include dependency graph for StringDictionary.cpp:

Go to the source code of this file.

Classes
struct	anonymous_namespace{StringDictionary.cpp}::ThreadInfo
class	anonymous_namespace{StringDictionary.cpp}::MapMaker

Namespaces
	anonymous_namespace{StringDictionary.cpp}

Functions
int	anonymous_namespace{StringDictionary.cpp}::checked_open (const char *path, const bool recover)
const uint64_t	anonymous_namespace{StringDictionary.cpp}::round_up_p2 (const uint64_t num)
string_dict_hash_t	anonymous_namespace{StringDictionary.cpp}::hash_string (const std::string_view &str)
template<class T >
void	anonymous_namespace{StringDictionary.cpp}::throw_encoding_error (std::string_view str, const DictRef &dict_ref)
void	anonymous_namespace{StringDictionary.cpp}::throw_string_too_long_error (std::string_view str, const DictRef &dict_ref)
bool	anonymous_namespace{StringDictionary.cpp}::is_like (const std::string &str, const std::string &pattern, const bool icase, const bool is_simple, const char escape)
bool	anonymous_namespace{StringDictionary.cpp}::is_regexp_like (const std::string &str, const std::string &pattern, const char escape)
void	order_translation_locks (const int32_t source_db_id, const int32_t source_dict_id, const int32_t dest_db_id, const int32_t dest_dict_id, std::shared_lock< std::shared_mutex > &source_read_lock, std::shared_lock< std::shared_mutex > &dest_read_lock)
void	translate_string_ids (std::vector< int32_t > &dest_ids, const LeafHostInfo &dict_server_host, const DictRef dest_dict_ref, const std::vector< int32_t > &source_ids, const DictRef source_dict_ref, const int32_t dest_generation)

Variables
bool	g_cache_string_hash {true}
const int	anonymous_namespace{StringDictionary.cpp}::SYSTEM_PAGE_SIZE = heavyai::get_page_size()
bool	g_enable_stringdict_parallel {false}

Function Documentation

void order_translation_locks	(	const int32_t	source_db_id,
		const int32_t	source_dict_id,
		const int32_t	dest_db_id,
		const int32_t	dest_dict_id,
		std::shared_lock< std::shared_mutex > &	source_read_lock,
		std::shared_lock< std::shared_mutex > &	dest_read_lock
	)

Definition at line 1794 of file StringDictionary.cpp.

Referenced by StringDictionary::buildDictionaryTranslationMap(), StringDictionaryProxy::buildIntersectionTranslationMapToOtherProxy(), and StringDictionaryProxy::buildUnionTranslationMapToOtherProxy().

                                                                              {
  const bool dicts_are_same =
      source_db_id == dest_db_id && source_dict_id == dest_dict_id;
  const bool source_dict_is_locked_first =
      source_db_id < dest_db_id ||
      (source_db_id == dest_db_id && source_dict_id < dest_dict_id);
  if (dicts_are_same) {
    // dictionaries are same, only take one write lock
    dest_read_lock.lock();
  } else if (source_dict_is_locked_first) {
    source_read_lock.lock();
    dest_read_lock.lock();
  } else {
    dest_read_lock.lock();
    source_read_lock.lock();
  }
}

Here is the caller graph for this function:

void translate_string_ids	(	std::vector< int32_t > &	dest_ids,
		const LeafHostInfo &	dict_server_host,
		const DictRef	dest_dict_ref,
		const std::vector< int32_t > &	source_ids,
		const DictRef	source_dict_ref,
		const int32_t	dest_generation
	)

Definition at line 1969 of file StringDictionary.cpp.

References StringDictionaryClient::translate_string_ids().

Referenced by anonymous_namespace{RelAlgTranslator.cpp}::fill_dictionary_encoded_in_vals(), and ResultSet::translateDictEncodedColumns().

                                                         {
  DictRef temp_dict_ref(-1, -1);
  StringDictionaryClient string_client(dict_server_host, temp_dict_ref, false);
  string_client.translate_string_ids(
      dest_ids, dest_dict_ref, source_ids, source_dict_ref, dest_generation);
}

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

bool g_cache_string_hash {true}

Definition at line 51 of file StringDictionary.cpp.

Referenced by CommandLineOptions::fillOptions(), Catalog_Namespace::Catalog::getMetadataForDict(), and RowSetMemoryOwner::getOrAddStringDictProxy().

bool g_enable_stringdict_parallel {false}

Definition at line 112 of file StringDictionary.cpp.

Referenced by CommandLineOptions::fillOptions(), and StringDictionary::getOrAddBulk().