anonymous_namespace{StringDictionary.cpp} Namespace Reference

Classes
struct	ThreadInfo
class	MapMaker

Functions
int	checked_open (const char *path, const bool recover)
const uint64_t	round_up_p2 (const uint64_t num)
string_dict_hash_t	hash_string (const std::string_view &str)
template<class T >
void	throw_encoding_error (std::string_view str, const DictRef &dict_ref)
void	throw_string_too_long_error (std::string_view str, const DictRef &dict_ref)
bool	is_like (const std::string &str, const std::string &pattern, const bool icase, const bool is_simple, const char escape)
bool	is_regexp_like (const std::string &str, const std::string &pattern, const char escape)

Variables
const int	SYSTEM_PAGE_SIZE = heavyai::get_page_size()

Function Documentation

int anonymous_namespace{StringDictionary.cpp}::checked_open	(	const char *	path,
		const bool	recover
	)

Definition at line 58 of file StringDictionary.cpp.

References logger::ERROR, LOG, and heavyai::open().

Referenced by StringDictionary::StringDictionary().

                                                       {
  auto fd = heavyai::open(path, O_RDWR | O_CREAT | (recover ? O_APPEND : O_TRUNC), 0644);
  if (fd > 0) {
    return fd;
  }
  auto err = std::string("Dictionary path ") + std::string(path) +
             std::string(" does not exist.");
  LOG(ERROR) << err;
  throw DictPayloadUnavailable(err);
}

Here is the call graph for this function:

Here is the caller graph for this function:

string_dict_hash_t anonymous_namespace{StringDictionary.cpp}::hash_string

(

const std::string_view &

str

)

Definition at line 87 of file StringDictionary.cpp.

Referenced by StringDictionary::buildDictionaryTranslationMap(), StringDictionary::getBulk(), StringDictionary::getOrAddBulk(), StringDictionary::getOrAddImpl(), StringDictionary::getUnlocked(), StringDictionary::hashStrings(), StringDictionary::increaseHashTableCapacity(), StringDictionary::increaseHashTableCapacityFromStorageAndMemory(), and StringDictionary::StringDictionary().

                                                          {
  string_dict_hash_t str_hash = 1;
  // rely on fact that unsigned overflow is defined and wraps
  for (size_t i = 0; i < str.size(); ++i) {
    str_hash = str_hash * 997 + str[i];
  }
  return str_hash;
}

Here is the caller graph for this function:

bool anonymous_namespace{StringDictionary.cpp}::is_like	(	const std::string &	str,
		const std::string &	pattern,
		const bool	icase,
		const bool	is_simple,
		const char	escape
	)

Definition at line 801 of file StringDictionary.cpp.

References string_ilike(), string_ilike_simple(), string_like(), and string_like_simple().

Referenced by StringDictionary::getLike(), and StringDictionaryProxy::getLike().

                                {
  return icase
             ? (is_simple ? string_ilike_simple(
                                str.c_str(), str.size(), pattern.c_str(), pattern.size())
                          : string_ilike(str.c_str(),
                                         str.size(),
                                         pattern.c_str(),
                                         pattern.size(),
                                         escape))
             : (is_simple ? string_like_simple(
                                str.c_str(), str.size(), pattern.c_str(), pattern.size())
                          : string_like(str.c_str(),
                                        str.size(),
                                        pattern.c_str(),
                                        pattern.size(),
                                        escape));
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool anonymous_namespace{StringDictionary.cpp}::is_regexp_like	(	const std::string &	str,
		const std::string &	pattern,
		const char	escape
	)

Definition at line 1091 of file StringDictionary.cpp.

References regexp_like().

Referenced by StringDictionary::getRegexpLike(), and StringDictionaryProxy::getRegexpLike().

                                       {
  return regexp_like(str.c_str(), str.size(), pattern.c_str(), pattern.size(), escape);
}

Here is the call graph for this function:

Here is the caller graph for this function:

const uint64_t anonymous_namespace{StringDictionary.cpp}::round_up_p2

(

const uint64_t

num

)

Definition at line 69 of file StringDictionary.cpp.

Referenced by StringDictionary::StringDictionary().

                                               {
  uint64_t in = num;
  in--;
  in |= in >> 1;
  in |= in >> 2;
  in |= in >> 4;
  in |= in >> 8;
  in |= in >> 16;
  in++;
  // TODO MAT deal with case where filesize has been increased but reality is
  // we are constrained to 2^31.
  // In that situation this calculation will wrap to zero
  if (in == 0 || (in > (UINT32_MAX))) {
    in = UINT32_MAX;
  }
  return in;
}

Here is the caller graph for this function:

template<class T >

void anonymous_namespace{StringDictionary.cpp}::throw_encoding_error	(	std::string_view	str,
		const DictRef &	dict_ref
	)

Definition at line 395 of file StringDictionary.cpp.

References logger::ERROR, LOG, StringDictionary::MAX_STRCOUNT, heavydb.dtypes::T, and dict_ref_t::toString().

                                                                       {
  std::ostringstream oss;
  oss << "The text encoded column using dictionary " << dict_ref.toString()
      << " has exceeded it's limit of " << sizeof(T) * 8 << " bits ("
      << static_cast<size_t>(max_valid_int_value<T>() + 1) << " unique values) "
      << "while attempting to add the new string '" << str << "'. ";

  if (sizeof(T) < 4) {
    // Todo: Implement automatic type widening for dictionary-encoded text
    // columns/all fixed length columm types (at least if not defined
    //  with fixed encoding size), or short of that, ALTER TABLE
    // COLUMN TYPE to at least allow the user to do this manually
    // without re-creating the table

    oss << "To load more data, please re-create the table with "
        << "this column as type TEXT ENCODING DICT(" << sizeof(T) * 2 * 8 << ") ";
    if (sizeof(T) == 1) {
      oss << "or TEXT ENCODING DICT(32) ";
    }
    oss << "and reload your data.";
  } else {
    // Todo: Implement TEXT ENCODING DICT(64) type which should essentially
    // preclude overflows.
    oss << "Currently dictionary-encoded text columns support a maximum of "
        << StringDictionary::MAX_STRCOUNT
        << " strings. Consider recreating the table with "
        << "this column as type TEXT ENCODING NONE and reloading your data.";
  }
  LOG(ERROR) << oss.str();
  throw std::runtime_error(oss.str());
}

Here is the call graph for this function:

void anonymous_namespace{StringDictionary.cpp}::throw_string_too_long_error	(	std::string_view	str,
		const DictRef &	dict_ref
	)

Definition at line 427 of file StringDictionary.cpp.

References logger::ERROR, LOG, StringDictionary::MAX_STRLEN, and dict_ref_t::toString().

Referenced by StringDictionary::getBulk().

                                                                              {
  std::ostringstream oss;
  oss << "The string '" << str << " could not be inserted into the dictionary "
      << dict_ref.toString() << " because it exceeded the maximum allowable "
      << "length of " << StringDictionary::MAX_STRLEN << " characters (string was "
      << str.size() << " characters).";
  LOG(ERROR) << oss.str();
  throw std::runtime_error(oss.str());
}

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

const int anonymous_namespace{StringDictionary.cpp}::SYSTEM_PAGE_SIZE = heavyai::get_page_size()

Definition at line 56 of file StringDictionary.cpp.

Referenced by StringDictionary::addMemoryCapacity(), and StringDictionary::addStorageCapacity().