#include "CodeGenerator.h"
#include "Execute.h"
#include "../Shared/funcannotations.h"
#include "../Shared/sqldefs.h"
#include "Parser/ParserNode.h"
#include "QueryEngine/ExpressionRewrite.h"
#include "StringOps/StringOps.h"
#include <boost/locale/conversion.hpp>

Include dependency graph for StringOpsIR.cpp:

Namespaces
	anonymous_namespace{StringOpsIR.cpp}

Macros
#define	DEF_APPLY_NUMERIC_STRING_OPS(value_type, value_name)

#define	DEF_CONVERT_TO_STRING_AND_ENCODE(value_type, value_name)

Functions
RUNTIME_EXPORT StringView	string_decode (int8_t *chunk_iter_, int64_t pos)

RUNTIME_EXPORT StringView	string_decompress (const int32_t string_id, const int64_t string_dict_handle)

RUNTIME_EXPORT int32_t	string_compress (const StringView string_view, const int64_t string_dict_handle)

RUNTIME_EXPORT int32_t	apply_string_ops_and_encode (const char *str_ptr, const int32_t str_len, const int64_t string_ops_handle, const int64_t string_dict_handle)

RUNTIME_EXPORT int32_t	apply_multi_input_string_ops_and_encode (const char str1_ptr, const int32_t str1_len, const char str2_ptr, const int32_t str2_len, const int64_t string_ops_handle, const int64_t string_dict_handle)

RUNTIME_EXPORT int32_t	intersect_translate_string_id_to_other_dict (const int32_t string_id, const int64_t source_string_dict_handle, const int64_t dest_string_dict_handle)

RUNTIME_EXPORT int32_t	union_translate_string_id_to_other_dict (const int32_t string_id, const int64_t source_string_dict_handle, const int64_t dest_string_dict_handle)

int32_t	write_string_to_proxy (const std::string &str, const int64_t string_dict_handle)

RUNTIME_EXPORT ALWAYS_INLINE int32_t	convert_to_string_and_encode_bool (const int8_t operand, const int64_t string_dict_handle)

RUNTIME_EXPORT ALWAYS_INLINE int32_t	convert_to_string_and_encode_decimal (const int64_t operand, const int32_t precision, const int32_t scale, const int64_t string_dict_handle)

RUNTIME_EXPORT ALWAYS_INLINE int32_t	convert_to_string_and_encode_time (const int64_t operand, const int64_t string_dict_handle)

RUNTIME_EXPORT ALWAYS_INLINE int32_t	convert_to_string_and_encode_timestamp (const int64_t operand, const int32_t dimension, const int64_t string_dict_handle)

RUNTIME_EXPORT ALWAYS_INLINE int32_t	convert_to_string_and_encode_date (const int64_t operand, const int64_t string_dict_handle)

std::vector < StringOps_Namespace::StringOpInfo >	getStringOpInfos (const Analyzer::StringOper *expr)

std::unique_ptr < StringDictionaryTranslationMgr >	translate_dict_strings (const Analyzer::StringOper expr, const ExecutorDeviceType device_type, Executor executor)

void	pre_translate_string_ops (const Analyzer::StringOper string_oper, Executor executor)

std::vector< int32_t >	anonymous_namespace{StringOpsIR.cpp}::get_compared_ids (const StringDictionaryProxy *dict, const SQLOps compare_operator, const std::string &pattern)

Macro Definition Documentation

#define DEF_APPLY_NUMERIC_STRING_OPS	(	value_type,
		value_name
	)

Value:

extern "C" RUNTIME_EXPORT ALWAYS_INLINE value_type                                     \
      apply_numeric_string_ops_##value_name(                                             \
          const char* str_ptr, const int32_t str_len, const int64_t string_ops_handle) { \
    const std::string_view raw_str(str_ptr, str_len);                                    \
    auto string_ops =                                                                    \
        reinterpret_cast<const StringOps_Namespace::StringOps*>(string_ops_handle);      \
    const auto result_datum = string_ops->numericEval(raw_str);                          \
    return result_datum.value_name##val;                                                 \
  }

Definition at line 127 of file StringOpsIR.cpp.

#define DEF_CONVERT_TO_STRING_AND_ENCODE	(	value_type,
		value_name
	)

Value:

extern "C" RUNTIME_EXPORT ALWAYS_INLINE int32_t                                   \
      convert_to_string_and_encode_##value_name(const value_type operand,           \
                                                const int64_t string_dict_handle) { \
    return write_string_to_proxy(std::to_string(operand), string_dict_handle);      \
  }

Definition at line 157 of file StringOpsIR.cpp.

Function Documentation

RUNTIME_EXPORT int32_t apply_multi_input_string_ops_and_encode	(	const char *	str1_ptr,
		const int32_t	str1_len,
		const char *	str2_ptr,
		const int32_t	str2_len,
		const int64_t	string_ops_handle,
		const int64_t	string_dict_handle
	)

Definition at line 77 of file StringOpsIR.cpp.

References StringDictionaryProxy::getOrAddTransient().

                                                                           {
   std::string_view raw_str1(str1_ptr, str1_len);
   std::string_view raw_str2(str2_ptr, str2_len);
   auto string_ops =
       reinterpret_cast<const StringOps_Namespace::StringOps*>(string_ops_handle);
   auto string_dict_proxy = reinterpret_cast<StringDictionaryProxy*>(string_dict_handle);
   const auto result_str = string_ops->multi_input_eval(raw_str1, raw_str2);
   if (result_str.empty()) {
     return inline_int_null_value<int32_t>();
   }
   return string_dict_proxy->getOrAddTransient(result_str);
 }

Here is the call graph for this function:

RUNTIME_EXPORT int32_t apply_string_ops_and_encode	(	const char *	str_ptr,
		const int32_t	str_len,
		const int64_t	string_ops_handle,
		const int64_t	string_dict_handle
	)

Definition at line 61 of file StringOpsIR.cpp.

References StringDictionaryProxy::getOrAddTransient().

                                                               {
   std::string raw_str(str_ptr, str_len);
   auto string_ops =
       reinterpret_cast<const StringOps_Namespace::StringOps*>(string_ops_handle);
   auto string_dict_proxy = reinterpret_cast<StringDictionaryProxy*>(string_dict_handle);
   const auto result_str = string_ops->operator()(raw_str);
   if (result_str.empty()) {
     return inline_int_null_value<int32_t>();
   }
   return string_dict_proxy->getOrAddTransient(result_str);
 }

Here is the call graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE int32_t convert_to_string_and_encode_bool	(	const int8_t	operand,
		const int64_t	string_dict_handle
	)

Definition at line 174 of file StringOpsIR.cpp.

References write_string_to_proxy().

                                                                     {
   return write_string_to_proxy(operand == 1 ? "true" : "false", string_dict_handle);
 }

Here is the call graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE int32_t convert_to_string_and_encode_date	(	const int64_t	operand,
		const int64_t	string_dict_handle
	)

Definition at line 211 of file StringOpsIR.cpp.

References shared::formatDate(), and write_string_to_proxy().

                                                                     {
   constexpr size_t buf_size = 64;
   char buf[buf_size];  // Hold "2000-03-01 12:34:56.123456789" and large years.
   shared::formatDate(buf, buf_size, operand);
   return write_string_to_proxy(buf, string_dict_handle);
 }

Here is the call graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE int32_t convert_to_string_and_encode_decimal	(	const int64_t	operand,
		const int32_t	precision,
		const int32_t	scale,
		const int64_t	string_dict_handle
	)

Definition at line 180 of file StringOpsIR.cpp.

References shared::power10inv(), and write_string_to_proxy().

                                                                        {
   constexpr size_t buf_size = 64;
   char buf[buf_size];  // Hold "2000-03-01 12:34:56.123456789" and large years.
   const double v = static_cast<double>(operand) * shared::power10inv(scale);
   snprintf(buf, buf_size, "%*.*f", precision, scale, v);
   return write_string_to_proxy(buf, string_dict_handle);
 }

Here is the call graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE int32_t convert_to_string_and_encode_time	(	const int64_t	operand,
		const int64_t	string_dict_handle
	)

Definition at line 192 of file StringOpsIR.cpp.

References shared::formatHMS(), and write_string_to_proxy().

                                                                     {
   constexpr size_t buf_size = 64;
   char buf[buf_size];
   shared::formatHMS(buf, buf_size, operand);
   return write_string_to_proxy(buf, string_dict_handle);
 }

Here is the call graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE int32_t convert_to_string_and_encode_timestamp	(	const int64_t	operand,
		const int32_t	dimension,
		const int64_t	string_dict_handle
	)

Definition at line 201 of file StringOpsIR.cpp.

References shared::formatDateTime(), and write_string_to_proxy().

                                                                          {
   constexpr size_t buf_size = 64;
   char buf[buf_size];  // Hold "2000-03-01 12:34:56.123456789" and large years.
   shared::formatDateTime(buf, buf_size, operand, dimension);
   return write_string_to_proxy(buf, string_dict_handle);
 }

Here is the call graph for this function:

std::vector<StringOps_Namespace::StringOpInfo> getStringOpInfos ( const Analyzer::StringOper * expr )

Definition at line 257 of file StringOpsIR.cpp.

References CHECK, and Analyzer::StringOper::getChainedStringOpExprs().

Referenced by CodeGenerator::codegenPerRowStringOper(), and translate_dict_strings().

                                     {
   std::vector<StringOps_Namespace::StringOpInfo> string_op_infos;
   auto chained_string_op_exprs = expr->getChainedStringOpExprs();
   if (chained_string_op_exprs.empty()) {
     // Likely will change the below to a CHECK but until we have more confidence
     // that all potential query patterns have nodes that might contain string ops folded,
     // leaving as an error for now
     throw std::runtime_error(
         "Expected folded string operator but found operator unfolded.");
   }
   // Consider encapsulating below in an Analyzer::StringOper method to dedup
   for (const auto& chained_string_op_expr : chained_string_op_exprs) {
     auto chained_string_op =
         dynamic_cast<const Analyzer::StringOper*>(chained_string_op_expr.get());
     CHECK(chained_string_op);
     StringOps_Namespace::StringOpInfo string_op_info(chained_string_op->get_kind(),
                                                      chained_string_op->get_type_info(),
                                                      chained_string_op->getLiteralArgs());
     string_op_infos.emplace_back(string_op_info);
   }
   return string_op_infos;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

RUNTIME_EXPORT int32_t intersect_translate_string_id_to_other_dict	(	const int32_t	string_id,
		const int64_t	source_string_dict_handle,
		const int64_t	dest_string_dict_handle
	)

Definition at line 96 of file StringOpsIR.cpp.

References StringDictionaryProxy::getString().

                                                                                    {
   const auto source_string_dict_proxy =
       reinterpret_cast<StringDictionaryProxy*>(source_string_dict_handle);
   auto dest_string_dict_proxy =
       reinterpret_cast<StringDictionaryProxy*>(dest_string_dict_handle);
   // Can we have StringDictionaryProxy::getString return a reference?
   const auto source_str = source_string_dict_proxy->getString(string_id);
   if (source_str.empty()) {
     return inline_int_null_value<int32_t>();
   }
   return dest_string_dict_proxy->getIdOfString(source_str);
 }

Here is the call graph for this function:

void pre_translate_string_ops	(	const Analyzer::StringOper *	string_oper,
		Executor *	executor
	)

Definition at line 606 of file StringOpsIR.cpp.

References CHECK, CHECK_GT, CHECK_NE, CPU, Analyzer::Expr::get_type_info(), Analyzer::StringOper::getArg(), Analyzer::StringOper::getArity(), TRANSIENT_DICT_ID, and translate_dict_strings().

Referenced by CodeGenerator::codegenDictLike(), and CodeGenerator::codegenDictRegexp().

                                                   {
   // If here we are operating on top of one or more string functions, i.e. LOWER(str),
   // and before running the dictionary LIKE/ILIKE or REGEXP_LIKE,
   // we need to translate the strings first.
 
   // This approach is a temporary solution until we can implement the next stage
   // of the string translation project, which will broaden the StringOper class to include
   // operations that operate on strings but do not neccessarily return strings like
   // LIKE/ILIKE/REGEXP_LIKE/CHAR_LENGTH At this point these aforementioned operators,
   // including LIKE/ILIKE, will just become part of a StringOps chain (which will also
   // avoid the overhead of serializing the transformed raw strings from previous string
   // opers to the dictionary to only read back out and perform LIKE/ILIKE.)
   CHECK_GT(string_oper->getArity(), 0UL);
   const auto& string_oper_primary_arg_ti = string_oper->getArg(0)->get_type_info();
   CHECK(string_oper_primary_arg_ti.is_dict_encoded_string());
   CHECK_NE(string_oper_primary_arg_ti.getStringDictKey().dict_id, TRANSIENT_DICT_ID);
   // Note the actual translation below will be cached by RowSetMemOwner
   translate_dict_strings(string_oper, ExecutorDeviceType::CPU, executor);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

RUNTIME_EXPORT int32_t string_compress	(	const StringView	string_view,
		const int64_t	string_dict_handle
	)

Definition at line 50 of file StringOpsIR.cpp.

References StringDictionaryProxy::getOrAddTransient(), and StringView::stringView().

                                                                                     {
   std::string_view const sv = string_view.stringView();
   if (sv.empty()) {
     return inline_int_null_value<int32_t>();
   }
   auto string_dict_proxy = reinterpret_cast<StringDictionaryProxy*>(string_dict_handle);
   return string_dict_proxy->getOrAddTransient(sv);
 }

Here is the call graph for this function:

RUNTIME_EXPORT StringView string_decode	(	int8_t *	chunk_iter_,
		int64_t	pos
	)

Definition at line 28 of file StringOpsIR.cpp.

References CHECK, ChunkIter_get_nth(), VarlenDatum::is_null, VarlenDatum::length, and VarlenDatum::pointer.

                                                                                      {
   auto chunk_iter = reinterpret_cast<ChunkIter*>(chunk_iter_);
   VarlenDatum vd;
   bool is_end;
   ChunkIter_get_nth(chunk_iter, pos, false, &vd, &is_end);
   CHECK(!is_end);
   return vd.is_null ? StringView{nullptr, 0u}
                     : StringView{reinterpret_cast<char const*>(vd.pointer), vd.length};
 }

Here is the call graph for this function:

RUNTIME_EXPORT StringView string_decompress	(	const int32_t	string_id,
		const int64_t	string_dict_handle
	)

Definition at line 38 of file StringOpsIR.cpp.

References CHECK, StringDictionaryProxy::getStringBytes(), and NULL_INT.

                                                                                          {
   if (string_id == NULL_INT) {
     return {nullptr, 0};
   }
   auto string_dict_proxy =
       reinterpret_cast<const StringDictionaryProxy*>(string_dict_handle);
   auto string_bytes = string_dict_proxy->getStringBytes(string_id);
   CHECK(string_bytes.first);
   return {string_bytes.first, string_bytes.second};
 }

Here is the call graph for this function:

std::unique_ptr<StringDictionaryTranslationMgr> translate_dict_strings	(	const Analyzer::StringOper *	expr,
		const ExecutorDeviceType	device_type,
		Executor *	executor
	)

Definition at line 447 of file StringOpsIR.cpp.

References CHECK, Data_Namespace::CPU_LEVEL, Analyzer::Expr::get_type_info(), Analyzer::StringOper::getArg(), SQLTypeInfo::getStringDictKey(), getStringOpInfos(), GPU, and Data_Namespace::GPU_LEVEL.

Referenced by CodeGenerator::codegen(), and pre_translate_string_ops().

                         {
   const auto& expr_ti = expr->get_type_info();
   const auto& primary_input_expr_ti = expr->getArg(0)->get_type_info();
   const auto& dict_id = primary_input_expr_ti.getStringDictKey();
   const auto string_op_infos = getStringOpInfos(expr);
   CHECK(string_op_infos.size());
 
   if (string_op_infos.back().getReturnType().is_dict_encoded_string()) {
     // string->string translation
     auto string_dictionary_translation_mgr =
         std::make_unique<StringDictionaryTranslationMgr>(
             dict_id,
             dict_id,
             false,  // translate_intersection_only
             expr_ti,
             string_op_infos,
             device_type == ExecutorDeviceType::GPU ? Data_Namespace::GPU_LEVEL
                                                    : Data_Namespace::CPU_LEVEL,
             executor->deviceCount(device_type),
             executor,
             executor->getDataMgr(),
             false /* delay_translation */);
     return string_dictionary_translation_mgr;
   } else {
     // string->numeric translation
     auto string_dictionary_translation_mgr =
         std::make_unique<StringDictionaryTranslationMgr>(
             dict_id,
             expr_ti,
             string_op_infos,
             device_type == ExecutorDeviceType::GPU ? Data_Namespace::GPU_LEVEL
                                                    : Data_Namespace::CPU_LEVEL,
             executor->deviceCount(device_type),
             executor,
             executor->getDataMgr(),
             false /* delay_translation */);
     return string_dictionary_translation_mgr;
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

RUNTIME_EXPORT int32_t union_translate_string_id_to_other_dict	(	const int32_t	string_id,
		const int64_t	source_string_dict_handle,
		const int64_t	dest_string_dict_handle
	)

Definition at line 112 of file StringOpsIR.cpp.

References StringDictionaryProxy::getString().

                                                                                {
   const auto source_string_dict_proxy =
       reinterpret_cast<StringDictionaryProxy*>(source_string_dict_handle);
   auto dest_string_dict_proxy =
       reinterpret_cast<StringDictionaryProxy*>(dest_string_dict_handle);
   // Can we have StringDictionaryProxy::getString return a reference?
   const auto source_str = source_string_dict_proxy->getString(string_id);
   if (source_str.empty()) {
     return inline_int_null_value<int32_t>();
   }
   return dest_string_dict_proxy->getOrAddTransient(source_str);
 }

Here is the call graph for this function:

int32_t write_string_to_proxy	(	const std::string &	str,
		const int64_t	string_dict_handle
	)

inline

Definition at line 148 of file StringOpsIR.cpp.

References StringDictionaryProxy::getOrAddTransient().

Referenced by convert_to_string_and_encode_bool(), convert_to_string_and_encode_date(), convert_to_string_and_encode_decimal(), convert_to_string_and_encode_time(), and convert_to_string_and_encode_timestamp().

                                                                        {
   if (str.empty()) {
     return inline_int_null_value<int32_t>();
   }
   auto string_dict_proxy = reinterpret_cast<StringDictionaryProxy*>(string_dict_handle);
   return string_dict_proxy->getOrAddTransient(str);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

Namespaces

Macros

Functions

Macro Definition Documentation

Function Documentation