StringOps_Namespace Namespace Reference

Namespaces
	anonymous_namespace{StringOps.cpp}

Classes
struct	StringOpInfo

Typedefs
using	LiteralArgMap = std::map< size_t, std::pair< SQLTypes, Datum >>

Functions
std::ostream &	operator<< (std::ostream &stream, const StringOpInfo &string_op_info)
std::ostream &	operator<< (std::ostream &stream, const std::vector< StringOpInfo > &string_op_infos)
std::string	toString (const std::vector< StringOpInfo > &string_op_infos)
double	compute_jaro_score (std::string_view s1, std::string_view s2)
double	compute_jaro_winkler_score (std::string_view s1, std::string_view s2)
template<typename T >
T	compute_levenshtein_distance_template (std::string_view s1, std::string_view s2)
int64_t	compute_levenshtein_distance (std::string_view s1, std::string_view s2)
std::unique_ptr< const StringOp >	gen_string_op (const StringOpInfo &string_op_info)
std::pair< std::string, bool >	apply_string_op_to_literals (const StringOpInfo &string_op_info)
Datum	apply_numeric_op_to_literals (const StringOpInfo &string_op_info)

Variables
constexpr int	winkler_k_prefix_length = 4
constexpr double	winkler_k_scaling_factor = 0.1

Typedef Documentation

using StringOps_Namespace::LiteralArgMap = typedef std::map<size_t, std::pair<SQLTypes, Datum>>

Definition at line 30 of file StringOpInfo.h.

Function Documentation

Datum StringOps_Namespace::apply_numeric_op_to_literals

(

const StringOpInfo &

string_op_info

)

Definition at line 1283 of file StringOps.cpp.

References CHECK, gen_string_op(), and StringOps_Namespace::StringOpInfo::hasVarStringLiteral().

Referenced by anonymous_namespace{ExpressionRewrite.cpp}::ConstantFoldingVisitor::visitStringOper().

                                                                       {
  CHECK(string_op_info.hasVarStringLiteral());
  const auto string_op = gen_string_op(string_op_info);
  return string_op->numericEval();
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::pair<std::string, bool > StringOps_Namespace::apply_string_op_to_literals

(

const StringOpInfo &

string_op_info

)

Definition at line 1272 of file StringOps.cpp.

References CHECK, gen_string_op(), StringOps_Namespace::StringOpInfo::hasNullLiteralArg(), and StringOps_Namespace::StringOpInfo::hasVarStringLiteral().

Referenced by TransientStringLiteralsVisitor::visitStringOper(), and anonymous_namespace{ExpressionRewrite.cpp}::ConstantFoldingVisitor::visitStringOper().

                                        {
  CHECK(string_op_info.hasVarStringLiteral());
  if (string_op_info.hasNullLiteralArg()) {
    const std::string null_str{""};
    return std::make_pair(null_str, true);
  }
  const auto string_op = gen_string_op(string_op_info);
  return string_op->operator()().toPair();
}

Here is the call graph for this function:

Here is the caller graph for this function:

double StringOps_Namespace::compute_jaro_score	(	std::string_view	s1,
		std::string_view	s2
	)

Definition at line 127 of file StringOps.cpp.

References score.

Referenced by compute_jaro_winkler_score().

                                                                {
  int s1_len = s1.size();
  int s2_len = s2.size();

  if (s1_len == 0 || s2_len == 0) {
    return 0.0;
  }

  int match_distance = std::max(s1_len, s2_len) / 2 - 1;
  std::vector<bool> s1_match(s1_len, false);
  std::vector<bool> s2_match(s2_len, false);

  int matches = 0;
  int transpositions = 0;

  for (int i = 0; i < s1_len; ++i) {
    int start = std::max(0, i - match_distance);
    int end = std::min(i + match_distance + 1, s2_len);

    for (int j = start; j < end; ++j) {
      if (s2_match[j]) {
        continue;
      }
      if (s1[i] != s2[j]) {
        continue;
      }
      s1_match[i] = true;
      s2_match[j] = true;
      ++matches;
      break;
    }
  }

  if (matches == 0) {
    return 0.0;
  }

  int k = 0;
  for (int i = 0; i < s1_len; ++i) {
    if (!s1_match[i]) {
      continue;
    }
    while (!s2_match[k]) {
      ++k;
    }
    if (s1[i] != s2[k]) {
      ++transpositions;
    }
    ++k;
  }

  double score = ((matches / (double)s1_len) + (matches / (double)s2_len) +
                  ((matches - transpositions / 2.0) / matches)) /
                 3.0;

  return score;
}

Here is the caller graph for this function:

double StringOps_Namespace::compute_jaro_winkler_score	(	std::string_view	s1,
		std::string_view	s2
	)

Definition at line 185 of file StringOps.cpp.

References Datum::bigintval, compute_jaro_score(), anonymous_namespace{Utm.h}::n, NullDatum(), UNREACHABLE, winkler_k_prefix_length, and winkler_k_scaling_factor.

                                                                        {
  double jaro_score = compute_jaro_score(s1, s2);

  int l = 0;
  int n = std::min({static_cast<int>(s1.size()),
                    static_cast<int>(s2.size()),
                    winkler_k_prefix_length});

  for (; l < n; ++l) {
    if (s1[l] != s2[l]) {
      break;
    }
  }

  double winkler_adjustment = l * winkler_k_scaling_factor * (1 - jaro_score);
  double jaro_winkler_score = jaro_score + winkler_adjustment;

  return jaro_winkler_score * 100;
}

Here is the call graph for this function:

int64_t StringOps_Namespace::compute_levenshtein_distance	(	std::string_view	s1,
		std::string_view	s2
	)

Definition at line 278 of file StringOps.cpp.

                                                                           {
  const size_t max_len = std::max(s1.size(), s2.size());

  if (max_len < 256) {
    return compute_levenshtein_distance_template<uint8_t>(s1, s2);
  } else if (max_len < 65536) {
    return compute_levenshtein_distance_template<uint16_t>(s1, s2);
  } else if (max_len < std::numeric_limits<uint32_t>::max()) {
    return compute_levenshtein_distance_template<uint32_t>(s1, s2);
  } else {
    return compute_levenshtein_distance_template<uint64_t>(s1, s2);
  }
}

template<typename T >

T StringOps_Namespace::compute_levenshtein_distance_template	(	std::string_view	s1,
		std::string_view	s2
	)

Definition at line 255 of file StringOps.cpp.

                                                                              {
  const size_t len1 = s1.size(), len2 = s2.size();
  std::vector<std::vector<T>> d(len1 + 1, std::vector<T>(len2 + 1));

  d[0][0] = 0;
  for (size_t i = 1; i <= len1; ++i) {
    d[i][0] = i;
  }
  for (size_t i = 1; i <= len2; ++i) {
    d[0][i] = i;
  }

  for (size_t i = 1; i <= len1; ++i) {
    for (size_t j = 1; j <= len2; ++j) {
      d[i][j] = std::min({d[i - 1][j] + 1,
                          d[i][j - 1] + 1,
                          d[i - 1][j - 1] + (s1[i - 1] == s2[j - 1] ? 0 : 1)});
    }
  }

  return d[len1][len2];
}

std::unique_ptr<const StringOp> StringOps_Namespace::gen_string_op

(

const StringOpInfo &

string_op_info

)

Definition at line 1039 of file StringOps.cpp.

References BASE64_DECODE, BASE64_ENCODE, CHECK_EQ, CHECK_GE, CHECK_LE, CONCAT, StringOps_Namespace::StringOpInfo::getIntLiteral(), StringOps_Namespace::StringOpInfo::getOpKind(), StringOps_Namespace::StringOpInfo::getReturnType(), StringOps_Namespace::StringOpInfo::getStringLiteral(), HASH, StringOps_Namespace::StringOpInfo::hasNullLiteralArg(), StringOps_Namespace::StringOpInfo::hasVarStringLiteral(), INITCAP, StringOps_Namespace::StringOpInfo::intLiteralArgAtIdxExists(), JAROWINKLER_SIMILARITY, JSON_VALUE, LEVENSHTEIN_DISTANCE, LOWER, LPAD, LTRIM, StringOps_Namespace::StringOpInfo::numLiterals(), StringOps_Namespace::StringOpInfo::numNonVariableLiterals(), OVERLAY, POSITION, RCONCAT, REGEXP_COUNT, REGEXP_REPLACE, REGEXP_SUBSTR, REPEAT, REPLACE, REVERSE, RPAD, RTRIM, SPLIT_PART, SUBSTRING, TRIM, TRY_STRING_CAST, UNREACHABLE, UPPER, URL_DECODE, and URL_ENCODE.

Referenced by apply_numeric_op_to_literals(), and apply_string_op_to_literals().

                                                                                {
  std::optional<std::string> var_string_optional_literal;
  const auto op_kind = string_op_info.getOpKind();
  const auto& return_ti = string_op_info.getReturnType();

  if (string_op_info.hasNullLiteralArg()) {
    return std::make_unique<const NullOp>(
        return_ti, var_string_optional_literal, op_kind);
  }

  const auto num_non_variable_literals = string_op_info.numNonVariableLiterals();
  if (string_op_info.hasVarStringLiteral()) {
    CHECK_EQ(num_non_variable_literals + 1UL, string_op_info.numLiterals());
    var_string_optional_literal = string_op_info.getStringLiteral(0);
  }

  switch (op_kind) {
    case SqlStringOpKind::LOWER: {
      CHECK_EQ(num_non_variable_literals, 0UL);
      return std::make_unique<const Lower>(var_string_optional_literal);
    }
    case SqlStringOpKind::UPPER: {
      CHECK_EQ(num_non_variable_literals, 0UL);
      return std::make_unique<const Upper>(var_string_optional_literal);
    }
    case SqlStringOpKind::INITCAP: {
      CHECK_EQ(num_non_variable_literals, 0UL);
      return std::make_unique<const InitCap>(var_string_optional_literal);
    }
    case SqlStringOpKind::REVERSE: {
      CHECK_EQ(num_non_variable_literals, 0UL);
      return std::make_unique<const Reverse>(var_string_optional_literal);
    }
    case SqlStringOpKind::REPEAT: {
      CHECK_EQ(num_non_variable_literals, 1UL);
      const auto num_repeats_literal = string_op_info.getIntLiteral(1);
      return std::make_unique<const Repeat>(var_string_optional_literal,
                                            num_repeats_literal);
    }
    case SqlStringOpKind::CONCAT:
    case SqlStringOpKind::RCONCAT: {
      CHECK_GE(num_non_variable_literals, 0UL);
      CHECK_LE(num_non_variable_literals, 1UL);
      if (num_non_variable_literals == 1UL) {
        const auto str_literal = string_op_info.getStringLiteral(1);
        // Handle lhs literals by having RCONCAT operator set a flag
        return std::make_unique<const Concat>(var_string_optional_literal,
                                              str_literal,
                                              op_kind == SqlStringOpKind::RCONCAT);
      } else {
        return std::make_unique<const Concat>(var_string_optional_literal);
      }
    }
    case SqlStringOpKind::LPAD:
    case SqlStringOpKind::RPAD: {
      CHECK_EQ(num_non_variable_literals, 2UL);
      const auto padded_length_literal = string_op_info.getIntLiteral(1);
      const auto padding_string_literal = string_op_info.getStringLiteral(2);
      return std::make_unique<Pad>(var_string_optional_literal,
                                   op_kind,
                                   padded_length_literal,
                                   padding_string_literal);
    }
    case SqlStringOpKind::TRIM:
    case SqlStringOpKind::LTRIM:
    case SqlStringOpKind::RTRIM: {
      CHECK_EQ(num_non_variable_literals, 1UL);
      const auto trim_chars_literal = string_op_info.getStringLiteral(1);
      return std::make_unique<Trim>(
          var_string_optional_literal, op_kind, trim_chars_literal);
    }
    case SqlStringOpKind::SUBSTRING: {
      CHECK_GE(num_non_variable_literals, 1UL);
      CHECK_LE(num_non_variable_literals, 2UL);
      const auto start_pos_literal = string_op_info.getIntLiteral(1);
      const bool has_length_literal = string_op_info.intLiteralArgAtIdxExists(2);
      if (has_length_literal) {
        const auto length_literal = string_op_info.getIntLiteral(2);
        return std::make_unique<const Substring>(
            var_string_optional_literal, start_pos_literal, length_literal);
      } else {
        return std::make_unique<const Substring>(var_string_optional_literal,
                                                 start_pos_literal);
      }
    }
    case SqlStringOpKind::OVERLAY: {
      CHECK_GE(num_non_variable_literals, 2UL);
      CHECK_LE(num_non_variable_literals, 3UL);
      const auto replace_string_literal = string_op_info.getStringLiteral(1);
      const auto start_pos_literal = string_op_info.getIntLiteral(2);
      const bool has_length_literal = string_op_info.intLiteralArgAtIdxExists(3);
      if (has_length_literal) {
        const auto length_literal = string_op_info.getIntLiteral(3);
        return std::make_unique<const Overlay>(var_string_optional_literal,
                                               replace_string_literal,
                                               start_pos_literal,
                                               length_literal);
      } else {
        return std::make_unique<const Overlay>(
            var_string_optional_literal, replace_string_literal, start_pos_literal);
      }
    }
    case SqlStringOpKind::REPLACE: {
      CHECK_GE(num_non_variable_literals, 2UL);
      CHECK_LE(num_non_variable_literals, 2UL);
      const auto pattern_string_literal = string_op_info.getStringLiteral(1);
      const auto replacement_string_literal = string_op_info.getStringLiteral(2);
      return std::make_unique<const Replace>(var_string_optional_literal,
                                             pattern_string_literal,
                                             replacement_string_literal);
    }
    case SqlStringOpKind::SPLIT_PART: {
      CHECK_GE(num_non_variable_literals, 2UL);
      CHECK_LE(num_non_variable_literals, 2UL);
      const auto delimiter_literal = string_op_info.getStringLiteral(1);
      const auto split_part_literal = string_op_info.getIntLiteral(2);
      return std::make_unique<const SplitPart>(
          var_string_optional_literal, delimiter_literal, split_part_literal);
    }
    case SqlStringOpKind::REGEXP_REPLACE: {
      CHECK_GE(num_non_variable_literals, 5UL);
      CHECK_LE(num_non_variable_literals, 5UL);
      const auto pattern_literal = string_op_info.getStringLiteral(1);
      const auto replacement_literal = string_op_info.getStringLiteral(2);
      const auto start_pos_literal = string_op_info.getIntLiteral(3);
      const auto occurrence_literal = string_op_info.getIntLiteral(4);
      const auto regex_params_literal = string_op_info.getStringLiteral(5);
      return std::make_unique<const RegexpReplace>(var_string_optional_literal,
                                                   pattern_literal,
                                                   replacement_literal,
                                                   start_pos_literal,
                                                   occurrence_literal,
                                                   regex_params_literal);
    }
    case SqlStringOpKind::REGEXP_SUBSTR: {
      CHECK_GE(num_non_variable_literals, 5UL);
      CHECK_LE(num_non_variable_literals, 5UL);
      const auto pattern_literal = string_op_info.getStringLiteral(1);
      const auto start_pos_literal = string_op_info.getIntLiteral(2);
      const auto occurrence_literal = string_op_info.getIntLiteral(3);
      const auto regex_params_literal = string_op_info.getStringLiteral(4);
      const auto sub_match_idx_literal = string_op_info.getIntLiteral(5);
      return std::make_unique<const RegexpSubstr>(var_string_optional_literal,
                                                  pattern_literal,
                                                  start_pos_literal,
                                                  occurrence_literal,
                                                  regex_params_literal,
                                                  sub_match_idx_literal);
    }
    case SqlStringOpKind::REGEXP_COUNT: {
      CHECK_GE(num_non_variable_literals, 3UL);
      CHECK_LE(num_non_variable_literals, 3UL);
      const auto pattern_literal = string_op_info.getStringLiteral(1);
      const auto start_pos_literal = string_op_info.getIntLiteral(2);
      const auto regex_params_literal = string_op_info.getStringLiteral(3);
      return std::make_unique<const RegexpCount>(var_string_optional_literal,
                                                 pattern_literal,
                                                 start_pos_literal,
                                                 regex_params_literal);
    }
    case SqlStringOpKind::JSON_VALUE: {
      CHECK_EQ(num_non_variable_literals, 1UL);
      const auto json_path_literal = string_op_info.getStringLiteral(1);
      return std::make_unique<const JsonValue>(var_string_optional_literal,
                                               json_path_literal);
    }
    case SqlStringOpKind::BASE64_ENCODE: {
      CHECK_EQ(num_non_variable_literals, 0UL);
      return std::make_unique<const Base64Encode>(var_string_optional_literal);
    }
    case SqlStringOpKind::BASE64_DECODE: {
      CHECK_EQ(num_non_variable_literals, 0UL);
      return std::make_unique<const Base64Decode>(var_string_optional_literal);
    }
    case SqlStringOpKind::URL_ENCODE: {
      CHECK_EQ(num_non_variable_literals, 0UL);
      return std::make_unique<const UrlEncode>(var_string_optional_literal);
    }
    case SqlStringOpKind::URL_DECODE: {
      CHECK_EQ(num_non_variable_literals, 0UL);
      return std::make_unique<const UrlDecode>(var_string_optional_literal);
    }
    case SqlStringOpKind::TRY_STRING_CAST: {
      CHECK_EQ(num_non_variable_literals, 0UL);
      return std::make_unique<const TryStringCast>(return_ti,
                                                   var_string_optional_literal);
    }
    case SqlStringOpKind::POSITION: {
      CHECK_GE(num_non_variable_literals, 1UL);
      CHECK_LE(num_non_variable_literals, 2UL);
      const auto search_literal = string_op_info.getStringLiteral(1);
      const bool has_start_pos_literal = string_op_info.intLiteralArgAtIdxExists(2);
      if (has_start_pos_literal) {
        const auto start_pos_literal = string_op_info.getIntLiteral(2);
        return std::make_unique<const Position>(
            var_string_optional_literal, search_literal, start_pos_literal);
      } else {
        return std::make_unique<const Position>(var_string_optional_literal,
                                                search_literal);
      }
    }
    case SqlStringOpKind::JAROWINKLER_SIMILARITY: {
      CHECK_GE(num_non_variable_literals, 0UL);
      CHECK_LE(num_non_variable_literals, 1UL);
      if (num_non_variable_literals == 1UL) {
        const auto str_literal = string_op_info.getStringLiteral(1);
        return std::make_unique<const JarowinklerSimilarity>(var_string_optional_literal,
                                                             str_literal);
      } else {
        return std::make_unique<const JarowinklerSimilarity>(var_string_optional_literal);
      }
    }
    case SqlStringOpKind::LEVENSHTEIN_DISTANCE: {
      CHECK_GE(num_non_variable_literals, 0UL);
      CHECK_LE(num_non_variable_literals, 1UL);
      if (num_non_variable_literals == 1UL) {
        const auto str_literal = string_op_info.getStringLiteral(1);
        return std::make_unique<const LevenshteinDistance>(var_string_optional_literal,
                                                           str_literal);
      } else {
        return std::make_unique<const LevenshteinDistance>(var_string_optional_literal);
      }
    }
    case SqlStringOpKind::HASH: {
      CHECK_EQ(num_non_variable_literals, 0UL);
      return std::make_unique<const Hash>(var_string_optional_literal);
    }
    default:
      UNREACHABLE();
      return {};
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::ostream& StringOps_Namespace::operator<<	(	std::ostream &	stream,
		const StringOpInfo &	string_op_info
	)

Definition at line 24 of file StringOpInfo.cpp.

References CHECK, extract_int_type_from_datum(), SQLTypeInfo::get_dimension(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_type(), StringOps_Namespace::StringOpInfo::getOpKind(), StringOps_Namespace::StringOpInfo::getReturnType(), IS_INTEGER, IS_STRING, StringOps_Namespace::StringOpInfo::isLiteralArgNull(), StringOps_Namespace::StringOpInfo::literal_arg_map_, and toString().

                                                                               {
  stream << "StringOp("
         << "operator: " << string_op_info.getOpKind()
         << "return_ti: " << toString(string_op_info.getReturnType().get_type())
         << " dim: " << string_op_info.getReturnType().get_dimension()
         << " scale: " << string_op_info.getReturnType().get_scale() << ", literals: [";
  bool first_elem = true;
  for (const auto& literal_arg : string_op_info.literal_arg_map_) {
    if (!first_elem) {
      stream << ", ";
    }
    first_elem = false;
    const auto datum_type = literal_arg.second.first;
    const auto& datum = literal_arg.second.second;
    stream << "{slot: " << literal_arg.first /* slot/idx */ << ", type: "
           << ::toString(datum_type) << ", value: ";
    if (string_op_info.isLiteralArgNull(datum_type, literal_arg.second.second)) {
      stream << "NULL";
    } else if (IS_STRING(datum_type)) {
      stream << *datum.stringval;
    } else {
      CHECK(IS_INTEGER(datum_type));
      const SQLTypeInfo ti(datum_type, false);
      stream << extract_int_type_from_datum(datum, ti);
    }
    stream << "}";
  }
  stream << "]";
  return stream;
}

Here is the call graph for this function:

std::ostream & StringOps_Namespace::operator<<	(	std::ostream &	stream,
		const std::vector< StringOpInfo > &	string_op_infos
	)

Definition at line 55 of file StringOpInfo.cpp.

                                                                         {
  stream << "[";
  bool first_elem = true;
  for (const auto& string_op_info : string_op_infos) {
    if (!first_elem) {
      stream << ", ";
    }
    first_elem = false;
    stream << string_op_info;
  }
  stream << "]";
  return stream;
}

std::string StringOps_Namespace::toString

(

const std::vector< StringOpInfo > &

string_op_infos

)

Definition at line 70 of file StringOpInfo.cpp.

Referenced by operator<<().

                                                                   {
  std::ostringstream oss;
  oss << string_op_infos;
  return oss.str();
}

Here is the caller graph for this function:

Variable Documentation

constexpr int StringOps_Namespace::winkler_k_prefix_length = 4

Definition at line 122 of file StringOps.cpp.

Referenced by compute_jaro_winkler_score().

constexpr double StringOps_Namespace::winkler_k_scaling_factor = 0.1

Definition at line 125 of file StringOps.cpp.

Referenced by compute_jaro_winkler_score().